From e1e5cbb36c374eafbb76dc35edc9a7d8695b9464 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Wed, 16 Sep 2026 19:37:06 +0200 Subject: [PATCH 01/27] fix musl compilation warning --- crates/exav/src/daemon.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/exav/src/daemon.rs b/crates/exav/src/daemon.rs index 77df8720..0e4a7c36 100644 --- a/crates/exav/src/daemon.rs +++ b/crates/exav/src/daemon.rs @@ -724,7 +724,7 @@ pub(crate) fn datadir_mtime(dir: &std::path::Path) -> Option Date: Sat, 19 Sep 2026 12:38:56 +0200 Subject: [PATCH 02/27] Performance optimizations --- Cargo.lock | 2 + crates/exav-core/Cargo.toml | 3 + crates/exav-core/examples/diag.rs | 30 + crates/exav-core/src/engine/logic.rs | 40 + crates/exav-core/src/engine/mod.rs | 1078 ++++++++++++++++++++-- crates/exav-core/src/engine/parse.rs | 216 ++++- crates/exav-core/src/lib.rs | 167 +++- crates/exav-core/src/profile.rs | 82 +- crates/exav-pe-emu/Cargo.toml | 5 + crates/exav-pe-emu/src/cpu.rs | 6 +- crates/exav-pe-emu/src/mem.rs | 6 +- crates/exav-pe-emu/src/run.rs | 21 +- crates/exav-pe-emu/src/win.rs | 14 +- crates/exav-unpack/src/formats/pdf.rs | 260 +++++- crates/exav-unpack/src/formats/pepack.rs | 9 +- crates/exav-unpack/src/lib.rs | 71 +- crates/exav-unpack/src/profile.rs | 85 ++ crates/exav/src/main.rs | 69 +- crates/exav/src/metrics.rs | 6 + crates/exav/src/perf.rs | 8 +- crates/exav/tests/json_output.rs | 87 ++ 21 files changed, 1989 insertions(+), 276 deletions(-) create mode 100644 crates/exav-unpack/src/profile.rs diff --git a/Cargo.lock b/Cargo.lock index f590e41a..a06d058f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -560,6 +560,7 @@ dependencies = [ "regex-automata", "regex-syntax", "rmp-serde", + "rustc-hash 2.1.2", "rustdct", "serde", "serde_json", @@ -589,6 +590,7 @@ name = "exav-pe-emu" version = "0.0.1" dependencies = [ "exav-x86", + "rustc-hash 2.1.2", ] [[package]] diff --git a/crates/exav-core/Cargo.toml b/crates/exav-core/Cargo.toml index 20357954..eb3440e0 100644 --- a/crates/exav-core/Cargo.toml +++ b/crates/exav-core/Cargo.toml @@ -53,6 +53,9 @@ transpose = "0.2" # HTTP(S) range-request backend (optional) ureq = { version = "2", optional = true } daachorse = "3" +# Anchor-frequency counting at database build: tens of millions of u64 gram +# keys, where the hash is the whole cost of the lookup. +rustc-hash = "2" serde = { version = "1.0.228", features = ["derive"] } rmp-serde = "1.3.1" regex = "1.12.4" diff --git a/crates/exav-core/examples/diag.rs b/crates/exav-core/examples/diag.rs index d3e142c4..064a4388 100644 --- a/crates/exav-core/examples/diag.rs +++ b/crates/exav-core/examples/diag.rs @@ -38,6 +38,16 @@ fn main() { } eprintln!(" REAL scan (min of 5): {real_ms:.1} ms"); + // Automaton pass alone (no verification), min of 3. + let mut ac_ms = f64::MAX; + let mut ac_hits = 0; + for _ in 0..3 { + let t = Instant::now(); + ac_hits = db.engine().scan_diag_hits(&data, ft); + ac_ms = ac_ms.min(t.elapsed().as_secs_f64() * 1000.0); + } + eprintln!(" AC pass only (min of 3): {ac_ms:.1} ms for {ac_hits} hits"); + let t = Instant::now(); let d = db.engine().scan_diag(&data, ft, layout.as_ref()); let ms = t.elapsed().as_secs_f64() * 1000.0; @@ -67,6 +77,26 @@ fn main() { eprintln!(" {len:>2}: {hits:>9} {fan:>11}"); } } + let (fanout, prunable, ldbs, dead) = db.engine().scan_diag_gate(&data, ft); + eprintln!( + " lsig anchor gate: {prunable}/{fanout} fan-out prunable ({:.1}%), {dead}/{ldbs} lsigs dead", + 100.0 * prunable as f64 / fanout.max(1) as f64 + ); + eprintln!(" hottest anchors (hits x gsize = fanout | bytes):"); + for (bytes, hits, gsize, bids) in db.engine().scan_diag_anchors(&data, ft) { + eprintln!( + " {hits:>8} x {gsize:>4} = {:>10} | {:?}", + hits * gsize as u64, + String::from_utf8_lossy(&bytes) + ); + for b in bids { + eprintln!( + " body[{b}] = {}\n {}", + db.engine().describe_body(b), + db.engine().describe_body_literals(b) + ); + } + } eprintln!(" top fan-out groups (alen gsize hits fanout | best_off coverage%):"); for (alen, gsize, hits, fan, off, cov) in db.engine().scan_diag_groups(&data, ft) { eprintln!(" alen={alen} gsize={gsize:>5} hits={hits:>7} fanout={fan:>10} | off={off:>4} cov={cov}%"); diff --git a/crates/exav-core/src/engine/logic.rs b/crates/exav-core/src/engine/logic.rs index cb7ca18f..3baacc0c 100644 --- a/crates/exav-core/src/engine/logic.rs +++ b/crates/exav-core/src/engine/logic.rs @@ -97,6 +97,46 @@ impl Node { } } + /// Sound satisfiability over-approximation for pruning *before* any body + /// subsignature is verified. `absent(i)` marks a subsig whose count is + /// certainly 0 because no body of it has an anchor hit anywhere in the + /// buffer; every other subsig is free to take any count. + /// + /// Distinct from [`Self::can_be_true`], whose unknowns are binary: a body + /// subsig can match many times, so a 0-or-1 unknown would wrongly prune + /// `SubCmp(i, Gt, 5)` and lose the signature. Returns false only when no + /// assignment of the free subsigs can satisfy the expression, so pruning + /// stays FN-safe; `total` and `distinct` in a group are over-approximated + /// independently, which can only yield a false "possible". + pub(super) fn can_be_true_absent(&self, absent: &dyn Fn(usize) -> bool) -> bool { + // A free count ranges over all of `u32`, so a comparison is satisfiable + // unless the bound itself rules every value out. + let free_cmp_ok = |c: Cmp, x: u32| match c { + Cmp::Eq => true, + Cmp::Gt => x < u32::MAX, + Cmp::Lt => x > 0, + }; + match self { + Node::Sub(i) => !absent(*i), + Node::SubCmp(i, c, x) => { + if absent(*i) { + cmp_ok(*c, 0, *x) + } else { + free_cmp_ok(*c, *x) + } + } + Node::GroupCmp(ids, c, x, y) => { + let nfree = ids.iter().filter(|&&i| !absent(i)).count() as u32; + if nfree == 0 { + return cmp_ok(*c, 0, *x) && y.is_none_or(|y| y == 0); + } + free_cmp_ok(*c, *x) && y.is_none_or(|y| nfree >= y) + } + Node::And(v) => v.iter().all(|n| n.can_be_true_absent(absent)), + Node::Or(v) => v.iter().any(|n| n.can_be_true_absent(absent)), + } + } + fn collect_ids(&self, out: &mut Vec) { match self { Node::Sub(i) | Node::SubCmp(i, _, _) => out.push(*i), diff --git a/crates/exav-core/src/engine/mod.rs b/crates/exav-core/src/engine/mod.rs index 5efbd34d..bed74e8e 100644 --- a/crates/exav-core/src/engine/mod.rs +++ b/crates/exav-core/src/engine/mod.rs @@ -346,6 +346,135 @@ pub fn scan_was_truncated() -> bool { SCAN_TRUNCATED.with(|c| c.get()) } +/// Cap on simulator steps drawn by one anchor group per scan (see [`GroupCost`]). +const MAX_GROUP_STEPS: u64 = 16_000_000; + +/// Per-scan cap override, via `EXAV_MAX_GROUP_STEPS`. Read once. +fn max_group_steps() -> u64 { + use std::sync::OnceLock; + static B: OnceLock = OnceLock::new(); + *B.get_or_init(|| { + std::env::var("EXAV_MAX_GROUP_STEPS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(MAX_GROUP_STEPS) + }) +} + +/// Per-scan account of simulator steps drawn per anchor group: the bound on +/// concentrated repeated-boilerplate work. +/// +/// One anchor shared by many bodies — a mailing-list footer, an XML boilerplate +/// tag — can hit thousands of times per buffer, and every hit re-verifies every +/// body in the group over overlapping suffixes. The group whose cumulative +/// simulator draw passes the cap is skipped from then on, and the scan is +/// flagged truncated through the same [`SCAN_TRUNCATED`] channel as pool +/// exhaustion. A would-be `Clean` is then downgraded to `LimitsExceeded`, so +/// the cap is never a silent clean. Note the limit of that: only the `Clean` +/// arms consult [`scan_was_truncated`], so in `--all-matches` mode a capped +/// group leaves no trace once some other signature has matched. +/// +/// Keyed on cost, not hit count, deliberately: a hit count cannot tell a benign +/// repetition from a pathological one, because the same shape produces both. +/// Measured against a live ClamAV set, one 12 MB spreadsheet relationships part +/// (112,084 hyperlinks) hits a single group 112,084 times, and the dearest +/// single verify behind those anchors ran 25-35 ms — it is this cap, not the +/// step pool, that ends that scan. The neighbouring 31 MB sheet part, with +/// comparable hit counts but cheap bodies, finishes clean and untouched. So the +/// discriminator has to be accumulated cost: hit counts alone rank these two +/// the same way round. +struct GroupCost { + steps: std::collections::HashMap, + cap: u64, +} + +impl GroupCost { + fn new(cap: u64) -> Self { + Self { + steps: std::collections::HashMap::new(), + cap, + } + } + + /// Whether partition `pidx`'s group `value` may still verify: false once + /// its cumulative simulator draw passed the cap (the scan is then flagged + /// truncated — the search did not complete). + fn admit(&mut self, pidx: usize, value: u32) -> bool { + let key = ((pidx as u64) << 32) | value as u64; + if self.steps.get(&key).is_some_and(|&s| s > self.cap) { + SCAN_TRUNCATED.with(|c| c.set(true)); + return false; + } + true + } + + /// Charge simulator steps drawn (both pools — the token path draws from + /// exactly one of them) to partition `pidx`'s group `value`. + fn charge(&mut self, pidx: usize, value: u32, drawn: u64) { + if drawn == 0 { + return; + } + let key = ((pidx as u64) << 32) | value as u64; + *self.steps.entry(key).or_insert(0) += drawn; + } +} + +/// Whether the build-time anchor re-pick runs, via `EXAV_ANCHOR_STATS=0`. Read +/// once. Triage and measurement: with it off the anchors are the structural +/// ones chosen while each body was parsed, which is what the selection did +/// before the statistics existed. Matching is identical either way — an anchor +/// is a prefilter and verification re-checks the whole pattern — so this +/// changes only speed, and lets the two be compared with one binary. +fn anchor_stats_enabled() -> bool { + use std::sync::OnceLock; + static B: OnceLock = OnceLock::new(); + *B.get_or_init(|| { + std::env::var("EXAV_ANCHOR_STATS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(1) + != 0 + }) +} + +/// One automaton sweep's anchor hits, in iteration order: +/// `(partition index, anchor group value, match start)`. +/// +/// Buffered rather than consumed inline so the logical-signature gate can run +/// *between* the sweep and verification. The gate needs to know which anchors +/// hit anywhere in the buffer, which is only settled once the sweep has +/// finished — and sweeping a second time to learn it costs more than the +/// verification it saves (the sweep is over half of a scan's work on hit-dense +/// content). Replaying a buffer keeps the automaton pass at one, and preserves +/// hit order exactly, so first-match semantics and the order the step pools are +/// drawn down are unchanged. +struct Sweep { + hits: Vec<(u32, u32, u32)>, +} + +/// Cap on buffered anchor hits (12 bytes each, so ~48 MiB). Past it the scan +/// drops the buffer and runs the direct inline path instead: no gate, no +/// speedup, no growing memory — a hostile hit-dense file costs what it costs +/// without any of this, plus the abandoned partial sweep. +const MAX_BUFFERED_HITS: usize = 4_000_000; + +/// Hit count from which the gate is worth its own setup (a bitmap over every +/// body). Below it the sweep is replayed ungated: what gating would save could +/// not pay for zeroing the bitmap. +const GATE_MIN_HITS: usize = 50_000; + +/// Buffered-hit cap override, via `EXAV_MAX_BUFFERED_HITS`. Read once. +fn max_buffered_hits() -> usize { + use std::sync::OnceLock; + static B: OnceLock = OnceLock::new(); + *B.get_or_init(|| { + std::env::var("EXAV_MAX_BUFFERED_HITS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(MAX_BUFFERED_HITS) + }) +} + thread_local! { /// Container types of every ancestor of the buffer being scanned, outermost /// first — what `Intermediates:` is evaluated against. `None` marks an @@ -424,8 +553,16 @@ fn scan_verify_budget() -> u64 { /// Per-buffer pool for the gap-split simulator. The simulator is polynomial and /// FP-safe (a positive answer is always a real match), so its backstop sits far -/// above the backtracking pool — it exists only to bound a deliberately -/// adversarial input, and on real content is essentially never spent. Kept +/// above the backtracking pool. +/// +/// It is NOT only reached by adversarial input, despite what its size suggests: +/// against a live ClamAV set an ordinary 6.7 MB spreadsheet spends it, because +/// the 52 MB of XML it unpacks to repeats one shape often enough that short +/// URL/email anchors hit over a hundred thousand times each. That scan reports +/// `LimitsExceeded` rather than `Clean` — honestly, since the search really did +/// not finish, but on a benign file. Raising this pool does not fix such a +/// scan; it just buys more of the same work (measured: 16x the pool changed +/// nothing, because the per-group cap ends it first). Kept /// separate from the backtracking pool so the (safe, cheap) simulator's generous /// headroom can never let the (dangerous, exponential) backtracker run longer. /// At the simulator's throughput this bounds a worst-case buffer to a few @@ -544,6 +681,18 @@ enum Prefix { Internal { anchor_idx: u32 }, } +impl Prefix { + /// Index into `elems` of the literal serving as the anchor. The three + /// variants differ in what precedes it, not in having one. + fn anchor_idx(&self) -> u32 { + match *self { + Prefix::Fixed { anchor_idx, .. } + | Prefix::Floating { anchor_idx } + | Prefix::Internal { anchor_idx } => anchor_idx, + } + } +} + /// Offset constraint on where the pattern start may sit. The overwhelmingly /// common case is [`Offset::Any`] (no constraint); the rare constrained kinds /// are boxed so this enum stays one word, keeping `Body` small across the @@ -1698,7 +1847,96 @@ impl EngineBuilder { /// implied by which partition a body lives in. Within a partition, identical /// anchors de-duplicate into one automaton pattern that fans out to every /// body sharing it. + /// Every candidate anchor literal in the database, handed to `f` once per + /// body that contains it. Bodies whose `elems` were dropped (a pure literal + /// whose anchor IS the whole pattern) still contribute: their literal is in + /// `anchor_buf`, and they are a large share of the set, so leaving them out + /// would understate how common the commonest literals are. + fn for_each_candidate_literal( + bodies: &[Body], + anchor_buf: &[u8], + anchor_ranges: &[(u32, u32)], + f: &mut dyn FnMut(&[u8]), + ) { + for (id, body) in bodies.iter().enumerate() { + match &body.elems { + Some(elems) => { + for e in elems { + if let Elem::Bytes(b) = e { + if b.len() >= MIN_ANCHOR { + f(b); + } + } + } + } + None => { + let (s, l) = anchor_ranges[id]; + f(&anchor_buf[s as usize..(s + l) as usize]); + } + } + } + } + + /// Re-choose every anchor now that the whole database is known. + /// + /// A body's anchor is picked while it is parsed, when the only thing + /// available is the pattern itself — so the choice can only be structural + /// (longest, most varied run). How *selective* a literal is depends on the + /// rest of the set, which does not exist yet. This pass revisits the choice + /// with that knowledge: see [`parse::GramStats`]. + /// + /// Cannot change what the engine matches. The anchor is a prefilter and + /// verification re-checks the entire pattern whatever triggered it, so a + /// different anchor changes only how often the check runs. + fn repick_anchors(&mut self) { + if !anchor_stats_enabled() { + return; + } + let old_buf = std::mem::take(&mut self.anchor_buf); + let old_ranges = std::mem::take(&mut self.anchor_ranges); + let bodies = &mut self.bodies; + let stats = parse::GramStats::build(bodies.len(), |f| { + Self::for_each_candidate_literal(bodies, &old_buf, &old_ranges, f) + }); + let mut buf = Vec::with_capacity(old_buf.len()); + let mut ranges = Vec::with_capacity(old_ranges.len()); + for (id, body) in bodies.iter_mut().enumerate() { + let (s, l) = old_ranges[id]; + let old_anchor = &old_buf[s as usize..(s + l) as usize]; + // A pure literal has one candidate and it is already the anchor. + let anchor: &[u8] = match &body.elems { + None => old_anchor, + Some(elems) => { + match parse::pick_anchor( + elems, + matches!(body.offset, Offset::Any), + Some(&stats), + ) { + // `pick_anchor` succeeded once with no statistics, so a + // `None` here would mean the scores changed which + // candidates are admissible at all. They cannot — the + // `MIN_ANCHOR` gate is on length — but keep the working + // anchor rather than drop a signature if it ever did. + Some(p) => match &elems[p.anchor_idx() as usize] { + Elem::Bytes(b) => { + body.prefix = p; + b + } + _ => old_anchor, + }, + None => old_anchor, + } + } + }; + ranges.push((buf.len() as u32, anchor.len() as u32)); + buf.extend_from_slice(anchor); + } + self.anchor_buf = buf; + self.anchor_ranges = ranges; + } + pub fn build_with_budget(mut self, max_build_mem: Option) -> SigEngine { + self.repick_anchors(); // The double-array construction transient scales ~linearly with the // anchor count, at a rough `BUILD_BYTES_PER_ANCHOR`; capping anchors per // shard bounds that peak so a huge set (main+daily) builds on a small @@ -1979,19 +2217,30 @@ impl SigEngine { ft: FileType, buf: &'a [u8], lower: &'a [u8], - ) -> impl Iterator, &'a [Vec], &'a [u8])> { + ) -> impl Iterator< + Item = ( + usize, + &'a DoubleArrayAhoCorasick, + &'a [Vec], + &'a [u8], + ), + > { // A `Target:5` (graphics) partition is normally skipped — exav doesn't // model graphics as a `FileType`. But when the buffer IS an image, its // Target:5 signatures legitimately apply, so run those partitions too // (their subsignature anchors must match for an image-scoped LDB — e.g. a // `Win.Phishing.*` raw-image sig — to become a candidate). + // The yielded partition index keys the per-scan [`GroupCost`] accounts + // (partitions shard the automaton value space, so the value alone does + // not identify a group). let is_image = crate::fuzzy_img::looks_like_image(buf); self.partitions .iter() - .filter(move |p| target_ok(p.target, ft) || (p.target == 5 && is_image)) - .map(move |p| { + .enumerate() + .filter(move |(_, p)| target_ok(p.target, ft) || (p.target == 5 && is_image)) + .map(move |(i, p)| { let hay: &[u8] = if p.nocase { lower } else { buf }; - (&p.ac, p.groups.as_slice(), hay) + (i, &p.ac, p.groups.as_slice(), hay) }) } @@ -2099,6 +2348,30 @@ impl SigEngine { self.scan_with_layout(buf, ft, None, None) } + /// Describe a body's compiled pattern with its literal bytes spelled out + /// (diagnostic). [`describe_body`] reports literal *lengths*, which cannot + /// answer "is a rarer literal available to anchor on instead?" — this can. + pub fn describe_body_literals(&self, bid: usize) -> String { + let b = &self.bodies[bid]; + let Some(elems) = &b.elems else { + return "".to_string(); + }; + elems + .iter() + .map(|x| match x { + Elem::Bytes(v) => format!("{:?}", String::from_utf8_lossy(v)), + Elem::AnyByte => "?".to_string(), + Elem::HiNibble(_) | Elem::LoNibble(_) => "n".to_string(), + Elem::Gap { min, max } => format!("G{{{min},{max:?}}}"), + Elem::Alt { opts, neg } => { + format!("A{}{}", if *neg { "!" } else { "" }, opts.len()) + } + Elem::AltMasked { opts } => format!("Am{}", opts.len()), + }) + .collect::>() + .join(" ") + } + /// Describe a body's compiled pattern (diagnostic). pub fn describe_body(&self, bid: usize) -> String { let b = &self.bodies[bid]; @@ -2211,6 +2484,7 @@ impl SigEngine { let treject = 0u64; let (mut lit, mut tok, mut ok) = (0u64, 0u64, 0u64); let mut budgets = Budgets::new(); + let mut occ = OccCache::new(); for p in self.partitions.iter().filter(|p| target_ok(p.target, ft)) { let hay: &[u8] = if p.nocase { &lower } else { buf }; for m in p.ac.find_overlapping_iter(hay) { @@ -2229,7 +2503,7 @@ impl SigEngine { tok += 1; } let t = std::time::Instant::now(); - if verify(body, buf, m.start(), layout, &lower, &mut budgets).is_some() { + if verify(body, buf, m.start(), layout, &lower, &mut budgets, &mut occ).is_some() { ok += 1; } let us = t.elapsed().as_micros() as u64; @@ -2363,6 +2637,113 @@ impl SigEngine { rows } + /// Diagnostic: what an anchor-presence gate on logical signatures could + /// prune. Returns `(fanout, prunable_fanout, ldbs_touched, ldbs_prunable)`. + /// + /// A body subsignature is only worth verifying if its logical signature can + /// still fire; when a sibling subsig the expression requires has no anchor + /// hit anywhere in the buffer, its count is certainly 0 and the whole lsig + /// is dead. `prunable_fanout` is how many of the `(hit, body)` verifies that + /// would remove — the ceiling on what such a gate can save, before paying + /// for whatever pass computes it. + pub fn scan_diag_gate(&self, buf: &[u8], ft: FileType) -> (u64, u64, usize, usize) { + let lower = buf.to_ascii_lowercase(); + // Pass 1: which bodies have an anchor hit anywhere. + let mut hit = vec![false; self.bodies.len()]; + for (_pidx, ac, groups, hay) in self.active(ft, buf, &lower) { + for m in ac.find_overlapping_iter(hay) { + for &bid in &groups[m.value() as usize] { + hit[bid] = true; + } + } + } + // Which logical sigs are still satisfiable given that. + let mut alive: std::collections::HashMap = std::collections::HashMap::new(); + for (bid, &h) in hit.iter().enumerate() { + if !h { + continue; + } + let li = self.body_ldb[bid]; + if li == u32::MAX { + continue; + } + alive.entry(li).or_insert_with(|| { + let ldb = &self.ldbs[li as usize]; + let absent = |i: usize| match ldb.subs.get(i) { + // Only body subsigs are decided by anchor presence; a + // PCRE/bcomp/fuzzy subsig has no anchor, so it is never + // "certainly absent" and must stay free. + Some(SubSig::Bodies(ids)) => !ids.iter().any(|&b| hit[b]), + _ => false, + }; + ldb.expr.can_be_true_absent(&absent) + }); + } + // Pass 2: count the fan-out a gate would skip. + let (mut fanout, mut prunable) = (0u64, 0u64); + for (_pidx, ac, groups, hay) in self.active(ft, buf, &lower) { + for m in ac.find_overlapping_iter(hay) { + for &bid in &groups[m.value() as usize] { + fanout += 1; + let li = self.body_ldb[bid]; + if li != u32::MAX && !alive.get(&li).copied().unwrap_or(true) { + prunable += 1; + } + } + } + } + let dead = alive.values().filter(|&&a| !a).count(); + (fanout, prunable, alive.len(), dead) + } + + /// Diagnostic: total anchor hits, running the automatons and nothing else. + /// Isolates the Aho-Corasick pass from the verification that follows it, + /// which is what says whether a scheme that trades an extra automaton pass + /// for fewer verifies can pay for itself. + pub fn scan_diag_hits(&self, buf: &[u8], ft: FileType) -> u64 { + let lower = buf.to_ascii_lowercase(); + let mut n = 0u64; + for (_pidx, ac, _groups, hay) in self.active(ft, buf, &lower) { + for _m in ac.find_overlapping_iter(hay) { + n += 1; + } + } + n + } + + /// Diagnostic: the anchors that hit most in one scan, as `(anchor bytes, + /// hits, group size)`, most fan-out first. `scan_diag_groups` reports an + /// anchor's *length*; this reports its *bytes*, which is what shows whether + /// a bad prefilter is bad because it is short or because it is ubiquitous in + /// real content (`http`, `www.`) despite scoring well on byte variety. + pub fn scan_diag_anchors( + &self, + buf: &[u8], + ft: FileType, + ) -> Vec<(Vec, u64, usize, Vec)> { + let lower = buf.to_ascii_lowercase(); + let mut hits: std::collections::HashMap<(usize, u32), (u64, Vec)> = + std::collections::HashMap::new(); + for (pidx, ac, _groups, hay) in self.active(ft, buf, &lower) { + for m in ac.find_overlapping_iter(hay) { + let e = hits + .entry((pidx, m.value())) + .or_insert_with(|| (0, hay[m.start()..m.end()].to_vec())); + e.0 += 1; + } + } + let mut rows: Vec<(Vec, u64, usize, Vec)> = hits + .into_iter() + .map(|((pidx, val), (h, bytes))| { + let g = &self.partitions[pidx].groups[val as usize]; + (bytes, h, g.len(), g.iter().copied().take(4).collect()) + }) + .collect(); + rows.sort_by_key(|(_, h, g, _)| std::cmp::Reverse(h * *g as u64)); + rows.truncate(25); + rows + } + /// As [`scan`], with PE layout for resolving `EP`/section-relative offsets. /// Logical-sig indices to evaluate for a scan: those owning a matched subsig /// body (`touched`) plus the always-check `fires_empty` set. Sorted+deduped, @@ -2401,6 +2782,150 @@ impl SigEngine { cand } + /// Sweep the active automatons, buffering every hit. `None` when the + /// buffer cap (or a buffer too large for `u32` positions) rules it out, + /// which tells the caller to take the direct inline path. + fn sweep_hits(&self, ft: FileType, buf: &[u8], lower: &[u8]) -> Option { + if buf.len() > u32::MAX as usize { + return None; + } + let cap = max_buffered_hits(); + let mut hits: Vec<(u32, u32, u32)> = Vec::new(); + for (pidx, ac, _groups, hay) in self.active(ft, buf, lower) { + for m in ac.find_overlapping_iter(hay) { + if hits.len() >= cap { + return None; + } + hits.push((pidx as u32, m.value(), m.start() as u32)); + } + } + Some(Sweep { hits }) + } + + /// Which logical signatures can still fire, indexed by ldb id, given the + /// anchors this sweep hit. `None` when gating is not worth setting up. + /// + /// A body subsignature is only worth verifying while its logical + /// signature is still satisfiable. When a subsig the expression requires + /// has no anchor hit anywhere in the buffer its count is certainly zero, + /// which can kill the whole signature — and with it every verify its + /// other subsigs would have drawn. On a spreadsheet relationships part + /// that is 86% of the fan-out; on the sheet itself, all of it. + /// + /// FN-safe: [`Node::can_be_true_absent`] only reports "impossible" when + /// no assignment of the free subsigs satisfies the expression, and a + /// subsig with no anchor (PCRE/bcomp/fuzzy) is never treated as absent. + fn ldb_alive(&self, sweep: &Sweep) -> Option> { + if self.ldbs.is_empty() || sweep.hits.len() < GATE_MIN_HITS { + return None; + } + let mut hit_body = vec![false; self.bodies.len()]; + for &(pidx, value, _) in &sweep.hits { + for &bid in &self.partitions[pidx as usize].groups[value as usize] { + hit_body[bid] = true; + } + } + let mut alive = vec![true; self.ldbs.len()]; + let mut seen = vec![false; self.ldbs.len()]; + for (bid, &h) in hit_body.iter().enumerate() { + if !h { + continue; + } + let li = self.body_ldb[bid]; + if li == u32::MAX || seen[li as usize] { + continue; + } + seen[li as usize] = true; + let ldb = &self.ldbs[li as usize]; + let absent = |i: usize| match ldb.subs.get(i) { + // Only body subsigs are decided by anchor presence; one + // without an anchor must stay free, never "absent". + Some(SubSig::Bodies(ids)) => !ids.iter().any(|&b| hit_body[b]), + _ => false, + }; + alive[li as usize] = ldb.expr.can_be_true_absent(&absent); + } + Some(alive) + } + + /// Verify every body reachable from this scan's anchor hits, in + /// automaton order, calling `on_match` for each one that matches. + /// Stops as soon as `on_match` returns `Break`. + /// + /// The one place the anchor sweep, the repeated-anchor cap, the step + /// pools and the logical-signature gate are applied, so the first-match, + /// all-match and logical-offset scans cannot drift apart. + fn walk_candidates( + &self, + buf: &[u8], + ft: FileType, + layout: Option<&PeLayout>, + lower: &[u8], + mut on_match: F, + ) where + F: FnMut(usize, u64) -> std::ops::ControlFlow<()>, + { + let mut budgets = Budgets::new(); + let mut occ = OccCache::new(); + let mut group_cost = GroupCost::new(max_group_steps()); + // Verify one body against one anchor hit, charging the group cost, + // and hand a match to the callback. A macro rather than a closure: + // `budgets`, `occ`, `group_cost` and `on_match` are all borrowed + // mutably here and in the loops around it. + macro_rules! try_body { + ($bid:expr, $pidx:expr, $value:expr, $start:expr) => {{ + let bid = $bid; + let body = &self.bodies[bid]; + let pool_before = budgets.sim + budgets.legacy; + let matched = + verify(body, buf, $start, layout, lower, &mut budgets, &mut occ); + group_cost.charge( + $pidx, + $value, + pool_before - (budgets.sim + budgets.legacy), + ); + if let Some(start) = matched { + if on_match(bid, start).is_break() { + return; + } + } + }}; + } + match self.sweep_hits(ft, buf, lower) { + Some(sweep) => { + let alive = self.ldb_alive(&sweep); + for &(pidx, value, start) in &sweep.hits { + let pidx = pidx as usize; + if !group_cost.admit(pidx, value) { + continue; + } + for &bid in &self.partitions[pidx].groups[value as usize] { + if let Some(alive) = &alive { + let li = self.body_ldb[bid]; + if li != u32::MAX && !alive[li as usize] { + continue; + } + } + try_body!(bid, pidx, value, start as usize); + } + } + } + // Over the buffer cap: the direct path, exactly as before. + None => { + for (pidx, ac, groups, hay) in self.active(ft, buf, lower) { + for m in ac.find_overlapping_iter(hay) { + if !group_cost.admit(pidx, m.value()) { + continue; + } + for &bid in &groups[m.value() as usize] { + try_body!(bid, pidx, m.value(), m.start()); + } + } + } + } + } + } + /// Scan `buf`, returning the first matching signature as /// `(clean_name, offset, unofficial)`. pub fn scan_with_layout( @@ -2447,34 +2972,31 @@ impl SigEngine { } else { Vec::new() }; - let mut budgets = Budgets::new(); - for (ac, groups, hay) in self.active(ft, buf, &lower) { - for m in ac.find_overlapping_iter(hay) { - let group = &groups[m.value() as usize]; - for &bid in group { - // The partition already guarantees this body's target matches - // `ft`, so no per-body target check is needed here. - let body = &self.bodies[bid]; - if let Some(start) = verify(body, buf, m.start(), layout, &lower, &mut budgets) { - match &body.owner { - Owner::Ndb { name, unofficial } => { - return Some((name.clone(), start, *unofficial)) - } - // counts is non-empty whenever LdbSub bodies exist. - Owner::LdbSub => { - if counts[bid] == 0 { - touched.push(bid); - } - counts[bid] = counts[bid].saturating_add(1); - let s = start.min(u32::MAX as u64 - 1) as u32; - if offs[bid] == u32::MAX || s < offs[bid] { - offs[bid] = s; - } - } - } + // The partition already guarantees each body's target matches `ft`, so + // no per-body target check is needed. + let mut ndb_hit: Option<(String, u64, bool)> = None; + self.walk_candidates(buf, ft, layout, &lower, |bid, start| { + match &self.bodies[bid].owner { + Owner::Ndb { name, unofficial } => { + ndb_hit = Some((name.clone(), start, *unofficial)); + return std::ops::ControlFlow::Break(()); + } + // counts is non-empty whenever LdbSub bodies exist. + Owner::LdbSub => { + if counts[bid] == 0 { + touched.push(bid); + } + counts[bid] = counts[bid].saturating_add(1); + let s = start.min(u32::MAX as u64 - 1) as u32; + if offs[bid] == u32::MAX || s < offs[bid] { + offs[bid] = s; } } } + std::ops::ControlFlow::Continue(()) + }); + if let Some(hit) = ndb_hit { + return Some(hit); } // If no subsignature matched, only logical sigs that fire on an empty @@ -2558,34 +3080,27 @@ impl SigEngine { } else { Vec::new() }; - let mut budgets = Budgets::new(); - for (ac, groups, hay) in self.active(ft, buf, &lower) { - for m in ac.find_overlapping_iter(hay) { - for &bid in &groups[m.value() as usize] { - // Partition membership implies the target match. - let body = &self.bodies[bid]; - if let Some(start) = verify(body, buf, m.start(), layout, &lower, &mut budgets) { - match &body.owner { - Owner::Ndb { name, unofficial } => { - if seen.insert(name.clone()) { - out.push((name.clone(), start, *unofficial)); - } - } - Owner::LdbSub => { - if counts[bid] == 0 { - touched.push(bid); - } - counts[bid] = counts[bid].saturating_add(1); - let s = start.min(u32::MAX as u64 - 1) as u32; - if offs[bid] == u32::MAX || s < offs[bid] { - offs[bid] = s; - } - } - } + // Partition membership implies the target match. + self.walk_candidates(buf, ft, layout, &lower, |bid, start| { + match &self.bodies[bid].owner { + Owner::Ndb { name, unofficial } => { + if seen.insert(name.clone()) { + out.push((name.clone(), start, *unofficial)); + } + } + Owner::LdbSub => { + if counts[bid] == 0 { + touched.push(bid); + } + counts[bid] = counts[bid].saturating_add(1); + let s = start.min(u32::MAX as u64 - 1) as u32; + if offs[bid] == u32::MAX || s < offs[bid] { + offs[bid] = s; } } } - } + std::ops::ControlFlow::Continue(()) + }); let body_off = |b: usize| -> Option { match offs[b] { u32::MAX => None, @@ -2658,32 +3173,24 @@ impl SigEngine { } else { Vec::new() }; - let mut budgets = Budgets::new(); - for (ac, groups, hay) in self.active(ft, buf, &lower) { - for m in ac.find_overlapping_iter(hay) { - for &bid in &groups[m.value() as usize] { - // Partition membership implies the target match. - let body = &self.bodies[bid]; - if let Some(start) = verify(body, buf, m.start(), layout, &lower, &mut budgets) { - if let Owner::LdbSub = &body.owner { - if counts[bid] == 0 { - touched.push(bid); - } - counts[bid] = counts[bid].saturating_add(1); - // Clamp below `u32::MAX` so a real offset can never - // collide with the `CLI_OFF_NONE` (`u32::MAX`) - // "no match" sentinel used in `offs` (only matters - // for the 4 GiB boundary, where a u32 offset is - // already lossy). - let s = start.min(u32::MAX as u64 - 1) as u32; - if offs[bid] == u32::MAX || s < offs[bid] { - offs[bid] = s; - } - } - } + // Partition membership implies the target match. + self.walk_candidates(buf, ft, layout, &lower, |bid, start| { + if let Owner::LdbSub = &self.bodies[bid].owner { + if counts[bid] == 0 { + touched.push(bid); + } + counts[bid] = counts[bid].saturating_add(1); + // Clamp below `u32::MAX` so a real offset can never collide + // with the `CLI_OFF_NONE` (`u32::MAX`) "no match" sentinel used + // in `offs` (only matters for the 4 GiB boundary, where a u32 + // offset is already lossy). + let s = start.min(u32::MAX as u64 - 1) as u32; + if offs[bid] == u32::MAX || s < offs[bid] { + offs[bid] = s; } } - } + std::ops::ControlFlow::Continue(()) + }); let body_off = |b: usize| -> Option { match offs[b] { u32::MAX => None, @@ -2801,8 +3308,9 @@ fn verify( layout: Option<&PeLayout>, lower: &[u8], budgets: &mut Budgets, + occ: &mut OccCache, ) -> Option { - let start = verify_inner(body, buf, anchor_start, layout, lower, budgets)?; + let start = verify_inner(body, buf, anchor_start, layout, lower, budgets, occ)?; let Some(len) = body.fullword_len else { return Some(start); }; @@ -2828,6 +3336,7 @@ fn verify_inner( layout: Option<&PeLayout>, lower: &[u8], budgets: &mut Budgets, + occ: &mut OccCache, ) -> Option { // Pure literal: the Aho-Corasick hit is already the full match; only the // offset constraint remains (no backtracking, so no budget is drawn). @@ -2862,7 +3371,7 @@ fn verify_inner( SCAN_TRUNCATED.with(|c| c.set(true)); return None; } - gap_split_match($toks, buf, $at, body.nocase, lower, &mut budgets.sim) + gap_split_match($toks, buf, $at, body.nocase, lower, &mut budgets.sim, occ) }}; } return match body.prefix { @@ -2901,6 +3410,7 @@ fn verify_inner( body.nocase, lower, &mut budgets.sim, + occ, ) .filter(|&start| off_at(start)) .map(|start| start as u64) @@ -3331,6 +3841,159 @@ fn coalesce(ivs: &mut Vec<(usize, usize)>) { ivs.truncate(w + 1); } +/// Per-scan memo of literal-token occurrences: the fix for redundant suffix +/// rescanning. +/// +/// Every anchor hit re-verifies its bodies over suffixes that overlap the +/// previous hit's almost entirely, and each `advance_literal` re-runs `memmem` +/// over megabytes per hit — tens of thousands of times per buffer for a +/// repeated footer anchor. The occurrence SET of a token in the buffer does not +/// depend on the hit, so it is enumerated once per scan and every verify reads +/// its intervals out of it with two binary searches. Same reachable sets as the +/// direct search (overlapping enumeration, ascending order), so verdicts are +/// unchanged — only the repeated scanning goes away. +/// +/// Budget honesty is preserved: cached reads charge one unit per source +/// interval plus one per occurrence emitted, exactly as the direct path does, +/// so the per-buffer pool trips at the same verify in the same order. The one +/// full enumeration itself is uncharged (it costs ~one direct search and is +/// amortised over every later read). +/// +/// Memory is bounded: past [`MAX_OCC_TOTAL`] cached positions a token falls +/// back to the direct per-verify search, remembered so later verifies skip +/// straight to it instead of re-enumerating just to discard the set again. +/// Correctness is identical either way — only speed differs — so no truncation +/// flag is involved. +struct OccCache { + /// Split by case-folding rather than keyed on `(needle, nocase)`: a tuple + /// key has to be *built* to be looked up, so every call — cache hits + /// included, which is nearly all of them — allocated a `Vec` for the + /// needle. `Box<[u8]>` borrows as `[u8]`, so with one map per folding a hit + /// is a plain `get(needle)` and only a first insert allocates. + cs: std::collections::HashMap, OccEntry>, + ci: std::collections::HashMap, OccEntry>, + total: usize, + /// Position budget left before memoising is refused. A field rather than + /// the bare constant so tests can force the direct path. + cap: usize, +} + +/// What the memo knows about one token. +enum OccEntry { + /// Enumerated occurrence starts, ascending. + Cached(Vec), + /// Refused (would exceed the memory cap): direct search, immediately. + Failed, +} + +/// Cap on cached occurrence positions per scan (~32 MB of positions). +const MAX_OCC_TOTAL: usize = 4_000_000; + +/// How one `advance_literal` call should find its occurrences. +enum OccUse<'a> { + /// Binary-search reads on the memoised sorted set. + Cached(&'a [usize]), + /// The direct per-interval `memmem` search. + Direct, +} + +impl OccCache { + fn new() -> Self { + Self::with_cap(MAX_OCC_TOTAL) + } + + /// A memo allowed `cap` cached positions. `0` refuses every token, forcing + /// the direct search — which is how the differential tests reach that path + /// on the small buffers they generate. + fn with_cap(cap: usize) -> Self { + Self { + cs: std::collections::HashMap::new(), + ci: std::collections::HashMap::new(), + total: 0, + cap, + } + } + + /// How to find `needle`'s occurrences: the memoised sorted set when the + /// token was seen before, else a first full enumeration (later uses read). + fn occurrences( + &mut self, + buf: &[u8], + lower: &[u8], + needle: &[u8], + nocase: bool, + ) -> OccUse<'_> { + if self.cap == 0 { + return OccUse::Direct; + } + let map = if nocase { &mut self.ci } else { &mut self.cs }; + // Looked up by borrowed slice, so a hit allocates nothing. + if !map.contains_key(needle) { + // Bounded by what is left of the budget, so the vector that the cap + // exists to bound cannot be built oversized first and measured + // after: a single-byte token on a 256 MB buffer would have + // allocated gigabytes only to be thrown away. + let room = self.cap - self.total; + let entry = match Self::enumerate(buf, lower, needle, nocase, room) { + Some(v) => { + self.total += v.len(); + OccEntry::Cached(v) + } + None => OccEntry::Failed, + }; + let map = if nocase { &mut self.ci } else { &mut self.cs }; + map.insert(needle.into(), entry); + } + let map = if nocase { &self.ci } else { &self.cs }; + match map.get(needle) { + Some(OccEntry::Cached(v)) => OccUse::Cached(v), + _ => OccUse::Direct, + } + } + + /// Every (overlapping) occurrence start of `needle` in the haystack + /// (`lower` when `nocase` — same length, so positions line up with `buf`), + /// ascending: the same set the direct per-interval search admits, so + /// memoised reads stay exact. + /// + /// `None` as soon as more than `room` positions exist, abandoning the work + /// rather than completing a set that would be refused anyway. + fn enumerate( + buf: &[u8], + lower: &[u8], + needle: &[u8], + nocase: bool, + room: usize, + ) -> Option> { + let n = buf.len(); + let l = needle.len(); + let haystack: &[u8] = if nocase { lower } else { buf }; + let folded; + let pat: &[u8] = if nocase { + folded = needle.to_ascii_lowercase(); + &folded + } else { + needle + }; + let mut v = Vec::new(); + let mut s = 0usize; + while s + l <= n { + match memchr::memmem::find(&haystack[s..], pat) { + Some(off) => { + let q = s + off; + if v.len() == room { + return None; + } + v.push(q); + s = q + 1; + } + None => break, + } + } + Some(v) + } +} + /// Advance a reachable set across a literal `needle` (length ≥ 1): for every /// start position `q` in the set with `buf[q..q+L] == needle` (ASCII-folded when /// `nocase`), the position `q+L` is reachable next. `nocase` searches the @@ -3338,6 +4001,12 @@ fn coalesce(ivs: &mut Vec<(usize, usize)>) { /// budget unit per source interval plus one per occurrence found; on exhaustion /// it flags truncation and returns whatever was found so the scan is never a /// silent clean. +/// +/// Occurrences come from the per-scan [`OccCache`] (one full enumeration per +/// token per scan, then binary-search reads) rather than a fresh `memmem` per +/// interval, which is what made repeated-anchor verifies rescan the same +/// suffixes thousands of times. +#[allow(clippy::too_many_arguments)] fn advance_literal( cur: &[(usize, usize)], buf: &[u8], @@ -3346,6 +4015,7 @@ fn advance_literal( nocase: bool, out: &mut Vec<(usize, usize)>, budget: &mut u64, + occ: &mut OccCache, ) { let n = buf.len(); let l = needle.len(); @@ -3353,6 +4023,34 @@ fn advance_literal( return; } let last_start = n - l; // greatest position a length-l needle can start at + if let OccUse::Cached(all) = occ.occurrences(buf, lower, needle, nocase) { + for &(lo, hi) in cur { + if lo > last_start { + continue; + } + if *budget == 0 { + SCAN_TRUNCATED.with(|c| c.set(true)); + return; + } + *budget -= 1; + let qmax = hi.min(last_start); + // Occurrences in `[lo, qmax]`, by binary search on the memoised + // sorted set. Ascending, so `out` keeps the order the direct + // enumeration produced (matters only for determinism of downstream + // first-match offsets, which read minima). + let js = all.partition_point(|&q| q < lo); + let je = all.partition_point(|&q| q <= qmax); + for &q in &all[js..je] { + out.push((q + l, q + l)); + if *budget == 0 { + SCAN_TRUNCATED.with(|c| c.set(true)); + return; + } + *budget -= 1; + } + } + return; + } let (hay, folded); if nocase { folded = needle.to_ascii_lowercase(); @@ -3601,6 +4299,7 @@ fn gap_split_match( nocase: bool, lower: &[u8], budget: &mut u64, + occ: &mut OccCache, ) -> bool { let mut cur: Vec<(usize, usize)> = vec![(pos, pos)]; let mut next: Vec<(usize, usize)> = Vec::new(); @@ -3615,7 +4314,7 @@ fn gap_split_match( match tok { Elem::Bytes(b) => { next.clear(); - advance_literal(&cur, buf, lower, b, nocase, &mut next, budget); + advance_literal(&cur, buf, lower, b, nocase, &mut next, budget, occ); coalesce(&mut next); std::mem::swap(&mut cur, &mut next); } @@ -3629,7 +4328,7 @@ fn gap_split_match( if o.is_empty() { next.extend_from_slice(&cur); } else { - advance_literal(&cur, buf, lower, o, nocase, &mut next, budget); + advance_literal(&cur, buf, lower, o, nocase, &mut next, budget, occ); } } coalesce(&mut next); @@ -3727,6 +4426,7 @@ fn gap_split_match_backward( nocase: bool, lower: &[u8], budget: &mut u64, + occ: &mut OccCache, ) -> Option { if toks.is_empty() { return Some(end); // no preceding context: the pattern starts at `end` @@ -3752,7 +4452,7 @@ fn gap_split_match_backward( let mut next = Vec::new(); match tok { Elem::Bytes(b) => { - advance_literal(cur, buf, lower, b, nocase, &mut next, budget); + advance_literal(cur, buf, lower, b, nocase, &mut next, budget, occ); coalesce(&mut next); } Elem::Alt { opts, neg: false } => { @@ -3760,7 +4460,7 @@ fn gap_split_match_backward( if o.is_empty() { next.extend_from_slice(cur); } else { - advance_literal(cur, buf, lower, o, nocase, &mut next, budget); + advance_literal(cur, buf, lower, o, nocase, &mut next, budget, occ); } } coalesce(&mut next); @@ -4668,6 +5368,132 @@ mod tests { let _ = b.build(); // must not panic } + /// A shorter but rare literal must outrank a longer ubiquitous one, and + /// `pick_anchor` must actually anchor on it. + /// + /// The pair is real. `"http://" {7-15} ":7878"` is a live signature, and a + /// 12 MB spreadsheet relationships part drove its `"http://"` anchor to + /// 61,489 hits while `":7878"` occurs in it zero times. Length-only ranking + /// picks `"http://"`; a live main+daily set has it in 9,225 bodies against + /// 1 for `":7878"`, which is what makes the right choice visible without + /// any reference corpus. + #[test] + fn a_rare_literal_outranks_a_popular_one() { + use super::parse::{anchor_score, pick_anchor, GramStats}; + // Each call is one body. The proportions are what the real set looks + // like: a literal hundreds of signatures are built around, against one + // that appears in a single signature. + let stats = GramStats::build(500, |f| { + for _ in 0..500 { + f(b"http://"); + } + f(b":7878"); + }); + let (rare, popular) = ( + anchor_score(b":7878", Some(&stats)), + anchor_score(b"http://", Some(&stats)), + ); + assert!(rare > popular, "\":7878\" ({rare}) must beat \"http://\" ({popular})"); + + // With no statistics — which is the state while a body is first parsed + // — the ranking is the original length order, so the build-time re-pick + // is doing real work rather than confirming a choice already made. + assert!(anchor_score(b":7878", None) < anchor_score(b"http://", None)); + + // A literal no signature shares keeps scoring its length, so ordinary + // binary patterns are untouched. + assert_eq!(anchor_score(b"\x9a\x3f\xe1\x07\xb2", Some(&stats)), 5); + + // End to end: the anchor moves past the gap onto the rare literal. + let elems = vec![ + Elem::Bytes(b"http://".to_vec()), + Elem::Gap { + min: 7, + max: Some(15), + }, + Elem::Bytes(b":7878".to_vec()), + ]; + assert!( + matches!( + pick_anchor(&elems, true, Some(&stats)), + Some(Prefix::Internal { anchor_idx: 2 }) + ), + "must anchor past the gap on the rare literal, got {:?}", + pick_anchor(&elems, true, Some(&stats)) + ); + } + + /// Grams in a single body are dropped to keep the counting map small; that + /// must not change any score, since `log2(1)` is already zero. + #[test] + fn single_body_grams_cost_nothing() { + use super::parse::{anchor_score, GramStats}; + let stats = GramStats::build(2, |f| { + f(b"ONLYONCE"); + f(b"ALSOONCE"); + }); + assert_eq!(anchor_score(b"ONLYONCE", Some(&stats)), 8); + assert_eq!(anchor_score(b"ONLYONCE", None), 8); + } + + /// A short literal must be charged for every body that CONTAINS it, not + /// only for bodies that are exactly it. + /// + /// Regression. Counting credited each literal only at the window width its + /// own lookup would use, so the 2-gram `"//"` was counted from two-byte + /// literals alone — a handful — while the tens of thousands of longer + /// literals containing it credited nothing. `"//"` then scored as rare, + /// beat the correctly-penalised `"http://"`, and anchors migrated onto the + /// most common fragments in the set: measured, +84% token verifies on one + /// spreadsheet part, with `"://"` and `"htt"` becoming hot anchors. + #[test] + fn a_short_literal_is_charged_for_every_body_containing_it() { + use super::parse::{anchor_score, GramStats}; + // No body IS "//" or "://" — they only ever appear inside a longer one. + let stats = GramStats::build(500, |f| { + for _ in 0..500 { + f(b"http://example"); + } + }); + for frag in [&b"//"[..], b"://", b"htt"] { + assert!( + anchor_score(frag, Some(&stats)) < anchor_score(b"http://", Some(&stats)), + "{:?} must not outrank the literal it is a fragment of ({} vs {})", + String::from_utf8_lossy(frag), + anchor_score(frag, Some(&stats)), + anchor_score(b"http://", Some(&stats)) + ); + } + } + + /// The repeat filter is an optimisation, so it must not change an answer. + /// Same literals through both paths — the exact single pass a small build + /// takes, and the filtered two-pass a full database takes — must score + /// identically. + #[test] + fn the_repeat_filter_changes_no_score() { + use super::parse::{anchor_score, GramStats, FILTER_MIN_BODIES}; + let feed = |f: &mut dyn FnMut(&[u8])| { + for _ in 0..300 { + f(b"POPULAR!"); + } + for _ in 0..7 { + f(b"UNCOMMON"); + } + f(b"SINGLETON"); + }; + let exact = GramStats::build(1, feed); + let filtered = GramStats::build(FILTER_MIN_BODIES, feed); + for lit in [&b"POPULAR!"[..], b"UNCOMMON", b"SINGLETON", b"ABSENT!!"] { + assert_eq!( + anchor_score(lit, Some(&exact)), + anchor_score(lit, Some(&filtered)), + "paths disagree on {:?}", + String::from_utf8_lossy(lit) + ); + } + } + // --- gap-split matcher equivalence ------------------------------------- /// Reference present/absent oracle: a straightforward recursive matcher that @@ -4833,12 +5659,23 @@ mod tests { let lower = buf.to_ascii_lowercase(); let want = bt_match(&toks, &buf, pos, nocase); + // Both occurrence paths, every case. These buffers are far too + // small to ever exhaust the real memo budget, so with only + // `OccCache::new()` the direct search — the path a large buffer + // takes once a token blows the position cap — went unexercised + // here, and a divergence in it would have passed silently. + for cap in [MAX_OCC_TOTAL, 0] { + let mut b2 = u64::MAX; + let mut occ = OccCache::with_cap(cap); + let got = gap_split_match(&toks, &buf, pos, nocase, &lower, &mut b2, &mut occ); + assert_eq!( + want, got, + "gap-split wrong (occ cap {cap}): toks={toks:?} buf={buf:?} pos={pos} nocase={nocase}" + ); + } let mut b2 = u64::MAX; - let got = gap_split_match(&toks, &buf, pos, nocase, &lower, &mut b2); - assert_eq!( - want, got, - "gap-split wrong: toks={toks:?} buf={buf:?} pos={pos} nocase={nocase}" - ); + let mut occ = OccCache::new(); + let got = gap_split_match(&toks, &buf, pos, nocase, &lower, &mut b2, &mut occ); // The legacy walk must never find a match the split path misses. let mut b1 = u64::MAX; if match_forward(&toks, &buf, pos, nocase, &mut b1) { @@ -4868,8 +5705,21 @@ mod tests { let mut b1 = u64::MAX; let want = match_backward(&toks, &buf, end, nocase, &mut b1); + // Memoised and direct occurrence paths alike — see the forward test. + for cap in [MAX_OCC_TOTAL, 0] { + let mut b2 = u64::MAX; + let mut occ = OccCache::with_cap(cap); + let got = + gap_split_match_backward(&toks, &buf, end, nocase, &lower, &mut b2, &mut occ); + assert_eq!( + want, got, + "backward split wrong (occ cap {cap}): toks={toks:?} buf={buf:?} end={end} nocase={nocase}" + ); + } let mut b2 = u64::MAX; - let got = gap_split_match_backward(&toks, &buf, end, nocase, &lower, &mut b2); + let mut occ = OccCache::new(); + let got = + gap_split_match_backward(&toks, &buf, end, nocase, &lower, &mut b2, &mut occ); assert_eq!( want, got, "backward split wrong: toks={toks:?} buf={buf:?} end={end} nocase={nocase}" @@ -4899,8 +5749,9 @@ mod tests { ]; let lower = buf.to_ascii_lowercase(); let mut b = u64::MAX; + let mut occ = OccCache::new(); assert_eq!( - gap_split_match_backward(&toks, &buf, buf.len(), false, &lower, &mut b), + gap_split_match_backward(&toks, &buf, buf.len(), false, &lower, &mut b, &mut occ), Some(4), "must return the start the backtracker returns (shortest gap first)" ); @@ -4939,8 +5790,9 @@ mod tests { // Simulator: a real "no match", with the budget barely touched. reset_scan_truncated(); let mut sim = VERIFY_BUDGET; + let mut occ = OccCache::new(); assert_eq!( - gap_split_match_backward(&toks, &buf, buf.len(), false, &lower, &mut sim), + gap_split_match_backward(&toks, &buf, buf.len(), false, &lower, &mut sim, &mut occ), None ); assert!(!scan_was_truncated(), "must not report an incomplete search"); @@ -4964,7 +5816,9 @@ mod tests { ]; let lower = buf.to_ascii_lowercase(); let mut budget = 5_000_000u64; - let got = gap_split_match_backward(&toks, &buf, buf.len(), false, &lower, &mut budget); + let mut occ = OccCache::new(); + let got = + gap_split_match_backward(&toks, &buf, buf.len(), false, &lower, &mut budget, &mut occ); assert!(got.is_some(), "must find the match"); assert!(!scan_was_truncated(), "must not report truncation"); } @@ -4976,7 +5830,8 @@ mod tests { let check = |toks: &[Elem], buf: &[u8], pos: usize, nocase: bool| -> bool { let lower = buf.to_ascii_lowercase(); let mut b = u64::MAX; - gap_split_match(toks, buf, pos, nocase, &lower, &mut b) + let mut occ = OccCache::new(); + gap_split_match(toks, buf, pos, nocase, &lower, &mut b, &mut occ) }; // Unbounded gap between two literals. let t = vec![ @@ -5050,4 +5905,33 @@ mod tests { Some(1) ); } + + #[test] + fn group_cost_caps_concentrated_draw_and_flags_truncation() { + // The flag is thread-local and test threads are reused: leave it as + // found so no other test observes a truncation it did not cause. + reset_scan_truncated(); + let mut g = GroupCost::new(100); + // Cheap verifies never trip the cap, however many run. + for _ in 0..10_000 { + assert!(g.admit(0, 7)); + g.charge(0, 7, 0); + } + assert!(!scan_was_truncated(), "zero-draw groups must not interact"); + // One group drawing past the cap is refused from then on. + g.charge(0, 7, 60); + assert!(g.admit(0, 7)); + g.charge(0, 7, 50); + assert!(!g.admit(0, 7), "110 drawn past a cap of 100 trips it"); + assert!(scan_was_truncated(), "tripping the cap is an incomplete search"); + // A different group — or the same value in another partition — is + // unaffected: the cap bounds concentrated cost, not total work. + reset_scan_truncated(); + let mut g = GroupCost::new(10); + g.charge(0, 7, 11); + assert!(g.admit(0, 8)); + assert!(g.admit(1, 7)); + assert!(!scan_was_truncated(), "distinct groups must not interact"); + reset_scan_truncated(); + } } diff --git a/crates/exav-core/src/engine/parse.rs b/crates/exav-core/src/engine/parse.rs index 49b02529..7acaf365 100644 --- a/crates/exav-core/src/engine/parse.rs +++ b/crates/exav-core/src/engine/parse.rs @@ -285,7 +285,7 @@ pub(super) fn parse_subsig(s: &str) -> Option> { /// with `0x00`, wildcards each match one wide char, and gaps count wide chars. pub(super) fn compile_wide(body: &str, allow_internal: bool) -> Option<(Vec, Vec, Prefix)> { let elems = widen_elems(parse_elems(body)?); - let prefix = pick_anchor(&elems, allow_internal)?; + let prefix = pick_anchor(&elems, allow_internal, None)?; let anchor_idx = match prefix { Prefix::Fixed { anchor_idx, .. } | Prefix::Floating { anchor_idx } @@ -356,7 +356,7 @@ pub(super) fn widen_elems(elems: Vec) -> Vec { /// Returns `None` if it uses an unsupported construct or has no usable anchor. pub(super) fn compile_body(hex: &str, allow_internal: bool) -> Option<(Vec, Vec, Prefix)> { let elems = parse_elems(hex)?; - let prefix = pick_anchor(&elems, allow_internal)?; + let prefix = pick_anchor(&elems, allow_internal, None)?; let anchor_idx = match prefix { Prefix::Fixed { anchor_idx, .. } | Prefix::Floating { anchor_idx } @@ -554,13 +554,189 @@ fn decode_masked_hex(s: &str) -> Option> { Some(out) } -/// Choose the literal anchor and classify its prefix. +/// How many signature bodies contain each short byte sequence, counted over the +/// database being built. +/// +/// This is the selectivity signal [`anchor_score`] needs, derived from the +/// signature set itself rather than shipped alongside it. A literal that +/// hundreds of signatures are written around — `"http://"`, `"target=\""`, +/// `", +} + +/// Bits per presence filter. 2^28 bits is 32 MiB, ~8% occupancy against the +/// ~22M distinct grams of a full set, so few singletons are wrongly promoted to +/// the counting map — and a promotion costs only the memory, never a wrong +/// count, since the second pass counts what it actually sees. +const SEEN_BITS: usize = 1 << 28; + +/// Body count from which the repeat filter is worth its fixed cost. Below it +/// the two bitsets would be 64 MiB to filter a handful of grams — which is what +/// every in-process engine build does, from a unit test to a small sig dir. +pub(super) const FILTER_MIN_BODIES: usize = 100_000; + +/// A gram's key: its bytes, tagged with the length so `"ab"` and the prefix of +/// `"abcd"` cannot collide. Lengths are 2..=4, so this is lossless. +fn gram_key(g: &[u8]) -> u64 { + let mut v = 0u64; + for &b in g { + v = (v << 8) | b as u64; + } + v | ((g.len() as u64) << 56) +} + +/// Every gram of `lit` to CREDIT while counting: all windows of 2, 3 and 4 +/// bytes. +/// +/// Counting and lookup have to agree on what a gram's count means, and the +/// obvious shortcut — credit only the window size the lookup will use — is +/// wrong. Lookup asks a 2-byte literal about a 2-gram; if only 2-byte literals +/// ever credited 2-grams, `"//"` would be counted from the handful of two-byte +/// signatures rather than from the tens of thousands of literals that contain +/// it. Short ubiquitous literals then look rare, score well, and win the +/// anchor — the exact inversion this is built to prevent. Measured: doing it +/// that way moved anchors onto `"://"`, `"//"` and `"htt"` and raised token +/// verifies on one spreadsheet part by 84%. +fn count_grams_of(lit: &[u8], out: &mut Vec) { + for n in 2..=4usize { + if lit.len() < n { + break; + } + for w in lit.windows(n) { + out.push(gram_key(w)); + } + } +} + +/// The grams of `lit` to QUERY: the widest window it supports, up to 4. A +/// literal occurs in at most as many bodies as its rarest gram, and the widest +/// window gives the tightest such bound. +fn query_grams_of(lit: &[u8]) -> impl Iterator + '_ { + let n = lit.len().min(4); + lit.windows(n).map(gram_key) +} + +impl GramStats { + /// Count grams over every candidate literal `each` yields. + /// + /// `bodies` sizes the work. Past [`FILTER_MIN_BODIES`] the repeat filter + /// pays for itself and `each` is called twice — once to find which grams + /// repeat, once to count only those. Below it the filter's fixed 64 MiB + /// would dwarf the data it is filtering, so the counts are taken exactly in + /// a single pass. Both give identical scores: a gram the filter drops + /// appears in one body, and `log2(1)` is the zero an absent gram scores. + pub(super) fn build(bodies: usize, mut each: impl FnMut(&mut dyn FnMut(&[u8]))) -> Self { + let mut buf: Vec = Vec::new(); + if bodies < FILTER_MIN_BODIES { + let mut counts: rustc_hash::FxHashMap = rustc_hash::FxHashMap::default(); + each(&mut |lit| { + buf.clear(); + count_grams_of(lit, &mut buf); + buf.sort_unstable(); + buf.dedup(); + for &k in &buf { + *counts.entry(k).or_insert(0) += 1; + } + }); + return GramStats { counts }; + } + let mut once = vec![0u64; SEEN_BITS / 64]; + let mut twice = vec![0u64; SEEN_BITS / 64]; + let mark = |bits: &mut [u64], i: usize| bits[i >> 6] |= 1 << (i & 63); + let test = |bits: &[u64], i: usize| bits[i >> 6] >> (i & 63) & 1 == 1; + let slot = |k: u64| { + // Multiply-shift: the key is already a good integer, it only needs + // its high bits spread into the index range. + (k.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> (64 - SEEN_BITS.trailing_zeros())) as usize + }; + // Pass 1: which grams appear in more than one body. + each(&mut |lit| { + buf.clear(); + count_grams_of(lit, &mut buf); + buf.sort_unstable(); + buf.dedup(); + for &k in &buf { + let i = slot(k); + if test(&once, i) { + mark(&mut twice, i); + } else { + mark(&mut once, i); + } + } + }); + drop(once); + // Pass 2: count the repeats. Per body, not per occurrence, so a literal + // repeated inside one signature does not look popular. + let mut counts: rustc_hash::FxHashMap = rustc_hash::FxHashMap::default(); + each(&mut |lit| { + buf.clear(); + count_grams_of(lit, &mut buf); + buf.sort_unstable(); + buf.dedup(); + for &k in &buf { + if test(&twice, slot(k)) { + *counts.entry(k).or_insert(0) += 1; + } + } + }); + GramStats { counts } + } + + /// How many bodies contain `lit`, bounded above: a literal occurs in at most + /// as many bodies as its rarest gram does. Erring high is the safe + /// direction — it can only make a literal look like a worse anchor. + fn popularity(&self, lit: &[u8]) -> u32 { + if lit.len() < 2 { + return 0; + } + query_grams_of(lit) + .map(|k| self.counts.get(&k).copied().unwrap_or(0)) + .min() + .unwrap_or(0) + } +} + +/// `floor(log2(c))`, and 0 for the "nothing known" counts 0 and 1 — so an +/// unseen literal is never penalised, and the penalty grows with an order of +/// magnitude rather than with a raw count that spans 1..10^6. +fn log2_floor(c: u32) -> isize { + if c <= 1 { + 0 + } else { + (31 - c.leading_zeros()) as isize + } +} + /// Selectivity score of a candidate anchor run. A low-entropy run (a constant /// byte like a zero/0xFF pad, or a 2-symbol repeat) matches repetitive content /// — PE padding, BSS — millions of times, so it is a terrible prefilter even /// when long. Down-rank such runs sharply so a shorter but varied run wins; a /// varied run scores its length (longer = rarer = better). -pub(super) fn anchor_score(b: &[u8]) -> usize { +/// +/// Length alone is not selectivity, though: `"http://"` is 7 varied bytes and +/// scored best by that rule, yet it is the single worst anchor on web/office +/// content — one 12 MB spreadsheet relationships part hit it 61,489 times, +/// while the same body's `":7878"` occurs essentially never. So a literal is +/// also penalised by how many signature bodies share it ([`GramStats`]), which +/// lets a shorter but rarer run win. `stats` is `None` while a body is first +/// parsed, because the count is not known until the whole set has been read — +/// the choice is then revisited at build time. +pub(super) fn anchor_score(b: &[u8], stats: Option<&GramStats>) -> isize { let mut seen = [false; 256]; let mut distinct = 0usize; for &x in b { @@ -569,14 +745,26 @@ pub(super) fn anchor_score(b: &[u8]) -> usize { distinct += 1; } } - match distinct { + let structural = match distinct { 0 | 1 => 1, // constant run: near-useless anchor 2 => 3.min(b.len()), // 2-symbol repeat (e.g. ababab): weak _ => b.len(), // varied: length is the selectivity - } + } as isize; + // Both terms are log-scaled — length stands in for how rare a run should be, + // the popularity count for how common it actually is — so the difference + // reads as a selectivity estimate. Deliberately signed and unsaturated: + // clamping at zero puts every ubiquitous literal in a tie that the length + // tie-break then resolves backwards, handing the anchor to the *longest* + // common run (`"http://"` at 7 bytes beating `"umber:"` at 6, which is + // exactly the pairing that costs a 12 MB relationships part 61,489 hits). + structural - stats.map_or(0, |s| log2_floor(s.popularity(b))) } -pub(super) fn pick_anchor(elems: &[Elem], allow_internal: bool) -> Option { +pub(super) fn pick_anchor( + elems: &[Elem], + allow_internal: bool, + stats: Option<&GramStats>, +) -> Option { // Pick the most *selective* literal run as the Aho-Corasick anchor (highest // [`anchor_score`], tie-break longer), not merely the longest: a long // constant run is a far worse prefilter than a shorter varied one. Fewer @@ -594,13 +782,13 @@ pub(super) fn pick_anchor(elems: &[Elem], allow_internal: bool) -> Option = None; - let mut best_any: Option<(usize, usize, usize)> = None; // (score, len, idx) + let mut best_fixed: Option<(isize, usize, Prefix)> = None; + let mut best_any: Option<(isize, usize, usize)> = None; // (score, len, idx) for (i, e) in elems.iter().enumerate() { if let Elem::Bytes(b) = e { if b.len() >= MIN_ANCHOR { - let score = anchor_score(b); - let better = |cur: &Option<(usize, usize, Prefix)>| match cur { + let score = anchor_score(b, stats); + let better = |cur: &Option<(isize, usize, Prefix)>| match cur { Some((s, l, _)) => score > *s || (score == *s && b.len() > *l), None => true, }; @@ -631,7 +819,11 @@ pub(super) fn pick_anchor(elems: &[Elem], allow_internal: bool) -> Option fixed_score { // A single leading gap is the cheap `Floating` fast path, not the // general backward-matching `Internal`. diff --git a/crates/exav-core/src/lib.rs b/crates/exav-core/src/lib.rs index 32528464..9dc812c8 100644 --- a/crates/exav-core/src/lib.rs +++ b/crates/exav-core/src/lib.rs @@ -105,6 +105,48 @@ use ml::Model; use patterns::PatternSet; use unpack::Budget; +/// Smallest object a whole-file hash signature (`.hdb`/`.hsb`) may be matched +/// against. +/// +/// ClamAV refuses to scan any object of 5 bytes or fewer at all — `fmap->len <= +/// 5` in `cli_magic_scan` and its four sibling entry points (libclamav +/// `scanners.c`, checked against 1.4.3) — so every hash signature it ships with +/// a smaller declared size is unreachable in ClamAV, standalone file or nested +/// layer alike. Six such signatures exist in a current main+daily set. +/// +/// exav extracts more aggressively and had no such floor, which turned those +/// six pieces of auto-generated junk into live detections. Measured against +/// ClamAV 1.4.3 on the same inputs, which reports OK for both: +/// +/// * a 1-byte file holding `V` matched `Win.Trojan.Agent-1720205`; +/// * the 2 bytes `\x00\x00` that a benign PDF's 2x2 image XObject decodes to +/// matched `Win.Malware.Agent-7761897-0`. +/// +/// Note where this does NOT apply: the allow-list (`.fp`/`.sfp`) is matched +/// with the same machinery but must stay unfloored, since a suppression that +/// silently stopped working would cause exactly the false positives this +/// prevents. +/// +/// Deliberately a bound on hash *matching*, not on scanning: a short object is +/// still extracted, still pattern-matched, still counted. Only the claim "this +/// hash identifies a file" is refused for something too small to be one. No +/// detection is lost, because every signature below the floor is one ClamAV +/// itself can never fire, so nothing can depend on it. +/// What a scan reports when a per-buffer bound — the step pool, or the +/// repeated-anchor cap — ended the signature search before it finished. +/// +/// Shared by the single-verdict and all-match paths so the two cannot drift +/// apart in what they call the same condition. +const SEARCH_INCOMPLETE: &str = "scan budget exhausted — signature search incomplete"; + +const MIN_HASH_MATCH_BYTES: u64 = 6; + +/// Whether a whole-file hash signature may be matched against an object of +/// this size. See [`MIN_HASH_MATCH_BYTES`]. +fn hash_matchable(size: u64) -> bool { + size >= MIN_HASH_MATCH_BYTES +} + /// Map a detected [`FileType`] to the extraction [`unpack::Format`], or `None` /// if the type isn't an extractable container. fn unpack_format(ft: FileType) -> Option { @@ -992,6 +1034,10 @@ impl Scanner { /// loaded `.ftm` magic rules only when content detection is inconclusive /// (`Unknown`), so native typing is never overridden. pub fn identify(&self, data: &[u8]) -> FileType { + profile::timed("filetype", data.len() as u64, || self.identify_inner(data)) + } + + fn identify_inner(&self, data: &[u8]) -> FileType { let mut ft = filetype::identify(data); if ft == FileType::Unknown { if let Some(f) = self.ftm.identify(data) { @@ -1205,7 +1251,9 @@ fn peek_type(file: &File) -> io::Result { let n = fill_prefix(&mut r, &mut prefix)?; let mut s = file; s.seek(SeekFrom::Start(0))?; - Ok(filetype::identify(&prefix[..n])) + Ok(profile::timed("filetype", n as u64, || { + filetype::identify(&prefix[..n]) + })) } /// Scan a sequential stream (Stream mode): stdin, pipes, S3 streaming GET. @@ -1350,7 +1398,7 @@ pub fn scan_seekable( let mut prefix = [0u8; 4096]; let n = fill_prefix(&mut reader, &mut prefix)?; reader.seek(SeekFrom::Start(0))?; - let ft = filetype::identify(&prefix[..n]); + let ft = profile::timed("filetype", n as u64, || filetype::identify(&prefix[..n])); // Natively-streaming containers (ZIP/tar/gzip) are walked member-by-member // off the seekable source — the whole container is never buffered and no @@ -2042,7 +2090,9 @@ fn scan_streamed_container( None => None, } }; - unpack::stream_members(fmt, reader, budget, &mut visit) + profile::timed("unpack", container_size, || { + unpack::stream_members(fmt, reader, budget, &mut visit) + }) }; // Reassembly happens only now. Nothing in a byte-split set's names says how // many parts it has, so `.001`+`.002` looks contiguous even when `.003` @@ -2269,10 +2319,11 @@ fn analyze_inner(db: &Scanner, data: &[u8], opts: &ScanOptions) -> ScanReport { DeepOutcome::Unscannable(reason) => ScanReport::unscannable(reason, findings), DeepOutcome::PasswordProtected(reason) => ScanReport::password_protected(reason, findings), // Cardinal rule: a would-be `Clean` is downgraded to `LimitsExceeded` if - // any wildcard verification was skipped because its per-buffer step budget - // ran out — the search did not fully complete, so we must not report `OK`. + // any wildcard verification was skipped because a per-buffer scan bound + // ran out (the step pool or the repeated-anchor cap) — the search did + // not fully complete, so we must not report `OK`. DeepOutcome::Clean if engine::scan_was_truncated() => ScanReport::limits( - "verify step budget exhausted — signature search incomplete".into(), + SEARCH_INCOMPLETE.into(), findings, ), DeepOutcome::Clean => ScanReport::clean(findings), @@ -2298,10 +2349,10 @@ fn report_of_outcome( DeepOutcome::Unscannable(reason) => ScanReport::unscannable(reason, findings), DeepOutcome::PasswordProtected(reason) => ScanReport::password_protected(reason, findings), // A would-be `Clean` is downgraded when wildcard verification ran out of - // its per-buffer step budget: the search did not complete, so `OK` would + // its per-buffer scan bounds: the search did not complete, so `OK` would // be a claim the scan cannot support. DeepOutcome::Clean if engine::scan_was_truncated() => ScanReport::limits( - "verify step budget exhausted — signature search incomplete".into(), + SEARCH_INCOMPLETE.into(), findings, ), DeepOutcome::Clean => ScanReport::clean(findings), @@ -2365,11 +2416,24 @@ pub fn analyze_all_with_outcome( data: &[u8], opts: &ScanOptions, ) -> (Vec<(String, Method)>, AllMatchOutcome) { + // The flag is thread-local and sticky, so it has to be cleared per scan the + // way `analyze` clears it. Before this function read it the omission was + // invisible; with the read below, a stale `true` from an earlier scan on + // this thread would report a complete search as truncated. + engine::reset_scan_truncated(); let (names, outcome) = analyze_all_raw(db, data, opts); let outcome = match outcome { Some(DeepOutcome::Limits(r)) => AllMatchOutcome::LimitsExceeded(r), Some(DeepOutcome::Unscannable(r)) => AllMatchOutcome::Unscannable(r), Some(DeepOutcome::PasswordProtected(r)) => AllMatchOutcome::PasswordProtected(r), + // The same cardinal rule the single-verdict paths apply, which this one + // was missing: a search that did not finish must not be presented as a + // complete one. There it downgrades a would-be `Clean`; here there is no + // verdict to downgrade — the detections stand — so it becomes the + // outcome that travels alongside them, and the caller reports both. + _ if engine::scan_was_truncated() => { + AllMatchOutcome::LimitsExceeded(SEARCH_INCOMPLETE.into()) + } _ => AllMatchOutcome::Complete, }; (names, outcome) @@ -2866,7 +2930,10 @@ fn member_content_scan( // (an embedded PE, etc.) keep their own type so embedded-executable // detection is preserved. Either way the member carries its container type // so `Container:`-scoped sigs are gated correctly. - let ft_override = if cx.container_is_ole && is_textual_type(filetype::identify(data)) { + let ft_override = if cx.container_is_ole + && is_textual_type(profile::timed("filetype", data.len() as u64, || { + filetype::identify(data) + })) { Some(FileType::Ole) } else { None @@ -3124,7 +3191,11 @@ fn deep_analyze( // (`x.7z.001`, `.002`, …) are held here and rejoined once the container // has ended — see the note at `finish()` below for why not sooner. let mut volumes = unpack::volume::Collector::new(budget.limits().max_buffer_bytes); - let outcome = + // Inclusive wall time: this drives nested member scans, so `unpack_us` + // contains the member matchers' time too — pure extraction/dispatch is + // `unpack_us` minus the nested matcher columns, and `emu_us` splits the + // emulator's share of it out. + let outcome = profile::timed("unpack", data.len() as u64, || { unpack::extract_each(fmt, data, budget, &mut |mut e: unpack::Entry, budget: &mut Budget| -> Option { @@ -3173,7 +3244,8 @@ fn deep_analyze( unpack::volume::Offer::PassThrough { data, .. } => data, }; member_content_scan(&cx, &mut tally, &data, budget, findings, sink) - }); + }) + }); // Reassembly happens only now. Nothing in a byte-split set's names says // how many parts it has, so `.001`+`.002` looks contiguous even when // `.003` follows: joining on arrival would emit a truncated prefix that @@ -3756,7 +3828,7 @@ fn core_scan_all( } } } - if !db.hashes.is_empty() { + if !db.hashes.is_empty() && hash_matchable(data.len() as u64) { if let Some((name, unofficial)) = db.hashes.lookup(&digests_of(data), data.len() as u64) { sink.hit( report_name(&name, unofficial, unofficial_suffix), @@ -3933,7 +4005,8 @@ fn scan_bytes_depth( // A packer may surface an embedded archive (e.g. an unpacked // payload that is itself a ZIP/gzip). Unpack it too, bounded by // the same extraction budget, so the real payload is reached. - let bft = filetype::identify(&buf); + let bft = + profile::timed("filetype", buf.len() as u64, || filetype::identify(&buf)); // This deep bytecode-extracted-buffer rescan runs only behind a // ClamAV `.cbc` unpacker firing; it isn't part of the common // diff-tested path, so it always runs at full capability. @@ -3993,7 +4066,7 @@ fn scan_bytes_depth( return Some((name, 0, Method::Hash, unofficial)); } } - if !db.hashes.is_empty() { + if !db.hashes.is_empty() && hash_matchable(data.len() as u64) { if let Some((name, unofficial)) = profile::timed("hashes", data.len() as u64, || { db.hashes.lookup(&digests_of(data), data.len() as u64) }) { @@ -4045,8 +4118,10 @@ fn stream_core(db: &Scanner, reader: R) -> io::Result> } let size = tee.bytes_read(); let digests = tee.finalize(); - if let Some((name, unofficial)) = db.hashes.lookup(&digests, size) { - return Ok(Some((name, 0, Method::Hash, unofficial))); + if hash_matchable(size) { + if let Some((name, unofficial)) = db.hashes.lookup(&digests, size) { + return Ok(Some((name, 0, Method::Hash, unofficial))); + } } Ok(None) } @@ -4370,6 +4445,66 @@ mod tests { } } + /// Whole-file hash signatures stop applying at [`MIN_HASH_MATCH_BYTES`], + /// and the boundary is exactly where ClamAV puts it: 5 bytes is refused, 6 + /// matches. Verified against ClamAV 1.4.3 with an equivalent `.hdb`, which + /// reports OK up to 5 bytes and FOUND from 6. + #[test] + fn hash_signatures_do_not_match_tiny_objects() { + for n in 1..=8usize { + let data: Vec = (0..n).map(|i| b'A' + (i % 26) as u8).collect(); + let mut db = Scanner::builtin(); + let d = digests_of(&data); + db.hashes + .extend_from_text(&format!("{}:{}:Test.Tiny{}\n", d.md5, n, n)); + db.hashes.finalize(); + let hit = matches!( + analyze(&db, &data, &ScanOptions::default()).verdict, + Verdict::Infected { .. } + ); + assert_eq!( + hit, + n >= 6, + "{n}-byte object: hash match should be {}", + n >= 6 + ); + // The streaming core has its own hash path and must agree. + let hit = matches!( + scan_stream(&db, Cursor::new(data.clone())).unwrap().verdict, + Verdict::Infected { .. } + ); + assert_eq!(hit, n >= 6, "{n}-byte object via stream_core"); + } + } + + /// The allow-list shares the hash machinery but must NOT inherit the floor: + /// a suppression that quietly stopped working would reintroduce exactly the + /// false positives the floor removes. + #[test] + fn allowlist_still_applies_below_the_hash_floor() { + // A 4-byte body carrying a pattern detection, then allow-listed by hash. + let mut db = Scanner::builtin(); + let mut eb = engine::EngineBuilder::new(); + eb.add_ndb("Test.Tiny.Pattern:0:*:61626364", false); // "abcd" + db.engine = eb.build(); + let r = analyze(&db, b"abcd", &ScanOptions::default()); + assert!( + matches!(r.verdict, Verdict::Infected { .. }), + "pattern must still match a 4-byte object: {:?}", + r.verdict + ); + let d = digests_of(b"abcd"); + db.allow + .extend_from_text(&format!("{}:4:Test.Allow\n", d.md5)); + db.allow.finalize(); + let r = analyze(&db, b"abcd", &ScanOptions::default()); + assert!( + !matches!(r.verdict, Verdict::Infected { .. }), + "allow-list must suppress below the hash floor: {:?}", + r.verdict + ); + } + #[test] fn clean_is_clean() { let db = Scanner::builtin(); diff --git a/crates/exav-core/src/profile.rs b/crates/exav-core/src/profile.rs index 2b8387b4..ed2593f4 100644 --- a/crates/exav-core/src/profile.rs +++ b/crates/exav-core/src/profile.rs @@ -1,79 +1,7 @@ //! Optional per-matcher profiling. //! -//! When enabled on the current thread (via [`enable`]), each instrumented -//! matcher records its wall time, invocation count, and bytes of input through -//! [`timed`]. After a scan, [`take`] returns the accumulated [`Profile`] so a -//! caller can emit a per-file breakdown (the CLI's `--perf-csv` mode) or fold it -//! into a running total (its `--profile-scans` mode, on the listeners). When -//! profiling is *not* enabled, [`timed`] is just the wrapped call plus one cheap -//! thread-local check, so it's safe to leave on the hot scan path. - -use std::cell::RefCell; -use std::collections::BTreeMap; -use std::time::Instant; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -/// Per-matcher accumulated statistics for one scan. -#[derive(Default, Clone, Debug)] -pub struct Profile { - matchers: BTreeMap<&'static str, Stat>, -} - -/// Time, call count, and input bytes attributed to one matcher. -#[derive(Default, Clone, Copy, Debug)] -pub struct Stat { - /// Total nanoseconds spent in this matcher. - pub ns: u64, - /// Number of times the matcher was invoked. - pub calls: u64, - /// Total bytes of input handed to the matcher. - pub bytes: u64, -} - -impl Profile { - /// Iterate `(matcher_name, stat)` in name order. - pub fn iter(&self) -> impl Iterator + '_ { - self.matchers.iter().map(|(k, v)| (*k, *v)) - } -} - -/// Begin profiling on this thread, discarding any in-progress profile. -pub fn enable() { - CURRENT.with(|c| *c.borrow_mut() = Some(Profile::default())); -} - -/// Stop profiling on this thread and return the accumulated profile, or `None` -/// if profiling was never enabled. -pub fn take() -> Option { - CURRENT.with(|c| c.borrow_mut().take()) -} - -#[inline] -fn active() -> bool { - CURRENT.with(|c| c.borrow().is_some()) -} - -/// Run `f`, attributing its wall time and `bytes` of input to matcher `name` -/// when profiling is active; otherwise just runs `f`. Re-entrant: no borrow is -/// held across `f`, so nested instrumented calls accumulate independently. -#[inline] -pub fn timed(name: &'static str, bytes: u64, f: impl FnOnce() -> T) -> T { - if !active() { - return f(); - } - let t = Instant::now(); - let r = f(); - let ns = t.elapsed().as_nanos() as u64; - CURRENT.with(|c| { - if let Some(p) = c.borrow_mut().as_mut() { - let e = p.matchers.entry(name).or_default(); - e.ns += ns; - e.calls += 1; - e.bytes += bytes; - } - }); - r -} +//! Implemented in [`exav_unpack::profile`](https://docs.rs/exav-unpack) — shared +//! scan infra at the bottom of the dependency stack, so the packer emulator in +//! `exav-unpack` and the matchers here record into the same thread-local +//! profile. Re-exported so existing paths keep working. +pub use exav_unpack::profile::{Profile, Stat, enable, take, timed}; diff --git a/crates/exav-pe-emu/Cargo.toml b/crates/exav-pe-emu/Cargo.toml index 5e2a1c42..387a2de3 100644 --- a/crates/exav-pe-emu/Cargo.toml +++ b/crates/exav-pe-emu/Cargo.toml @@ -28,6 +28,11 @@ path = "src/main.rs" # scanned file supplies control flow rather than data, so the bytes reach the # decoder directly and in volume. exav-x86 = { path = "../exav-x86", version = "0.0.1" } +# The emulator looks pages and export traps up once per executed instruction, +# tens of millions of times per packed file. `HashMap`'s DoS-resistant hasher +# costs more than the lookup; `FxHashMap` hashes a `u32` key in a multiply and +# an xor. No other use in this crate needs a cryptographic hasher either. +rustc-hash = "2" # docs.rs builds with every feature on, so the parts gated behind a feature # (in particular exav-core's `unstable-internals`) appear on the rendered page diff --git a/crates/exav-pe-emu/src/cpu.rs b/crates/exav-pe-emu/src/cpu.rs index 473ea8bb..0f4e7c99 100644 --- a/crates/exav-pe-emu/src/cpu.rs +++ b/crates/exav-pe-emu/src/cpu.rs @@ -762,7 +762,11 @@ impl Cpu { // meant to save. A self-modifying stub turns the cache into pure // overhead, by orders of magnitude, unless invalidation is this cheap. let gen = mem.code_generation(); - let slot = (self.eip as usize >> 1) & (CACHE_SLOTS - 1); + // Indexed by the full address, not `eip >> 1`: consecutive + // instructions often differ by one byte (`inc ecx; dec ecx`), and + // folding that bit away maps neighbours onto the same slot, so + // straight-line code evicts itself every step and never hits. + let slot = self.eip as usize & (CACHE_SLOTS - 1); let insn = match &self.cache[slot] { Some((g, ip, insn)) if *g == gen && *ip == self.eip => insn.clone(), _ => { diff --git a/crates/exav-pe-emu/src/mem.rs b/crates/exav-pe-emu/src/mem.rs index 8894ec84..ab672700 100644 --- a/crates/exav-pe-emu/src/mem.rs +++ b/crates/exav-pe-emu/src/mem.rs @@ -18,7 +18,7 @@ //! into a page the stub wrote is the classic tail transfer to the original //! entry point. -use std::collections::HashMap; +use rustc_hash::FxHashMap; /// Page size of the emulated machine. x86 pages are 4 KiB, and packers align /// their `VirtualAlloc` requests to it. @@ -54,7 +54,7 @@ struct Page { pub struct Mem { pages: Vec, /// Page number (`addr >> 12`) to slot in `pages`. - index: HashMap, + index: FxHashMap, /// Hard cap on resident pages; hitting it is a budget stop, not a fault. max_pages: usize, /// Single-entry lookup cache. Emulated code has extreme page locality (a @@ -91,7 +91,7 @@ impl Mem { pub fn new(max_pages: usize) -> Self { Self { pages: Vec::new(), - index: HashMap::new(), + index: FxHashMap::default(), max_pages, cache_page: 0, cache_slot: 0, diff --git a/crates/exav-pe-emu/src/run.rs b/crates/exav-pe-emu/src/run.rs index 55b9ee2c..797b9eac 100644 --- a/crates/exav-pe-emu/src/run.rs +++ b/crates/exav-pe-emu/src/run.rs @@ -366,7 +366,20 @@ pub fn unpack(file: &[u8], limits: &EmuLimits) -> Report { // The tail transfer: into the image, into a different section from the // stub's, onto a page the stub wrote. - if oep.is_none() && eip >= image.base && eip < image_end { + // + // The conditions are ordered cheapest-first: the watched-range count is + // a field read, and until the stub has rebuilt `min_unpack_dirty` no + // jump can be the entry point, so most steps — every spinner, every + // stub that faults early — never reach the section lookup or the dirty + // hash probe. Reordering is semantics-preserving: every condition is + // side-effect free except `rebuilt`, which stays last under the same + // call conditions as before (so its recheck throttle behaves identically). + if oep.is_none() + && mem.watched_dirty_bytes() >= limits.min_unpack_dirty + && eip >= image.base + && eip < image_end + && mem.is_dirty(eip) + { let rva = eip - image.base; let sec = image.section_of(rva); // The transfer has to cross into another SECTION, not merely onto a @@ -377,11 +390,7 @@ pub fn unpack(file: &[u8], limits: &EmuLimits) -> Report { // then stops at what only looks like an entry point and dumps a // *half-decompressed* image. Stopping late and dumping everything // beats stopping at the first plausible-looking jump. - if sec != stub_section - && mem.watched_dirty_bytes() >= limits.min_unpack_dirty - && mem.is_dirty(eip) - && rebuilt(&image, sec, &mut mem, &mut recheck, ticks) - { + if sec != stub_section && rebuilt(&image, sec, &mut mem, &mut recheck, ticks) { oep = Some(rva); stop_reason = format!("reached the original entry point at RVA {rva:#x}"); break; diff --git a/crates/exav-pe-emu/src/win.rs b/crates/exav-pe-emu/src/win.rs index 53373765..579a8cbe 100644 --- a/crates/exav-pe-emu/src/win.rs +++ b/crates/exav-pe-emu/src/win.rs @@ -26,7 +26,7 @@ //! that would are present so a stub gets a plausible answer, and they fail the //! way they would on a machine where the operation is not permitted. -use std::collections::HashMap; +use rustc_hash::FxHashMap; use crate::cpu::{Cpu, Stop, EAX, ESP}; use crate::mem::{Mem, PAGE_SIZE}; @@ -707,11 +707,11 @@ pub struct Env<'a> { /// touches no host file: the only path that resolves is the program's own. file: &'a [u8], /// Open handles onto that file, with their read positions. - open_files: HashMap, + open_files: FxHashMap, next_handle: u32, /// Scratch cell handed back by the CRT's `__p__*` accessors. crt_cell: u32, - traps: HashMap, + traps: FxHashMap, /// Base address of the emulated program image (what `GetModuleHandle(NULL)` /// returns). pub image_base: u32, @@ -723,7 +723,7 @@ pub struct Env<'a> { last_error: u32, tick: u32, perf: u64, - tls: HashMap, + tls: FxHashMap, tls_next: u32, /// Exports that were called but not implemented, for diagnostics. pub missing_apis: Vec, @@ -754,17 +754,17 @@ impl<'a> Env<'a> { let mut env = Env { modules: Vec::new(), file, - open_files: HashMap::new(), + open_files: FxHashMap::default(), next_handle: FILE_HANDLE_BASE, crt_cell: ENV_BASE + 0x100, - traps: HashMap::new(), + traps: FxHashMap::default(), image_base, heap_next: HEAP_BASE, allocations: Vec::new(), last_error: 0, tick: 0x0001_0000, perf: 0x0010_0000, - tls: HashMap::new(), + tls: FxHashMap::default(), tls_next: 1, missing_apis: Vec::new(), trace: false, diff --git a/crates/exav-unpack/src/formats/pdf.rs b/crates/exav-unpack/src/formats/pdf.rs index f8727d00..03d243a3 100644 --- a/crates/exav-unpack/src/formats/pdf.rs +++ b/crates/exav-unpack/src/formats/pdf.rs @@ -186,22 +186,26 @@ pub(crate) fn extract_pdf( } let raw_len = stream_data.len() as u64; let cap = budget.reserve()?; - let (buf, truncated) = apply_filters(&stream_data, &info, cap)?; - if truncated { + let outcome = apply_filters(&stream_data, &info, cap)?; + if outcome.truncated { return Err(LimitHit::new("pdf stream exceeds budget".to_string())); } + let buf = outcome.buf; ratio_guard(raw_len, buf.len() as u64, budget)?; budget.commit(buf.len() as u64); - // A stream that HAD raw bytes and decoded to nothing is content we - // failed to produce, not an empty object. Emitting it as a zero-byte - // member says "scanned, nothing there" about bytes nobody read — - // the silent skip this scanner exists to avoid. Say so instead. + // A stream whose bytes an implemented filter failed to decode is + // content we failed to produce, not an empty object. Emitting it + // as a zero-byte member says "scanned, nothing there" about bytes + // nobody read — the silent skip this scanner exists to avoid. Say + // so instead. But a stream that DECODED to empty (legal — an + // empty page-content stream carries no bytes to hide) is a + // scanned empty member, not an undecodable one. // - // The usual cause is an image codec: `apply_filters` stops at a - // filter it does not implement (DCTDecode, JPXDecode, CCITTFaxDecode) - // and passes the remainder through, which yields nothing when that - // filter was the only one and the caller expected decoded output. - let entry = if buf.is_empty() && raw_len > 0 { + // Image codecs (`apply_filters` stops at DCTDecode, JPXDecode, + // CCITTFaxDecode) pass the raw bytes through, so those streams + // are scanned as their raw content — the signatures match the + // bytes either way. + let entry = if buf.is_empty() && raw_len > 0 && outcome.decode_error { Entry::unsupported( format!("pdf-obj-{obj_id}-{gen}"), raw_len, @@ -308,68 +312,134 @@ fn harvest_inner( } } +/// What decoding a stream's `/Filter` chain produced. +struct FilterOutcome { + buf: Vec, + /// An output cap was hit: the caller should treat the stream as over budget. + truncated: bool, + /// An implemented filter failed and nothing was recovered: the caller + /// should report the stream undecodable rather than scan an empty buffer + /// as though its bytes were examined. Distinct from "decoded to empty", + /// which is a successful decode of an empty stream (legal — e.g. an empty + /// page-content stream) and must NOT be reported undecodable. + decode_error: bool, +} + /// Decode a stream's raw bytes by applying its `/Filter` chain left to right, /// honouring `/DecodeParms` (a single dict or an array aligned with the /// filters). FlateDecode runs on the bounded `flate2` reader; the other PDF /// filters use the pure decoders in [`filters`]. An unknown/unsupported filter /// (e.g. an image codec like DCTDecode) stops the chain and passes through -/// whatever was decoded so far. Returns `(bytes, truncated)` where `truncated` -/// means an output cap was hit and the caller should treat it as over budget. +/// whatever was decoded so far. fn apply_filters( stream_data: &[u8], info: &HashMap, cap: u64, -) -> Result<(Vec, bool), LimitHit> { +) -> Result { let names = filter_names(info); if names.is_empty() { let take = (stream_data.len() as u64).min(cap) as usize; - return Ok((stream_data[..take].to_vec(), stream_data.len() as u64 > cap)); + return Ok(FilterOutcome { + buf: stream_data[..take].to_vec(), + truncated: stream_data.len() as u64 > cap, + // No decoding involved: raw bytes pass through, nothing to fail. + decode_error: false, + }); } let parms = decode_parms_list(info, names.len()); let mut buf = stream_data.to_vec(); + // Whether an implemented filter failed with nothing recovered. Sticky + // across the chain: only a successful decode producing content clears the + // question, and the caller reports `unsupported` solely on + // empty-output-with-error — a valid empty decode is a scanned empty member, + // not an undecodable stream. + let mut decode_error = false; for (idx, name) in names.iter().enumerate() { let parm = parms.get(idx).copied().flatten(); - let (out, truncated) = match name.as_str() { + let (out, truncated, err) = match name.as_str() { "FlateDecode" | "Fl" => { - // Salvage: keep whatever inflated before a truncation/corruption - // error rather than discarding the object — malware content is - // often in the recoverable prefix, and PDF streams are commonly - // truncated or patched. A leading-junk retry handles streams whose - // zlib header is preceded by stray bytes. - let (mut out, mut truncated) = bounded_read_salvage( - flate2::read::ZlibDecoder::new(Cursor::new(&buf)), - cap, - true, - ) - .unwrap_or((Vec::new(), false)); - if out.is_empty() { - if let Some(z) = buf.iter().position(|&b| b == 0x78) { - let r = bounded_read_salvage( - flate2::read::ZlibDecoder::new(Cursor::new(&buf[z..])), + // Strict first: a valid-but-empty stream must succeed (the + // caller emits it as a scanned empty member). Only on strict + // failure fall back to salvage — and only if salvage ALSO + // recovers nothing is the stream genuinely unreadable. + // Salvage still matters: malware content is often in the + // recoverable prefix, and PDF streams are commonly truncated + // or patched. A leading-junk retry handles streams whose zlib + // header is preceded by stray bytes. + match bounded_read(flate2::read::ZlibDecoder::new(Cursor::new(&buf)), cap) { + Ok((out, truncated)) => (out, truncated, false), + Err(_) => { + let (mut out, mut truncated) = bounded_read_salvage( + flate2::read::ZlibDecoder::new(Cursor::new(&buf)), cap, true, ) .unwrap_or((Vec::new(), false)); - out = r.0; - truncated = r.1; + if out.is_empty() { + if let Some(z) = buf.iter().position(|&b| b == 0x78) { + let r = bounded_read_salvage( + flate2::read::ZlibDecoder::new(Cursor::new(&buf[z..])), + cap, + true, + ) + .unwrap_or((Vec::new(), false)); + out = r.0; + truncated = r.1; + } + } + // Empty input decodes vacuously (nothing to fail on); + // only non-empty input decoding to nothing is an error. + let err = out.is_empty() && !buf.is_empty(); + (out, truncated, err) } } - (out, truncated) } - "LZWDecode" | "LZW" => filters::lzw_decode(&buf, early_change(parm), cap), - "ASCII85Decode" | "A85" => filters::ascii85_decode(&buf, cap), - "ASCIIHexDecode" | "AHx" => filters::ascii_hex_decode(&buf, cap), - "RunLengthDecode" | "RL" => filters::run_length_decode(&buf, cap), - // Unknown/unsupported filter: stop and emit what we have so far. + // The pure decoders report no error channel, so an empty output + // conservatively counts as a failure — today's behaviour, + // unchanged. (A valid-but-empty LZW/ASCII85 stream is legal but + // vanishingly rare in the wild, unlike empty Flate streams.) + "LZWDecode" | "LZW" => { + let (out, truncated) = filters::lzw_decode(&buf, early_change(parm), cap); + let err = out.is_empty() && !buf.is_empty(); + (out, truncated, err) + } + "ASCII85Decode" | "A85" => { + let (out, truncated) = filters::ascii85_decode(&buf, cap); + let err = out.is_empty() && !buf.is_empty(); + (out, truncated, err) + } + "ASCIIHexDecode" | "AHx" => { + let (out, truncated) = filters::ascii_hex_decode(&buf, cap); + let err = out.is_empty() && !buf.is_empty(); + (out, truncated, err) + } + "RunLengthDecode" | "RL" => { + let (out, truncated) = filters::run_length_decode(&buf, cap); + let err = out.is_empty() && !buf.is_empty(); + (out, truncated, err) + } + // Unknown/unsupported filter: stop and emit what we have so far + // (an image codec over raw bytes reads as the raw bytes — same as + // ClamAV, which matches signatures against the stream content + // without decoding image codecs either). _ => break, }; + decode_error |= err; if truncated { - return Ok((out, true)); + return Ok(FilterOutcome { + buf: out, + truncated: true, + decode_error, + }); } buf = out; } let truncated = buf.len() as u64 > cap; - Ok((buf, truncated)) + Ok(FilterOutcome { + buf, + truncated, + decode_error, + }) } /// Collect the ordered list of filter names from `/Filter` (a single name or an @@ -588,9 +658,111 @@ mod tests { ]), ); - let (out, truncated) = apply_filters(&raw, &info, 1 << 20).unwrap(); - assert!(!truncated); - assert_eq!(out, original); + let outcome = apply_filters(&raw, &info, 1 << 20).unwrap(); + assert!(!outcome.truncated); + assert!(!outcome.decode_error); + assert_eq!(outcome.buf, original); + } + + /// A valid-but-empty Flate stream (8 bytes of zlib wrapping zero content + /// bytes — common as empty page-content) decodes successfully to nothing. + /// It must surface as a scanned empty member, never as undecodable: the + /// old emptiness check reported whole benign PDFs `UNSCANNABLE` for this. + #[test] + fn empty_flate_stream_is_clean_not_undecodable() { + let raw = zlib_compress(b""); + assert!(!raw.is_empty(), "the fixture itself must carry bytes"); + let mut info: HashMap = HashMap::new(); + info.insert( + "Filter".to_string(), + Primitive::Name("FlateDecode".to_string()), + ); + let outcome = apply_filters(&raw, &info, 1 << 20).unwrap(); + assert!(!outcome.truncated); + assert!(!outcome.decode_error); + assert!(outcome.buf.is_empty()); + + // End to end: no `unsupported` member may come out of it. + let mut pdf = format!( + "%PDF-1.5\n1 0 obj\n<< /Length {} /Filter /FlateDecode >>\nstream\n", + raw.len() + ) + .into_bytes(); + pdf.extend_from_slice(&raw); + pdf.extend_from_slice(b"\nendstream\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF"); + let mut budget = Budget::new(Limits::default()); + let entries = extract(Format::Pdf, &pdf, &mut budget).unwrap(); + assert!( + entries.iter().all(|e| e.unsupported.is_none()), + "a valid empty stream must not poison the verdict: {:?}", + entries.iter().map(|e| &e.name).collect::>() + ); + } + + /// Garbage behind `/FlateDecode` (no zlib header anywhere for the + /// leading-junk retry to find) recovers nothing: still `unsupported`, so a + /// file whose bytes are present but unread never scans as clean. + #[test] + fn corrupt_flate_stream_stays_undecodable() { + // No zlib header anywhere (deliberately no 0x78 byte, so even the + // leading-junk retry finds nothing to decode). + let raw = b"\x00\x01\x02\x03not zlib, nothing usable here...."; + let mut info: HashMap = HashMap::new(); + info.insert( + "Filter".to_string(), + Primitive::Name("FlateDecode".to_string()), + ); + let outcome = apply_filters(raw, &info, 1 << 20).unwrap(); + assert!(!outcome.truncated); + assert!(outcome.decode_error); + assert!(outcome.buf.is_empty()); + + let mut pdf = format!( + "%PDF-1.5\n1 0 obj\n<< /Length {} /Filter /FlateDecode >>\nstream\n", + raw.len() + ) + .into_bytes(); + pdf.extend_from_slice(raw); + pdf.extend_from_slice(b"\nendstream\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF"); + let mut budget = Budget::new(Limits::default()); + let entries = extract(Format::Pdf, &pdf, &mut budget).unwrap(); + assert!( + entries + .iter() + .any(|e| e.unsupported == Some("PDF stream could not be decoded")), + "a corrupt stream must stay surfaced: {:?}", + entries + .iter() + .map(|e| (&e.name, e.unsupported)) + .collect::>() + ); + } + + /// A solid-white page image (1 MB of one byte, ratio ~1000:1) is ordinary + /// scanner content, not a bomb: it must extract as a clean member, not + /// trip the compression-ratio bomb guard into `LIMITS-EXCEEDED`. + #[test] + fn blank_page_image_is_not_a_ratio_bomb() { + let white = vec![0xffu8; 1_030_656]; // 768x1342 grayscale, all white + let raw = zlib_compress(&white); + assert!( + raw.len() as u64 * 1000 < white.len() as u64, + "fixture must actually exceed the 1000:1 ratio" + ); + let mut pdf = format!( + "%PDF-1.5\n1 0 obj\n<< /Type /XObject /Subtype /Image /Width 768 /Height 1342 /BitsPerComponent 8 /ColorSpace /DeviceGray /Length {} /Filter /FlateDecode >>\nstream\n", + raw.len() + ) + .into_bytes(); + pdf.extend_from_slice(&raw); + pdf.extend_from_slice(b"\nendstream\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF"); + let mut budget = Budget::new(Limits::default()); + let entries = extract(Format::Pdf, &pdf, &mut budget).unwrap(); + let img = entries + .iter() + .find(|e| e.unsupported.is_none() && e.data == white) + .expect("the white image must come back as scanned content"); + assert_eq!(img.data.len(), white.len()); } /// `/DecodeParms << /EarlyChange 0 >>` is threaded to the LZW decoder. diff --git a/crates/exav-unpack/src/formats/pepack.rs b/crates/exav-unpack/src/formats/pepack.rs index f813b7d4..bcc809d7 100644 --- a/crates/exav-unpack/src/formats/pepack.rs +++ b/crates/exav-unpack/src/formats/pepack.rs @@ -58,7 +58,7 @@ pub(crate) fn emulate_pe( trace, ..Default::default() }; - let r = x86::run::unpack(data, &limits); + let r = crate::profile::timed("emu", data.len() as u64, || x86::run::unpack(data, &limits)); let mut line = format!( "ticks={} dirty={}KiB stop={}", r.ticks, @@ -825,8 +825,8 @@ pub(crate) enum Recovered { /// Instruction budget for one stub. Runtime packers spend on the order of a /// hundred instructions per byte they produce, so this is what bounds the size /// of image the emulator can follow to completion — and what bounds the CPU a -/// hostile file can demand. At roughly 20M ticks a second it is a few seconds -/// in the worst case, and only for files that already look packed. +/// hostile file can demand. At tens of millions of ticks a second it is a few +/// seconds in the worst case, and only for files that already look packed. #[cfg(feature = "pe-emu")] const MAX_EMU_TICKS: u64 = 120_000_000; @@ -870,7 +870,8 @@ pub(crate) fn emulated_unpack( max_pages: cap.min(MAX_EMU_MEMORY) / exav_pe_emu::PAGE_SIZE, ..Default::default() }; - let report = x86::run::unpack(data, &limits); + let report = + crate::profile::timed("emu", data.len() as u64, || x86::run::unpack(data, &limits)); // Payloads the stub built in memory it allocated. Emitted whether or not it // also rebuilt its own image: a loader that unfolds the original program diff --git a/crates/exav-unpack/src/lib.rs b/crates/exav-unpack/src/lib.rs index e032d45e..30c21aef 100644 --- a/crates/exav-unpack/src/lib.rs +++ b/crates/exav-unpack/src/lib.rs @@ -66,6 +66,7 @@ pub(crate) mod formats; use formats::*; mod stream; +pub mod profile; pub mod volume; #[cfg(feature = "pdf")] pub use formats::has_obfuscated_name_object; @@ -1758,11 +1759,21 @@ pub(crate) fn cap_prealloc(requested: usize) -> usize { /// Absolute byte caps are the primary bomb defense; this is a fast reject /// for the obvious cases. A declared input of 0 is ignored (we cannot trust /// it) and left to the absolute caps. +/// +/// Enforced only past [`RATIO_FLOOR_BYTES`] of output: below it the absolute +/// caps already bound the allocation and the content is cheap to scan, while a +/// bare ratio trips on ordinary content — a 1 KB-compressed blank scanned page +/// (1 MB of one byte) has a ratio over 1000:1 without being anyone's bomb. A +/// real bomb still trips as soon as its output crosses the floor, milliseconds +/// into the decompression. // Dead only in a build with none of the compressing formats; see // `cap_prealloc` for why the feature list is not spelled out. #[allow(dead_code)] pub(crate) fn ratio_guard(input: u64, output: u64, budget: &Budget) -> Result<(), LimitHit> { - if input > 0 && output / input > budget.limits.max_compression_ratio { + if output >= RATIO_FLOOR_BYTES + && input > 0 + && output / input > budget.limits.max_compression_ratio + { return Err(LimitHit::new(format!( "compression ratio {} > {}", output / input, @@ -1772,6 +1783,21 @@ pub(crate) fn ratio_guard(input: u64, output: u64, budget: &Budget) -> Result<() Ok(()) } +/// Output size from which the compression-ratio bomb check applies (4 MiB). +/// +/// Below it, rejecting by ratio produces false `LIMITS-EXCEEDED` verdicts on +/// ordinary content: a blank scanned page is 1 MB of one repeated byte and +/// compresses past 1000:1 without being anyone's bomb. The absolute caps +/// (`max_buffer_bytes` per object, `max_extracted_bytes` in total) bound such +/// output anyway, and it is cheap to scan. +/// +/// Sized to the evidence rather than generously: 4x the 1 MB page image that +/// makes the floor necessary. Every byte of headroom past that is ratio +/// checking given up for nothing — a bomb is still caught the moment its output +/// crosses the floor, milliseconds into decompression, and real bombs overshoot +/// by orders of magnitude. +const RATIO_FLOOR_BYTES: u64 = 4 * 1024 * 1024; + /// The EICAR anti-virus test string, assembled at runtime from its reverse. /// /// The 68-byte sequence exists nowhere in this source tree, and nowhere in any @@ -2621,6 +2647,49 @@ mod tests { assert!(extract(Format::Gzip, &gz_bad_crc(payload), &mut b).is_err()); } + #[test] + fn ratio_guard_ignores_small_high_ratio_output() { + // A 1 KB-compressed blank scanned page expanding to 1 MB (ratio + // ~1000:1) is ordinary content, not a bomb: below the floor the + // absolute caps already bound it, so the ratio must not trip. + let b = Budget::new(Limits::default()); + assert!(ratio_guard(1021, 1_030_656, &b).is_ok()); + assert!(ratio_guard(100, 90_000, &b).is_ok()); + } + + #[test] + fn ratio_guard_still_trips_past_the_floor() { + // The same shape at bomb scale must still trip loudly: past the floor + // the ratio means resource exhaustion, not a blank page. + let b = Budget::new(Limits::default()); + assert!(ratio_guard(1021, 100 * 1024 * 1024, &b).is_err()); + assert!(ratio_guard(1021, RATIO_FLOOR_BYTES, &b).is_err()); + } + + /// Pins the floor at SHIPPED settings. Every other bomb test overrides + /// `max_compression_ratio`, so before this one nothing exercised the guard + /// as deployed — the floor could have been any value, or ineffective, and + /// the suite would have stayed green. + #[test] + fn ratio_guard_boundary_at_default_limits() { + let b = Budget::new(Limits::default()); + assert_eq!( + b.limits().max_compression_ratio, + 1000, + "the shipped ratio; this test is about the default configuration" + ); + // One byte under the floor is exempt no matter how extreme the ratio... + assert!(ratio_guard(1, RATIO_FLOOR_BYTES - 1, &b).is_ok()); + // ...and the guard resumes exactly at it. + assert!(ratio_guard(1, RATIO_FLOOR_BYTES, &b).is_err()); + // The 1 MB blank page that the floor exists for keeps 4x of room. + assert!(ratio_guard(1021, 1_030_656, &b).is_ok()); + assert_eq!(RATIO_FLOOR_BYTES, 4 * 1024 * 1024); + // A ratio at or under the limit is fine even well past the floor, so + // the floor only ever relaxes the check, never tightens it. + assert!(ratio_guard(1024 * 1024, 64 * 1024 * 1024, &b).is_ok()); + } + fn tar_of(members: &[(&str, &[u8])]) -> Vec { let mut buf = Vec::new(); { diff --git a/crates/exav-unpack/src/profile.rs b/crates/exav-unpack/src/profile.rs new file mode 100644 index 00000000..b0ce40e5 --- /dev/null +++ b/crates/exav-unpack/src/profile.rs @@ -0,0 +1,85 @@ +//! Optional per-stage profiling: named wall-time/calls/bytes accumulators. +//! +//! Shared scan infra at the bottom of the dependency stack, so both this crate +//! (the `emu` stage around the packer emulator) and `exav-core` (matchers, +//! `unpack`, `filetype`) record into the same thread-local [`Profile`]. +//! Re-exported from `exav_core::profile`, which is the path callers use. +//! +//! When enabled on the current thread (via [`enable`]), each instrumented +//! stage records its wall time, invocation count, and bytes of input through +//! [`timed`]. After a scan, [`take`] returns the accumulated [`Profile`]. +//! When profiling is *not* enabled, [`timed`] is just the wrapped call plus +//! one cheap thread-local check, so it's safe to leave on hot paths. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::time::Instant; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +/// Per-stage accumulated statistics for one scan. +#[derive(Default, Clone, Debug)] +pub struct Profile { + matchers: BTreeMap<&'static str, Stat>, +} + +/// Time, call count, and input bytes attributed to one stage. +#[derive(Default, Clone, Copy, Debug)] +pub struct Stat { + /// Total nanoseconds spent in this stage. + pub ns: u64, + /// Number of times the stage ran. + pub calls: u64, + /// Total bytes of input handed to the stage. + pub bytes: u64, +} + +impl Profile { + /// Iterate `(stage_name, stat)` in name order. + pub fn iter(&self) -> impl Iterator + '_ { + self.matchers.iter().map(|(k, v)| (*k, *v)) + } +} + +/// Begin profiling on this thread, discarding any in-progress profile. +pub fn enable() { + CURRENT.with(|c| *c.borrow_mut() = Some(Profile::default())); +} + +/// Stop profiling on this thread and return the accumulated profile, or `None` +/// if profiling was never enabled. +pub fn take() -> Option { + CURRENT.with(|c| c.borrow_mut().take()) +} + +#[inline] +fn active() -> bool { + CURRENT.with(|c| c.borrow().is_some()) +} + +/// Run `f`, attributing its wall time and `bytes` of input to stage `name` +/// when profiling is active; otherwise just runs `f`. Re-entrant: no borrow is +/// held across `f`, so nested instrumented calls accumulate independently. +/// +/// Nesting double-counts by design: a stage wrapping other instrumented calls +/// (e.g. `unpack`, which drives member scans) reports inclusive wall time. +#[inline] +pub fn timed(name: &'static str, bytes: u64, f: impl FnOnce() -> T) -> T { + if !active() { + return f(); + } + let t = Instant::now(); + let r = f(); + let ns = t.elapsed().as_nanos() as u64; + CURRENT.with(|c| { + if let Some(p) = c.borrow_mut().as_mut() { + let e = p.matchers.entry(name).or_default(); + e.ns += ns; + e.calls += 1; + e.bytes += bytes; + } + }); + r +} diff --git a/crates/exav/src/main.rs b/crates/exav/src/main.rs index 9ba62b2d..f76b3d16 100644 --- a/crates/exav/src/main.rs +++ b/crates/exav/src/main.rs @@ -3932,8 +3932,43 @@ fn scan_one_allmatch( exav_core::analyze_all_with_outcome(db, &data, opts) })); match found { - Ok((dets, _)) if !dets.is_empty() => { + Ok((dets, outcome)) if !dets.is_empty() => { totals.infected += 1; + // A detection does not mean the search finished. All-match exists to + // report everything that matched, so "here is what matched" and "we + // stopped early, there may be more" are different claims and the + // first must not silently imply the second. The no-detection arm + // below already turns this outcome into a verdict; with detections + // present the status word is FOUND, which cannot also say PARTIAL, + // so the incompleteness is reported alongside rather than instead. + // + // Reusing `Verdict` rather than mapping the outcome to a tag here: + // `status_tag`/`detail` then produce the same words as every other + // line, and adding a fourth partial condition cannot leave this path + // behind. + let partial = match outcome { + exav_core::AllMatchOutcome::Complete => None, + exav_core::AllMatchOutcome::LimitsExceeded(reason) => { + Some(Verdict::LimitsExceeded { reason }) + } + exav_core::AllMatchOutcome::Unscannable(reason) => { + Some(Verdict::Unscannable { reason }) + } + exav_core::AllMatchOutcome::PasswordProtected(reason) => { + Some(Verdict::PasswordProtected { reason }) + } + } + // `--partial-as ok` asked for partials to pass and `--partial-as + // found` for them to read as detections — which this object already + // is. Either way the extra line would be noise the operator asked + // not to get, so it is only emitted for the two statuses that mean + // "tell me". + .filter(|v| { + matches!( + policy::current().for_tag(v.status_tag()), + policy::PartialStatus::Partial | policy::PartialStatus::Error + ) + }); if cli.json { let sigs: Vec = dets .iter() @@ -3944,12 +3979,22 @@ fn scan_one_allmatch( // `status` and no `category`, the shape `emit_json_result` uses: // `signatures` is the plural of its `signature`, which is the // one thing all-match genuinely adds. - println!( - "{}", - serde_json::json!({ - "file": name, "status": "FOUND", "signatures": sigs - }) - ); + let mut obj = serde_json::Map::new(); + obj.insert("file".into(), name.as_str().into()); + obj.insert("status".into(), "FOUND".into()); + obj.insert("signatures".into(), sigs.into()); + // Absent rather than `false` when the search did finish, like + // `category`: a consumer reads "is this key here", never a + // tri-state. Plural because one object can stop short for more + // than one reason once the outcome carries more than one. + if let Some(v) = &partial { + obj.insert("partial".into(), true.into()); + obj.insert( + "partial_reasons".into(), + serde_json::Value::Array(vec![v.detail().unwrap_or_default().into()]), + ); + } + println!("{}", serde_json::Value::Object(obj)); } else { for (sig, method) in dets { outln!("{name}: {sig} FOUND"); @@ -3957,6 +4002,16 @@ fn scan_one_allmatch( println!(" [method] {}", method.as_str()); } } + // Same grammar as every other line: `path: [reason ][CATEGORY ] + // STATUS`, status word last. + if let Some(v) = &partial { + outln!( + "{name}: {} {} {}", + v.detail().unwrap_or_default(), + v.status_tag(), + status_str(VerdictCategory::Partial, v.status_tag()) + ); + } if cli.bell { print!("\x07"); } diff --git a/crates/exav/src/metrics.rs b/crates/exav/src/metrics.rs index f69f0cba..a9f76507 100644 --- a/crates/exav/src/metrics.rs +++ b/crates/exav/src/metrics.rs @@ -45,6 +45,9 @@ pub(crate) const MATCHERS: &[&str] = &[ "bytecode", "static", "normalize", + "unpack", + "emu", + "filetype", ]; /// Counters for one matcher. @@ -86,6 +89,9 @@ static MATCHER_CELLS: [MatcherCell; CELLS] = [ MatcherCell::new(), MatcherCell::new(), MatcherCell::new(), + MatcherCell::new(), + MatcherCell::new(), + MatcherCell::new(), ]; static SCANS: AtomicU64 = AtomicU64::new(0); diff --git a/crates/exav/src/perf.rs b/crates/exav/src/perf.rs index c9139f1a..c1fb935f 100644 --- a/crates/exav/src/perf.rs +++ b/crates/exav/src/perf.rs @@ -10,7 +10,10 @@ use std::time::Duration; /// Matchers profiled by `--profile`, in fixed column order so the header and /// every row line up. Must match the names passed to `profile::timed` in the -/// scan path. +/// scan path. Appended, never reordered: the `verdict` column's position is +/// covered by test. `unpack` is inclusive wall time inside container-extraction +/// dispatch (nested member scans included — see the call site); `emu` splits +/// the packer emulator's share of it back out. const MATCHERS: &[&str] = &[ "engine", "yara", @@ -21,6 +24,9 @@ const MATCHERS: &[&str] = &[ "bytecode", "static", "normalize", + "unpack", + "emu", + "filetype", ]; /// The header row: metadata columns + `_us/_calls/_bytes` per matcher. diff --git a/crates/exav/tests/json_output.rs b/crates/exav/tests/json_output.rs index 00297734..3186609c 100644 --- a/crates/exav/tests/json_output.rs +++ b/crates/exav/tests/json_output.rs @@ -208,3 +208,90 @@ fn profile_reports_the_same_verdict_as_a_plain_scan() { ); } } + +/// A detection does not mean the search finished. In `--all-matches` the status +/// word is `FOUND`, which cannot also say `PARTIAL`, so an incomplete search has +/// to be reported alongside the detections instead of replacing them — on its +/// own line in the normal grammar, and as `partial`/`partial_reasons` in JSON. +/// +/// Before this, the all-match path discarded the outcome whenever it had +/// detections, so a truncated scan was indistinguishable from a complete one. +#[test] +fn all_matches_reports_an_incomplete_search_alongside_detections() { + let db = TempDir::new().unwrap(); + // `Test.Hit` is the detection. `Test.Wild` exists to burn the per-group + // step budget: its anchor repeats tens of thousands of times and the tail it + // searches for is never present, so every hit scans to the end of the file. + std::fs::write( + db.path().join("t.ndb"), + "Test.Hit:0:*:4558415654455354\nTest.Wild:0:*:51574552*5a584356\n", + ) + .unwrap(); + let files = TempDir::new().unwrap(); + let sample = files.path().join("sample.bin"); + let mut data = b"EXAVTEST".to_vec(); + data.extend(std::iter::repeat_n(*b"QWER", 50_000).flatten()); + std::fs::write(&sample, &data).unwrap(); + + let run = |cap: Option<&str>, json: bool| { + let mut c = exav(); + c.arg("-d").arg(db.path()).arg("--all-matches"); + if json { + c.arg("--json"); + } + if let Some(cap) = cap { + // Forces the repeated-anchor cap to refuse the group after its first + // charged verify, which is what flags the search truncated. + c.env("EXAV_MAX_GROUP_STEPS", cap); + } + c.arg(&sample).output().expect("run exav") + }; + + // Search completes: the detection stands alone, with nothing added. + let out = run(None, false); + let text = String::from_utf8_lossy(&out.stdout); + assert!(text.contains("Test.Hit FOUND"), "{text}"); + assert!( + !text.contains("PARTIAL"), + "a complete search must not be reported partial: {text}" + ); + assert_eq!(out.status.code(), Some(1)); + + // Search truncated: the detection still stands, and the incompleteness is + // reported next to it in the `[reason ][CATEGORY ]STATUS` grammar. + let out = run(Some("1"), false); + let text = String::from_utf8_lossy(&out.stdout); + assert!(text.contains("Test.Hit FOUND"), "{text}"); + let partial = text + .lines() + .find(|l| l.ends_with("PARTIAL")) + .unwrap_or_else(|| panic!("no PARTIAL line: {text}")); + assert!( + partial.ends_with("LIMITS-EXCEEDED PARTIAL"), + "category must precede the status word: {partial}" + ); + assert_eq!( + out.status.code(), + Some(1), + "a confirmed detection still decides the exit code" + ); + + // Same thing in JSON, as fields rather than a second record. + let out = run(Some("1"), true); + let text = String::from_utf8_lossy(&out.stdout); + let row: serde_json::Value = serde_json::from_str(text.lines().next().unwrap()).unwrap(); + assert_eq!(row["status"], "FOUND"); + assert_eq!(row["partial"], true); + assert!( + row["partial_reasons"] + .as_array() + .is_some_and(|r| !r.is_empty()), + "partial_reasons must name the cause: {row}" + ); + // And absent, not false, when the search finished. + let out = run(None, true); + let text = String::from_utf8_lossy(&out.stdout); + let row: serde_json::Value = serde_json::from_str(text.lines().next().unwrap()).unwrap(); + assert!(row.get("partial").is_none(), "{row}"); + assert!(row.get("partial_reasons").is_none(), "{row}"); +} From 2767f1927eb1d36368e2fb867d6fafcd0024c447 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Mon, 28 Sep 2026 09:38:25 +0200 Subject: [PATCH 03/27] pre-streaming refactor --- CHANGELOG.md | 89 ++ Makefile | 13 +- README.md | 21 +- crates/exav-core/README.md | 2 +- crates/exav-core/src/database.rs | 5 +- crates/exav-core/src/engine/mod.rs | 72 +- crates/exav-core/src/lib.rs | 1192 +++++++++++------ crates/exav-core/src/loader.rs | 103 +- crates/exav-core/src/profile.rs | 2 +- crates/exav-core/src/yara/mod.rs | 91 +- .../tests/suites/encrypted_archives.rs | 30 + .../tests/suites/heuristic_alerts.rs | 26 + .../exav-core/tests/suites/limits_alerts.rs | 60 + crates/exav-core/tests/suites/mod.rs | 2 + crates/exav-core/tests/suites/one_scan.rs | 94 ++ .../tests/suites/oversize_container.rs | 75 +- crates/exav-core/tests/suites/salvage.rs | 204 +++ crates/exav-pe-emu/src/win.rs | 126 +- crates/exav-unpack/Cargo.toml | 2 +- crates/exav-unpack/src/formats/cab.rs | 56 +- crates/exav-unpack/src/formats/cpio.rs | 35 +- crates/exav-unpack/src/formats/gzip.rs | 55 +- crates/exav-unpack/src/formats/iso.rs | 27 +- crates/exav-unpack/src/formats/machofat.rs | 9 +- crates/exav-unpack/src/formats/mod.rs | 2 +- crates/exav-unpack/src/formats/ole.rs | 97 +- crates/exav-unpack/src/formats/onenote.rs | 14 +- crates/exav-unpack/src/formats/partition.rs | 26 +- crates/exav-unpack/src/formats/pdf.rs | 52 +- crates/exav-unpack/src/formats/pepack.rs | 36 +- .../exav-unpack/src/formats/sevenz/entry.rs | 28 +- crates/exav-unpack/src/formats/sfx.rs | 9 +- crates/exav-unpack/src/formats/tnef.rs | 37 +- crates/exav-unpack/src/formats/zip.rs | 255 +++- crates/exav-unpack/src/inflate.rs | 415 ++++++ crates/exav-unpack/src/lib.rs | 194 ++- crates/exav-unpack/src/stream.rs | 274 ++-- crates/exav-unpack/src/volume.rs | 55 +- crates/exav-unpack/tests/suites/mod.rs | 2 + .../exav-unpack/tests/suites/pe_emulation.rs | 32 + .../tests/suites/stream_read_errors.rs | 80 ++ crates/exav-unpack/tests/suites/zip_codecs.rs | 53 + crates/exav-update/src/lib.rs | 47 +- crates/exav/README.md | 6 +- crates/exav/src/daemon.rs | 918 ++++++++++--- crates/exav/src/icap/chunked.rs | 8 + crates/exav/src/icap/mod.rs | 25 +- crates/exav/src/icap/server.rs | 173 ++- crates/exav/src/main.rs | 272 ++-- crates/exav/src/policy.rs | 4 +- crates/exav/src/spill.rs | 3 +- crates/exav/tests/daemon_client.rs | 175 ++- crates/exav/tests/icap_server.rs | 14 +- crates/exav/tests/listeners.rs | 256 +++- crates/exav/tests/one_scan.rs | 201 +++ docs/BYTECODE.md | 33 + docs/COMPARISON_NOTES.md | 106 ++ docs/PARTIAL.md | 6 +- docs/STREAMING_FORMATS.md | 34 + www/src/content/docs/concepts/architecture.md | 73 +- .../docs/concepts/archive-extraction.md | 347 +++-- .../content/docs/concepts/bytecode-sandbox.md | 128 +- .../docs/concepts/design-principles.md | 196 ++- .../docs/concepts/differential-testing.md | 138 +- www/src/content/docs/concepts/pe-emulation.md | 209 ++- www/src/content/docs/concepts/quirks.md | 1174 ++++++---------- .../content/docs/concepts/streaming-memory.md | 166 ++- .../docs/getting-started/installation.mdx | 35 +- .../docs/getting-started/introduction.md | 88 +- .../docs/getting-started/quick-start.mdx | 88 +- www/src/content/docs/guides/daemon.mdx | 254 ++-- www/src/content/docs/guides/docker.mdx | 151 +-- www/src/content/docs/guides/icap.mdx | 406 +++--- www/src/content/docs/guides/library-usage.md | 32 +- .../docs/guides/migrating-from-clamav.mdx | 251 ++-- .../content/docs/guides/prebuilt-database.mdx | 137 +- www/src/content/docs/guides/scanning.mdx | 157 +-- www/src/content/docs/guides/signatures.mdx | 152 +-- .../content/docs/guides/troubleshooting.md | 116 +- www/src/content/docs/guides/wasm-sandbox.mdx | 90 +- www/src/content/docs/guides/yara.mdx | 100 +- www/src/content/docs/index.mdx | 71 +- .../docs/project/comparison-with-clamav.md | 939 +++++-------- www/src/content/docs/project/contributing.md | 118 +- www/src/content/docs/project/license.md | 37 +- www/src/content/docs/project/roadmap.md | 157 +-- www/src/content/docs/project/security.md | 156 ++- .../docs/reference/clamav-flag-matrix.md | 423 +++--- www/src/content/docs/reference/cli.md | 569 ++++---- .../content/docs/reference/configuration.md | 117 +- .../content/docs/reference/dependencies.md | 155 +-- .../content/docs/reference/feature-flags.md | 127 +- www/src/content/docs/reference/formats.md | 427 +++--- www/src/content/docs/reference/limits.md | 125 +- www/src/content/docs/reference/verdicts.md | 147 +- www/src/content/docs/subprojects/exav-core.md | 38 +- www/src/content/docs/subprojects/exav-grep.md | 44 +- .../content/docs/subprojects/exav-pe-emu.md | 81 +- .../content/docs/subprojects/exav-unpack.md | 139 +- .../content/docs/subprojects/exav-update.md | 28 +- www/src/content/docs/subprojects/exav-x86.md | 53 +- www/src/content/docs/subprojects/index.md | 34 +- www/src/styles/custom.css | 2 +- 103 files changed, 8647 insertions(+), 6161 deletions(-) create mode 100644 crates/exav-core/tests/suites/one_scan.rs create mode 100644 crates/exav-unpack/src/inflate.rs create mode 100644 crates/exav-unpack/tests/suites/stream_read_errors.rs create mode 100644 crates/exav/tests/one_scan.rs create mode 100644 docs/COMPARISON_NOTES.md create mode 100644 docs/STREAMING_FORMATS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5acd3b79..dd72e372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,95 @@ Notable changes per release. Dates are release dates; the format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) loosely, and versions follow [semantic versioning](https://semver.org/). +## [Unreleased] + +### Changed + +- A file or archive member larger than `--max-object-bytes` (256 MiB by + default) is reported `LIMITS-EXCEEDED` unless a literal signature or a + whole-file hash matches it. It was reported `OK` after a literal-and-hash pass + that skipped wildcard and logical signatures, YARA and bytecode. An archive over + the limit is walked as before and reported `LIMITS-EXCEEDED` too. +- YARA detections are named `YARA.` outside `--clamav-compat`; the + `.UNOFFICIAL` suffix is compat-only, as for every other signature. +- A signature reload lets scans in progress finish: old workers and the old ICAP + child exit once idle instead of being killed. One still busy after + `--max-scan-secs` plus 5 seconds (an open session, a client that never + finishes) is stopped. +- A worker stopped mid-job (shutdown, reload deadline, supervisor gone) answers + the job with `ERROR` instead of closing the connection without a reply. +- `off` and `0` on `--max-object-bytes` and `--max-matcher-bytes` mean no limit, + as their help said; they used to set a limit of zero. +- Every input goes through one scan: a file, stdin, a FIFO given as a path, + `INSTREAM`, `EXINSTREAM`, ICAP and `http(s)://`. In particular: + - Past `--max-input-bytes`, the first bytes get the whole scan. A file got + only literal signatures and hashes there, and stdin and the daemon's stream + verbs no scan at all. + - A stream larger than the spill budgets is scanned as far as it was held and + reported `LIMITS-EXCEEDED`. It was `UNSCANNABLE`, unscanned. + - Both follow `--partial-as` as a file does; under `found` they are + `Heuristics.Limits.Exceeded.MaxFileSize` everywhere. + - A container over `--max-object-bytes` gets the literal and hash pass over + its own bytes on every entry point, not only a file path. + - Stdin honours `--all-matches`, `--profile` and `--max-process-bytes`, and + counts in the summary. +- exav-core: `scan_stream` is removed (the one scan is `scan_seekable`, and + `scan_path` hands a file to it); `warm_up` builds the lazily-initialised + structures before a fork. +- A stream that ends before its terminator is answered `ERROR` on `INSTREAM` + and `EXINSTREAM`, whatever `--partial-as` says. `EXINSTREAM` used to answer + `PARTIAL` with a `TRUNCATED` category, and `INSTREAM` `UNSCANNABLE`. +- A compressed stream damaged part way (gzip, ZIP, PDF, embedded PowerPoint + storages) is `UNSCANNABLE` when the part before the damage holds no + detection, wherever it sits; a checksum mismatch after a full decode is not. + Nested in another container it used to be `OK`, and a gzip with only a bad + CRC was `UNSCANNABLE` at the top level. + +### Added + +- `--max-pe-emulation-steps` (default 1,000,000,000): instructions the PE + unpacking emulator may run across one top-level file, reported as + `LIMITS-EXCEEDED` / `Heuristics.Limits.Exceeded.MaxScanTime`. +- An encrypted ZIP appended to a picture or document is reported + `PASSWORD-PROTECTED` when its central directory checks out. +- `make miri`. + +### Fixed + +- A top-level archive now gets the full engine over its own bytes (wildcard and + logical signatures, YARA, bytecode, whole-file hashes), as a nested one did, on + every entry point including stdin, `INSTREAM` and ICAP. +- `.ign`/`.ign2` entries and `.fp`/`.sfp` allowlists apply on every scan path, + and an ignored signature no longer hides a later detection in the same file. +- ZIP members absent from the central directory are scanned on the streamed path + the top-level scan takes. +- Clean ZIP members compressed with LZMA, bzip2, XZ, zstd or Deflate64 were also + reported as an unsupported codec on the streamed path. +- `.db` signatures were never matched; PUA literal signatures matched without + `--detect pua`. +- A read error from the source is reported instead of being taken for the end of + a container. +- A job stopped by `--max-scan-secs` is answered `LIMITS-EXCEEDED` instead of the + connection closing with no reply. +- A YARA rule set that runs out of evaluation steps makes the scan + `LIMITS-EXCEEDED` instead of a silent non-match. +- A reload no longer adds a worker to the pool for every worker it retires. +- A worker exit, `RELOAD` or shutdown signal that arrives while the daemon's + supervisor is busy is handled at once, not up to 10 seconds later. +- `EXINSTREAM` answers a stream it cannot hold with a verdict, as `INSTREAM` + does, instead of an error; `EXINSTREAM MULTI` answers that file alone and + continues. +- A FIFO given as a path is scanned; it was an `Illegal seek` error. +- The daemon reads its cgroup memory limit from its own cgroup and its parents. +- Percent-encoded credentials in update URLs are decoded before use. +- The signature count no longer counts literal signatures twice. +- What a deflate stream (gzip, ZIP, PDF, PowerPoint storages) decoded in the + read that met damage, tens of KiB, was dropped unscanned; a detection just + before the damage was missed. +- The bytes a damaged CAB or ZOO member decoded before the damage are scanned; + they were reported as scanned and skipped. +- The `ole` feature builds on its own. + ## [0.0.1] - 2026-09-16 First public release. Everything below is new, so this section describes what diff --git a/Makefile b/Makefile index f4dc5a50..f7204042 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ DBDIR ?= exav-db EXAVDB ?= exav.exavdb .DEFAULT_GOAL := build -.PHONY: build release test test-native test-yara-diff test-wasm test-js test-www wasm-sizes lint fmt msrv av-audit publish-check publish fuzz db exavdb cache daily clean www-dev www-build help +.PHONY: build release test test-native test-yara-diff test-wasm test-js test-www wasm-sizes lint fmt msrv av-audit miri publish-check publish fuzz db exavdb cache daily clean www-dev www-build help ## build: compile the release binary build release: @@ -137,6 +137,17 @@ av-audit: fi; \ echo "no detections" +## miri: run the container-walking tests of exav-unpack under Miri, which checks +## the `unsafe` inside dependencies (memchr, the compression and crypto +## crates) along real code paths. Needs a nightly toolchain with the +## component: `rustup toolchain install nightly --component miri`. +## A subset, chosen by test-name filter in MIRI_TESTS: Miri runs code +## orders of magnitude slower, and the decoder round-trip tests alone take +## hours. Its own target dir, so it never invalidates the normal build. +MIRI_TESTS ?= formats::zip formats::cpio formats::tar formats::iso formats::partition formats::cab +miri: + CARGO_TARGET_DIR=target/miri rustup run nightly cargo miri test -p exav-unpack --lib -- $(MIRI_TESTS) + ## fuzz: smoke-build the fuzz targets fuzz: cd fuzz && $(CARGO) build diff --git a/README.md b/README.md index 9a9cb8cf..024ed0d0 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,9 @@ **A fast, memory-safe malware scanner written in Rust.** -exav scans files of any size in constant memory, reads ClamAV's signature -databases, and answers the `clamd` and ICAP wire protocols — so it drops into -an existing ClamAV setup without changing what talks to it. It prints -`clamscan`'s output format; its flags are its own. +exav reads ClamAV's signature databases and answers the `clamd` and ICAP wire +protocols, so it drops into an existing ClamAV setup without changing what talks +to it. It prints `clamscan`'s output format; its flags are its own. One deliberate difference: a file exav could not fully examine is never reported clean. It gets status `PARTIAL` (exit 3) instead of ClamAV's `OK` @@ -15,8 +14,8 @@ reported clean. It gets status `PARTIAL` (exit 3) instead of ClamAV's `OK` cargo install exav ``` -exav ships **no signatures** (the ClamAV database is GPL) — with none loaded, -it refuses to run rather than report clean. +exav ships **no signatures** (the ClamAV database is GPL); with none loaded, it +refuses to run rather than report clean. **Full documentation: [exav.org](https://exav.org).** Start here: [Install](https://exav.org/getting-started/installation/), @@ -24,12 +23,12 @@ it refuses to run rather than report clean. [Migrating from ClamAV](https://exav.org/guides/migrating-from-clamav/), [CLI reference](https://exav.org/reference/cli/). -> **Status: beta.** Tested hard — differentially against ClamAV, plus fuzzing — -> but young. Please report anything that looks wrong. +> **Status: beta.** Tested differentially against ClamAV and fuzzed, but young. +> Please report anything that looks wrong. -- **Library** — [using exav as a Rust library](https://exav.org/guides/library-usage/). +- **Library:** [using exav as a Rust library](https://exav.org/guides/library-usage/). The `exav-core` API is `0.0.x` and may break in any release. -- **Contributing** — see [`CONTRIBUTING.md`](CONTRIBUTING.md) and the +- **Contributing:** see [`CONTRIBUTING.md`](CONTRIBUTING.md) and the [guide](https://exav.org/project/contributing/). Clean-room rule: nothing derived from ClamAV's GPL sources. -- **Security** — see [`SECURITY.md`](SECURITY.md). **License** — [MIT](LICENSE). +- **Security:** see [`SECURITY.md`](SECURITY.md). **License:** [MIT](LICENSE). diff --git a/crates/exav-core/README.md b/crates/exav-core/README.md index 93b91863..90189c98 100644 --- a/crates/exav-core/README.md +++ b/crates/exav-core/README.md @@ -4,7 +4,7 @@ The core scanning engine of [**exav**](https://github.com/sylvinus/exav), a memory-safe ClamAV replacement written in Rust. `exav-core` provides the library API for loading ClamAV signature databases and -scanning files or streams in constant memory: signature-database parsing +scanning files or streams in bounded memory: signature-database parsing (`.ndb`/`.ldb`/`.hdb`/`.cvd`/…), pattern/hash/fuzzy matching, optional YARA (via the native engine in `exav_core::yara`), a sandboxed `.cbc` bytecode interpreter, and PE/ELF/Mach-O parsing. Archive/container extraction lives in the companion diff --git a/crates/exav-core/src/database.rs b/crates/exav-core/src/database.rs index b2f9cb4d..9a1d4ad2 100644 --- a/crates/exav-core/src/database.rs +++ b/crates/exav-core/src/database.rs @@ -10,9 +10,8 @@ //! //! The file starts with a magic tag and a format version; a mismatch is //! rejected rather than misread. The database is a trusted artifact (you build -//! and fetch it yourself, over a channel you trust) — it is not a safe target -//! for arbitrary untrusted bytes, since the automaton bytes are deserialized -//! without full validation. +//! and fetch it yourself, over a channel you trust): its contents decide what is +//! detected, and nothing here checks who produced it. use std::io::{self, Read, Seek, SeekFrom, Write}; use std::path::Path; diff --git a/crates/exav-core/src/engine/mod.rs b/crates/exav-core/src/engine/mod.rs index bed74e8e..83f11af0 100644 --- a/crates/exav-core/src/engine/mod.rs +++ b/crates/exav-core/src/engine/mod.rs @@ -339,6 +339,13 @@ pub fn reset_scan_truncated() { SCAN_TRUNCATED.with(|c| c.set(false)); } +/// Record that a matcher outside this module (YARA's condition step budget) +/// stopped short, so the scan is reported as incomplete. +#[cfg_attr(not(feature = "yara"), allow(dead_code))] +pub(crate) fn mark_scan_truncated() { + SCAN_TRUNCATED.with(|c| c.set(true)); +} + /// Did any wildcard verification get skipped this scan because its per-buffer /// verify budget was exhausted? If so the scan was INCOMPLETE and must not be /// reported `Clean`. @@ -1459,6 +1466,11 @@ impl EngineBuilder { /// built-in EICAR test pattern into the engine so in-memory scans don't /// also need the streaming literal automaton. pub fn add_literal(&mut self, name: &str, bytes: &[u8]) { + self.add_literal_prov(name, bytes, false); + } + + /// As [`Self::add_literal`], with `unofficial` provenance (a loose `.db`). + pub fn add_literal_prov(&mut self, name: &str, bytes: &[u8], unofficial: bool) { if bytes.len() < MIN_ANCHOR { return; } @@ -1476,7 +1488,7 @@ impl EngineBuilder { fullword_len: None, owner: Owner::Ndb { name: name.to_string(), - unofficial: false, + unofficial, }, }); } @@ -1835,18 +1847,6 @@ impl EngineBuilder { self.build_with_budget(None) } - /// As [`EngineBuilder::build`], but bounds the per-shard automaton-build - /// transient to roughly `max_build_mem` bytes by sharding each large - /// `(target, case)` partition into multiple automatons. `None` builds one - /// automaton per partition (fastest, largest transient). Only the build-time - /// peak is affected — scan results are identical. - /// - /// Anchors are partitioned by `(target, case)`: a scan then runs only the - /// partitions whose `target` matches the file type (a PE file never - /// traverses the ELF/HTML/text automatons), so the per-body target check is - /// implied by which partition a body lives in. Within a partition, identical - /// anchors de-duplicate into one automaton pattern that fans out to every - /// body sharing it. /// Every candidate anchor literal in the database, handed to `f` once per /// body that contains it. Bodies whose `elems` were dropped (a pure literal /// whose anchor IS the whole pattern) still contribute: their literal is in @@ -1935,6 +1935,18 @@ impl EngineBuilder { self.anchor_ranges = ranges; } + /// As [`EngineBuilder::build`], but bounds the per-shard automaton-build + /// transient to roughly `max_build_mem` bytes by sharding each large + /// `(target, case)` partition into multiple automatons. `None` builds one + /// automaton per partition (fastest, largest transient). Only the build-time + /// peak is affected; scan results are identical. + /// + /// Anchors are partitioned by `(target, case)`: a scan then runs only the + /// partitions whose `target` matches the file type (a PE file never + /// traverses the ELF/HTML/text automatons), so the per-body target check is + /// implied by which partition a body lives in. Within a partition, identical + /// anchors de-duplicate into one automaton pattern that fans out to every + /// body sharing it. pub fn build_with_budget(mut self, max_build_mem: Option) -> SigEngine { self.repick_anchors(); // The double-array construction transient scales ~linearly with the @@ -2782,11 +2794,12 @@ impl SigEngine { cand } - /// Sweep the active automatons, buffering every hit. `None` when the - /// buffer cap (or a buffer too large for `u32` positions) rules it out, - /// which tells the caller to take the direct inline path. + /// Sweep the active automatons, buffering every hit. `None` when there + /// is nothing to gate (no logical signatures), or the buffer cap (or a + /// buffer too large for `u32` positions) rules it out, which tells the + /// caller to take the direct inline path. fn sweep_hits(&self, ft: FileType, buf: &[u8], lower: &[u8]) -> Option { - if buf.len() > u32::MAX as usize { + if self.ldbs.is_empty() || buf.len() > u32::MAX as usize { return None; } let cap = max_buffered_hits(); @@ -2947,6 +2960,25 @@ impl SigEngine { layout: Option<&PeLayout>, container: Option, icon_ctx: Option<&IconCtx>, + ) -> Option<(String, u64, bool)> { + self.scan_first(buf, ft, layout, container, icon_ctx, &|_, _| false) + } + + /// As [`Self::scan_with_icons`], passing over every signature for which + /// `skip(clean_name, unofficial)` holds and carrying on to the next match. + /// + /// This is how an ignore list (`.ign`/`.ign2`) takes effect. Filtering the + /// result afterwards is not enough: a first-match scan stops at the ignored + /// signature, so a real one matching later in the same buffer is never + /// looked for. + pub(crate) fn scan_first( + &self, + buf: &[u8], + ft: FileType, + layout: Option<&PeLayout>, + container: Option, + icon_ctx: Option<&IconCtx>, + skip: &dyn Fn(&str, bool) -> bool, ) -> Option<(String, u64, bool)> { // Per-LDB-subsig match counts; only allocated when there are logical // signatures (otherwise no LdbSub body exists and counts is unused). @@ -2978,6 +3010,9 @@ impl SigEngine { self.walk_candidates(buf, ft, layout, &lower, |bid, start| { match &self.bodies[bid].owner { Owner::Ndb { name, unofficial } => { + if skip(name, *unofficial) { + return std::ops::ControlFlow::Continue(()); + } ndb_hit = Some((name.clone(), start, *unofficial)); return std::ops::ControlFlow::Break(()); } @@ -3037,6 +3072,9 @@ impl SigEngine { RETYPE.with(|c| c.set(Some((buf.as_ptr() as usize, buf.len(), ft)))); continue; } + if skip(&ldb.name, ldb.unofficial) { + continue; + } return Some((ldb.name.clone(), 0, ldb.unofficial)); } } diff --git a/crates/exav-core/src/lib.rs b/crates/exav-core/src/lib.rs index 9dc812c8..3984790f 100644 --- a/crates/exav-core/src/lib.rs +++ b/crates/exav-core/src/lib.rs @@ -1,10 +1,11 @@ //! Streaming, size-unbounded file scanner. //! -//! The pattern and whole-file-hash core ([`scan_stream`]) reads any input in -//! a single forward pass with constant memory, matching across internal -//! buffer boundaries, so file size is unbounded. Structural analysis -//! (archive extraction, PE parsing, ML/fuzzy) needs the whole object in -//! memory and so runs only for objects within [`ScanOptions::deep_analysis_max`]. +//! Every input goes through one scan, [`scan_seekable`] ([`scan_path`] opens a +//! file and hands it over). Structural analysis (archive extraction, PE +//! parsing, ML/fuzzy) needs the whole object in memory and so runs for +//! objects within [`ScanOptions::deep_analysis_max`]; a larger one gets the +//! constant-memory literal and whole-file-hash pass and is reported +//! `LimitsExceeded` unless that pass finds something. //! //! # Anti-evasion invariants //! @@ -17,11 +18,12 @@ //! is [`Verdict::Infected`] — never downgraded to `LimitsExceeded`/`Clean` //! because some *other* part of the input tripped a budget. Limits bound //! work; they never suppress a hit already found. -//! 2. **Never refuse by size without scanning.** A file over `--max-scan-size` -//! is not skipped wholesale — the flat pattern/hash core is still run over -//! the bytes within budget, so a detectable payload in the scanned prefix is -//! reported. Only then, with nothing found, do we fall back to a limit -//! verdict. (Otherwise `cat malware huge.pad > evil` is a one-line bypass.) +//! 2. **Never refuse by size without scanning.** An input over +//! `--max-input-bytes` is not skipped wholesale: its first +//! `--max-input-bytes` get the same scan as a smaller input, so a +//! detectable payload there is reported. Only then, with nothing found, do +//! we fall back to a limit verdict. (Otherwise `cat malware huge.pad > evil` +//! is a one-line bypass.) //! 3. **Not-fully-scanned is never `Clean`.** Anything we couldn't fully examine //! — a size/ratio/depth/scan-byte limit ([`Verdict::LimitsExceeded`]) or a //! recognised-but-undecodable container ([`Verdict::Unscannable`], e.g. an @@ -132,6 +134,8 @@ use unpack::Budget; /// hash identifies a file" is refused for something too small to be one. No /// detection is lost, because every signature below the floor is one ClamAV /// itself can never fire, so nothing can depend on it. +const MIN_HASH_MATCH_BYTES: u64 = 6; + /// What a scan reports when a per-buffer bound — the step pool, or the /// repeated-anchor cap — ended the signature search before it finished. /// @@ -139,8 +143,6 @@ use unpack::Budget; /// apart in what they call the same condition. const SEARCH_INCOMPLETE: &str = "scan budget exhausted — signature search incomplete"; -const MIN_HASH_MATCH_BYTES: u64 = 6; - /// Whether a whole-file hash signature may be matched against an object of /// this size. See [`MIN_HASH_MATCH_BYTES`]. fn hash_matchable(size: u64) -> bool { @@ -387,6 +389,12 @@ fn carrier_cltype(ft: FileType) -> Option { }) } +/// Bytes read from the front of a ZIP to detect its OOXML sub-type: the OOXML +/// part names (`[Content_Types].xml`, `word/document.xml`, …) sit in the leading +/// local file headers, so a bounded prefix reliably classifies real Office docs +/// without buffering a large archive. +const OOXML_DETECT_PREFIX: usize = 1 << 20; + /// Detect whether a ZIP is an Office Open XML document and which kind, by the /// member names present in its central directory. A `.docx`/`.xlsx`/`.pptx` is /// the `CL_TYPE_OOXML_*` container type (not plain `CL_TYPE_ZIP`) in the @@ -394,12 +402,6 @@ fn carrier_cltype(ft: FileType) -> Option { /// way keeps such sigs firing on real Office documents while staying off plain /// ZIPs. The marker is the conventional top-level part for each kind plus the /// OOXML `[Content_Types].xml` package descriptor. -/// Bytes read from the front of a ZIP to detect its OOXML sub-type: the OOXML -/// part names (`[Content_Types].xml`, `word/document.xml`, …) sit in the leading -/// local file headers, so a bounded prefix reliably classifies real Office docs -/// without buffering a large archive. -const OOXML_DETECT_PREFIX: usize = 1 << 20; - fn ooxml_subtype(data: &[u8]) -> Option { use engine::ClType; // Cheap necessary condition: every OOXML package carries this part. @@ -639,24 +641,47 @@ impl ScanReport { } fn limits(reason: String, findings: Vec) -> Self { Self { - verdict: Verdict::LimitsExceeded { reason }, + verdict: Verdict::LimitsExceeded { + reason: one_line(reason), + }, findings, } } fn unscannable(reason: String, findings: Vec) -> Self { Self { - verdict: Verdict::Unscannable { reason }, + verdict: Verdict::Unscannable { + reason: one_line(reason), + }, findings, } } fn password_protected(reason: String, findings: Vec) -> Self { Self { - verdict: Verdict::PasswordProtected { reason }, + verdict: Verdict::PasswordProtected { + reason: one_line(reason), + }, findings, } } } +/// A verdict reason with its control characters replaced. +/// +/// Reasons quote container member names, which come from the file being +/// scanned. Every front-end prints the reason inside a line-framed reply (the +/// clamd protocol, the clamscan-style CLI line, the `--log` file), so a newline +/// in a member name would let the file write a line of its own, such as a +/// second `stream: OK` for a client that reads the last line it gets. +fn one_line(reason: String) -> String { + if !reason.chars().any(char::is_control) { + return reason; + } + reason + .chars() + .map(|c| if c.is_control() { '_' } else { c }) + .collect() +} + /// Options controlling a scan and its limits. /// /// New fields may appear in any release. Build with @@ -726,15 +751,14 @@ pub struct ScanOptions { /// reach beyond stock ClamAV); off under `--clamav-compat` and via `--no-base64`. /// FP-safe: only a decode with a valid executable header is rescanned. pub decode_base64: bool, - /// Append `.UNOFFICIAL` (and the `YARA.` prefix) to signature names that come - /// from unofficial databases, matching stock `clamscan`'s output. Purely + /// Append `.UNOFFICIAL` to signature names that come from unofficial + /// databases, matching stock `clamscan`'s output. Purely /// cosmetic — it changes only how a detection is *named*, never whether it /// fires. Default off. (The other behaviour `--clamav-compat` turns on.) pub unofficial_suffix: bool, /// Password(s) to try when decrypting encrypted archive members. Empty by /// default. When a scan returns [`Verdict::PasswordProtected`], a caller can - /// set this and re-scan. (The decryptors that consume it are a follow-up; - /// the field is the stable API the verdict points callers to.) + /// set this and re-scan. pub passwords: Vec, /// Verify archive checksums (CRCs) during extraction. **Off by default** — a /// scanner scans decompressed content regardless of integrity metadata, so a @@ -1064,9 +1088,10 @@ impl Scanner { ft } + /// Distinct signatures loaded. The streaming literal set is not counted: it + /// is a copy of literals the engine also holds. pub fn signature_count(&self) -> usize { - self.patterns.len() - + self.engine.signature_count() + self.engine.signature_count() + self.hashes.len() + self.sections.len() + self.fuzzy.len() @@ -1102,9 +1127,10 @@ impl Scanner { } } -/// Scan a local file path (Seekable mode). The pattern+hash core handles -/// any size in constant memory; structural analysis runs for files within -/// `deep_analysis_max`. +/// Scan a local file: [`scan_seekable`] over the opened file, with its path +/// as the YARA `filepath`/`filename`/`extension` unless `opts.filename` is +/// set. The file must be able to seek; a FIFO or other stream is buffered by +/// the caller first. /// /// # Errors /// @@ -1138,93 +1164,45 @@ pub fn scan_path(db: &Scanner, path: &Path, opts: &ScanOptions) -> io::Result max { - // Invariant 2: don't refuse by size without looking. Run the flat - // core over the budgeted prefix so a detection there is still - // reported; only with nothing found do we return a (non-clean) - // limit verdict for the unscanned remainder. - if let Some(hit) = stream_core(db, (&file).take(max))? { - return Ok(ScanReport::infected_hit( - hit, - opts.unofficial_suffix, - Vec::new(), - )); - } - return Ok(max_file_size_report(size, max, opts)); - } - } - - let ft = peek_type(&file)?; - - // Natively-streaming containers (ZIP/tar/gzip) are walked member-by-member - // off the file handle — the container is never buffered whole, so the - // `deep_analysis_max` cap does not apply and an arbitrarily large on-disk - // archive is scanned in bounded memory (one decoded member at a time). This - // is what closes the ">deep-analysis-max archive hiding a payload" gap for - // these formats: a 4 GiB `.tar.gz` is now unpacked, not skipped. - if streams_natively(ft) { - // Raw-container scan (whole-file hash + patterns over the compressed/ - // structural bytes), in constant memory — this preserves the whole-file - // hash and raw-pattern detections that the buffered `analyze` path runs - // via `scan_bytes_core` before unpacking. (A range-backed reader in - // `scan_seekable` intentionally skips this so a range-GET fetches only the - // members it scans; a local file always pays the cheap full read.) - if let Some(hit) = stream_core(db, &file)? { - return Ok(ScanReport::infected_hit( - hit, - opts.unofficial_suffix, - Vec::new(), - )); - } - return Ok(scan_container_stream(db, file, opts, ft)); - } + scan_seekable(db, file, size, opts) +} - if size <= opts.deep_analysis_max { - // Read from the handle we already stat'd (no second open, no TOCTOU) - // and bound the read so a special file whose metadata under-reports - // its length (a FIFO/device reports 0) can't stream unbounded. - let mut data = Vec::new(); - (&file) - .take(opts.deep_analysis_max.saturating_add(1)) - .read_to_end(&mut data)?; - if data.len() as u64 > opts.deep_analysis_max { - return Ok(ScanReport::limits( - format!("input exceeds deep-analysis-max {}", opts.deep_analysis_max), - Vec::new(), - )); - } - return Ok(analyze(db, &data, opts)); - } +/// The report for a file larger than `deep_analysis_max` on which the +/// streaming core found nothing. +/// +/// That core matches literal `.ndb` signatures and whole-file hashes only: no +/// wildcard or logical signature, no YARA, no normalised text, no unpacking. +/// Whatever the file's type, that is not a complete scan, so it is not clean. +fn over_deep_analysis_limit(ft: FileType, size: u64, opts: &ScanOptions) -> ScanReport { + let reason = format!( + "{} is {size} bytes, over the {}-byte deep-analysis limit \ + (--max-object-bytes); only literal signatures and whole-file hashes \ + were matched", + ft.as_str(), + opts.deep_analysis_max + ); + report_of_outcome( + limits_outcome(opts, unpack::LimitKind::MaxFileSize, reason), + Vec::new(), + opts, + ) +} - // Too large to buffer for structural analysis. Stream the pattern+hash - // core; that alone is a complete scan for flat content. - if let Some(hit) = stream_core(db, &file)? { - return Ok(ScanReport::infected_hit( - hit, - opts.unofficial_suffix, - Vec::new(), - )); - } - // But an archive or executable can hide detections that only structural - // parsing would find, and we skipped that. We cannot call it clean. - if unpack_format(ft).is_some() || ft.is_executable() { - return Ok(ScanReport::limits( - format!( - "{} is {size} bytes (> deep-analysis-max {}); contents not unpacked", - ft.as_str(), - opts.deep_analysis_max - ), - Vec::new(), - )); - } - Ok(ScanReport::clean(vec![Finding::new( - "type", - format!("{} (flat content, fully scanned)", ft.as_str()), - )])) +/// The report for a file that was under the deep-analysis limit when sized and +/// over it by the time it was read (it grew, or it is a special file). +fn grew_past_deep_analysis_limit(opts: &ScanOptions) -> ScanReport { + let reason = format!( + "input grew past the {}-byte deep-analysis limit (--max-object-bytes) \ + while it was read", + opts.deep_analysis_max + ); + report_of_outcome( + limits_outcome(opts, unpack::LimitKind::MaxFileSize, reason), + Vec::new(), + opts, + ) } /// Read into `buf` until it is full or EOF, retrying short/interrupted reads. @@ -1243,29 +1221,44 @@ fn fill_prefix(reader: &mut R, buf: &mut [u8]) -> io::Result { Ok(filled) } -/// Read a prefix to identify the type, then rewind so the caller can scan -/// from the start. -fn peek_type(file: &File) -> io::Result { - let mut prefix = [0u8; 4096]; - let mut r = file; - let n = fill_prefix(&mut r, &mut prefix)?; - let mut s = file; - s.seek(SeekFrom::Start(0))?; - Ok(profile::timed("filetype", n as u64, || { - filetype::identify(&prefix[..n]) - })) +/// The whole of a seekable source, or `None` when it is larger than `cap`. +/// Leaves the reader at the start either way. +fn read_bounded(reader: &mut R, cap: u64) -> io::Result>> { + let size = reader.seek(SeekFrom::End(0))?; + reader.seek(SeekFrom::Start(0))?; + if size > cap { + return Ok(None); + } + let mut buf = Vec::new(); + reader.by_ref().take(cap).read_to_end(&mut buf)?; + reader.seek(SeekFrom::Start(0))?; + Ok(Some(buf)) +} + +/// Whether `data`'s whole-file hash is on the `.fp`/`.sfp` allowlist. +fn allowlisted(db: &Scanner, data: &[u8]) -> bool { + !db.allow.is_empty() + && db + .allow + .lookup(&digests_of(data), data.len() as u64) + .is_some() +} + +/// Build every structure a scan initialises lazily (engine automata, compiled +/// YARA rules, the constant-memory pass's automaton), so a process that forks +/// workers after loading shares them copy-on-write instead of each worker +/// building its own. +pub fn warm_up(db: &Scanner) { + const PROBE: &[u8] = b"MZ\x90\x00\x00\x00\x00\x00"; + let _ = analyze(db, PROBE, &ScanOptions::default()); + let _ = stream_core(db, PROBE, false); } -/// Scan a sequential stream (Stream mode): stdin, pipes, S3 streaming GET. -/// Runs the constant-memory pattern + hash core. (Structural analysis of -/// random-access formats needs a bounded buffer-upgrade or the seekable -/// backend — tracked for the S3 range-GET source.) -pub fn scan_stream(db: &Scanner, reader: R) -> io::Result { - // No `ScanOptions` here, so compat naming isn't applied: the streaming path - // reports clean signature names (exav's default). Use [`scan_seekable`] / - // [`scan_path`] (which take `ScanOptions`) when compat `.UNOFFICIAL` output is - // required. - if let Some(hit) = stream_core(db, reader)? { +/// The constant-memory pass alone, for the tests that pin it down. +#[cfg(test)] +pub(crate) fn scan_stream(db: &Scanner, reader: R) -> io::Result { + engine::reset_scan_truncated(); + if let Some(hit) = stream_core(db, reader, false)? { return Ok(suppress_name( db, ScanReport::infected_hit(hit, false, Vec::new()), @@ -1367,89 +1360,138 @@ pub fn scan_seekable_located( Ok((report, loc)) } -/// Scan a seekable input (a local file, or `source::HttpRangeReader` with the -/// `http` feature). A ZIP -/// is read through the seekable reader, so only its central directory and the -/// members actually scanned are touched, and scanning stops at the first -/// detection — so a range-backed reader never fetches the rest. Non-ZIP -/// inputs within `deep_analysis_max` are buffered and analyzed; larger ones -/// fall back to the streaming pattern+hash core. +/// Scan an input: the one scan every entry point runs, whether the bytes come +/// from a file, a buffered stream (stdin, `INSTREAM`, ICAP) or an HTTP range +/// source. +/// +/// `size` is the input's full size. Past `--max-input-bytes` the reader need +/// only hold the first `max_scan_size` bytes: those get the whole scan, and +/// with no detection in them the input is reported over the limit. +/// +/// A ZIP, tar or gzip is walked member by member off the reader, so only the +/// parts actually scanned are read and a range-backed reader never fetches the +/// rest. Anything else within `deep_analysis_max` is buffered and analyzed; a +/// larger one gets the constant-memory literal and whole-file-hash pass. pub fn scan_seekable( db: &Scanner, mut reader: R, size: u64, opts: &ScanOptions, ) -> io::Result { + // Thread-local, so it carries over from whatever this thread scanned last + // unless every entry point clears it. + engine::reset_scan_truncated(); if let Some(max) = opts.max_scan_size { if size > max { - // Invariant 2 (see crate docs): scan the budgeted prefix before - // refusing by size, so a detection in it still wins. - reader.seek(SeekFrom::Start(0))?; - if let Some(hit) = stream_core(db, (&mut reader).take(max))? { - return Ok(ScanReport::infected_hit( - hit, - opts.unofficial_suffix, - Vec::new(), - )); + // Invariant 2 (see crate docs): scan what is within the limit + // before refusing by size, so a detection in it still wins. + let head = scan_within(db, Window::new(&mut reader, max)?, max, opts)?; + if head.verdict.category() == VerdictCategory::Infected { + return Ok(head); } return Ok(max_file_size_report(size, max, opts)); } } + scan_within(db, reader, size, opts) +} + +/// [`scan_seekable`] for an input within `--max-input-bytes`. +fn scan_within( + db: &Scanner, + mut reader: R, + size: u64, + opts: &ScanOptions, +) -> io::Result { let mut prefix = [0u8; 4096]; let n = fill_prefix(&mut reader, &mut prefix)?; reader.seek(SeekFrom::Start(0))?; let ft = profile::timed("filetype", n as u64, || filetype::identify(&prefix[..n])); // Natively-streaming containers (ZIP/tar/gzip) are walked member-by-member - // off the seekable source — the whole container is never buffered and no - // `deep_analysis_max` cap applies, so an arbitrarily large on-disk archive is - // scanned in bounded memory (one member at a time). + // off the reader: the whole container is never buffered and no + // `deep_analysis_max` cap applies to the walk, so an arbitrarily large + // archive is scanned in bounded memory (one member at a time). if streams_natively(ft) { - reader.seek(SeekFrom::Start(0))?; + // A container that fits under `deep_analysis_max` gets the full + // engine over its own bytes inside `scan_container_stream`. A larger + // one cannot be held, so it gets the constant-memory pass here, and + // the walk reports what it could not run. + if size > opts.deep_analysis_max { + if let Some(hit) = stream_core(db, &mut reader, opts.unofficial_suffix)? { + return Ok(core_hit_report(db, hit, opts)); + } + reader.seek(SeekFrom::Start(0))?; + } return Ok(scan_container_stream(db, reader, opts, ft)); } if size <= opts.deep_analysis_max { + // Bounded, so a source whose size under-reports its length cannot + // stream unbounded. let mut data = Vec::new(); (&mut reader) .take(opts.deep_analysis_max.saturating_add(1)) .read_to_end(&mut data)?; if data.len() as u64 > opts.deep_analysis_max { - return Ok(ScanReport::limits( - format!("input exceeds deep-analysis-max {}", opts.deep_analysis_max), - Vec::new(), - )); + return Ok(grew_past_deep_analysis_limit(opts)); } return Ok(analyze(db, &data, opts)); } - // Large non-ZIP: stream the pattern+hash core from the start. - if let Some(hit) = stream_core(db, &mut reader)? { - return Ok(suppress_name( - db, - ScanReport::infected_hit(hit, opts.unofficial_suffix, Vec::new()), - )); + // Too large to buffer. The constant-memory pass still runs, so a literal + // signature or a whole-file hash anywhere in the input is found. + if let Some(hit) = stream_core(db, &mut reader, opts.unofficial_suffix)? { + return Ok(core_hit_report(db, hit, opts)); } - // The same test `scan_path` applies, and for the same reason: a container - // hides detections that only structural parsing finds, and the cap meant it - // was never parsed. Checking `is_executable` alone would call an oversize - // RAR, PDF or OLE clean — the flat core saw its bytes, but nothing opened - // it. This path backs the daemon's stream verbs and the HTTP range source, - // so the two entry points have to agree. - if unpack_format(ft).is_some() || ft.is_executable() { - return Ok(ScanReport::limits( - format!( - "{} is {size} bytes (> deep-analysis-max {}); not fully parsed", - ft.as_str(), - opts.deep_analysis_max - ), - Vec::new(), - )); + Ok(over_deep_analysis_limit(ft, size, opts)) +} + +/// The report for a hit of the constant-memory pass. +fn core_hit_report(db: &Scanner, hit: CoreHit, opts: &ScanOptions) -> ScanReport { + suppress_name( + db, + ScanReport::infected_hit(hit, opts.unofficial_suffix, Vec::new()), + ) +} + +/// The first `len` bytes of a seekable source, as a seekable source of its own. +struct Window { + inner: R, + len: u64, + pos: u64, +} + +impl Window { + fn new(mut inner: R, len: u64) -> io::Result { + inner.seek(SeekFrom::Start(0))?; + Ok(Self { inner, len, pos: 0 }) + } +} + +impl Read for Window { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let room = self.len.saturating_sub(self.pos); + let want = (buf.len() as u64).min(room) as usize; + if want == 0 { + return Ok(0); + } + let n = self.inner.read(&mut buf[..want])?; + self.pos += n as u64; + Ok(n) + } +} + +impl Seek for Window { + fn seek(&mut self, to: SeekFrom) -> io::Result { + let target = match to { + SeekFrom::Start(o) => Some(o), + SeekFrom::End(d) => self.len.checked_add_signed(d), + SeekFrom::Current(d) => self.pos.checked_add_signed(d), + } + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "seek before the start"))?; + self.pos = self.inner.seek(SeekFrom::Start(target))?; + Ok(self.pos) } - Ok(ScanReport::clean(vec![Finding::new( - "type", - format!("{} (flat content, fully scanned)", ft.as_str()), - )])) } /// Build an extraction [`Budget`] carrying the password pool for decrypting @@ -1469,20 +1511,6 @@ fn scan_budget(db: &Scanner, opts: &ScanOptions) -> Budget { b } -/// Map an extraction [`unpack::LimitHit`] to the right top-level report: a -/// resource bound is `LimitsExceeded`; undecodable content is `Unscannable`. -fn report_for_hit(hit: unpack::LimitHit, findings: Vec, opts: &ScanOptions) -> ScanReport { - if hit.is_corrupt() { - return ScanReport::unscannable(hit.reason, findings); - } - if opts.alert_exceeds_max { - if let Some(name) = limits_alert_name(hit.kind) { - return ScanReport::infected(name.to_string(), 0, Method::Heuristic, findings); - } - } - ScanReport::limits(hit.reason, findings) -} - /// ClamAV's alert name for a budget stop. The kind is carried on the /// [`unpack::LimitHit`] as a type rather than parsed back out of the reason /// text, so rewording a message can never silently change the alert. @@ -1493,6 +1521,7 @@ fn limits_alert_name(kind: unpack::LimitKind) -> Option<&'static str> { K::MaxFileSize => Some("Heuristics.Limits.Exceeded.MaxFileSize"), K::MaxFiles => Some("Heuristics.Limits.Exceeded.MaxFiles"), K::MaxRecursion => Some("Heuristics.Limits.Exceeded.MaxRecursion"), + K::MaxScanTime => Some("Heuristics.Limits.Exceeded.MaxScanTime"), // Malformed input, not a budget stop, so no budget alert names it. K::Corrupt => None, // `LimitKind` is `#[non_exhaustive]`. A kind added later and not named @@ -1520,7 +1549,7 @@ fn max_file_size_report(size: u64, max: u64, opts: &ScanOptions) -> ScanReport { } } ScanReport::limits( - format!("file size {size} exceeds max-scan-size {max}; scanned first {max} bytes only"), + format!("file size {size} exceeds max-input-bytes {max}; scanned first {max} bytes only"), Vec::new(), ) } @@ -1544,7 +1573,9 @@ fn limits_outcome(opts: &ScanOptions, kind: unpack::LimitKind, reason: String) - DeepOutcome::Limits(reason) } -/// As [`report_for_hit`] but for the recursive [`DeepOutcome`] path. +/// Map an extraction [`unpack::LimitHit`] to an outcome: a resource bound is +/// `Limits` (or its alert under `--alert-exceeds-max`); undecodable content is +/// `Unscannable`. fn outcome_for_hit(hit: unpack::LimitHit, opts: &ScanOptions) -> DeepOutcome { if hit.is_corrupt() { DeepOutcome::Unscannable(hit.reason) @@ -1664,12 +1695,12 @@ fn member_stream_scan( buf.truncate(cap as usize); let scanned = member_content_scan(cx, tally, &buf, budget, findings, sink); // We scanned every byte the stream yielded. If it simply ran out of - // input (truncation — the missing tail is *absent*, not hidden), a - // clean result is a real Clean: exav scans for malware, it is not a - // file-integrity validator, so a damaged-but-payload-free file is not - // flagged. A mid-stream *corruption* (undecodable bytes still present) - // keeps the not-fully-scanned verdict on a clean salvage. - if scanned.is_none() && e.kind() != std::io::ErrorKind::UnexpectedEof { + // input (truncation: the missing tail is *absent*, not hidden) or + // decoded in full and failed its checksum, a clean result is a real + // Clean: exav scans for malware, it is not a file-integrity + // validator. A mid-stream *corruption* (undecodable bytes still + // present) keeps the not-fully-scanned verdict on a clean salvage. + if scanned.is_none() && unpack::decode_error_hides_content(&e) { tally.unscannable.get_or_insert_with(|| { format!( "member decode error (salvaged {} B, no match): {e}", @@ -1692,20 +1723,29 @@ fn member_stream_scan( // the constant-memory pattern+hash core (prefix + tail chained so a match // straddling the boundary is still caught). No full-member buffer is held. let rest = head.into_inner(); - match stream_core(cx.db, std::io::Cursor::new(&buf).chain(rest)) { + match stream_core( + cx.db, + std::io::Cursor::new(&buf).chain(rest), + cx.opts.unofficial_suffix, + ) { Ok(Some((sig, off, method, unofficial))) => sink.hit( report_name(&sig, unofficial, cx.opts.unofficial_suffix), off, method, ), Ok(None) => { - tally.unscannable.get_or_insert_with(|| { - format!( - "member exceeds deep-analysis-max {cap}; pattern-scanned, \ - structural analysis skipped" - ) - }); - None + let reason = format!( + "a member is over the {cap}-byte deep-analysis limit \ + (--max-object-bytes); only literal signatures and hashes were \ + matched in it" + ); + match limits_outcome(cx.opts, unpack::LimitKind::MaxFileSize, reason) { + DeepOutcome::Limits(r) => { + tally.limits.get_or_insert(r); + None + } + alert => Some(alert), + } } Err(ref e) if unpack::is_budget_overflow(e) => Some(DeepOutcome::Limits( "member exceeds per-member decompression budget".to_string(), @@ -1735,7 +1775,6 @@ fn scan_container_stream( ft: FileType, ) -> ScanReport { let mut findings = vec![Finding::new("type", ft.as_str())]; - let mut budget = scan_budget(db, opts); // Every type that reaches this walk came from `filetype_of_format`, so the // inverse exists — `scan_dispatch_covers_every_format` asserts the // round-trip over `Format::ALL`. Defaulting to some format anyway would @@ -1750,12 +1789,57 @@ fn scan_container_stream( }; let mut reader = reader; + // The container's own bytes get the full matching core (wildcard and + // logical signatures, type-scoped ones, YARA, bytecode, hashes), as they + // do when the same container is a member of another one + // (`member_content_scan`). Without it a top-level archive was matched by + // literal patterns at best, and a signature on the archive itself fired + // only once the archive was nested. Held up to `deep_analysis_max`, the + // bound every in-memory pass has; past it the walk reports what it skipped. + let whole = match read_bounded(&mut reader, opts.deep_analysis_max) { + Ok(w) => w, + Err(e) => return ScanReport::unscannable(format!("container unreadable: {e}"), findings), + }; + if whole.as_deref().is_some_and(|w| allowlisted(db, w)) { + return ScanReport::clean(findings); + } + let outcome = walk_container(db, reader, opts, fmt, ft, whole.as_deref(), &mut findings); + // The members first, so a hit in one is attributed to it rather than to the + // container that happens to hold its bytes stored. Anything short of a + // detection still gets this pass: a detection beats a limit. + let outcome = match outcome { + hit @ DeepOutcome::Infected { .. } => hit, + other => whole + .as_deref() + .and_then(|w| core_scan(db, w, None, None, opts, &mut Sink::First(&db.ignored))) + .unwrap_or(other), + }; + report_of_outcome(outcome, findings, opts) +} + +/// Walk a top-level container's members for [`scan_container_stream`], with the +/// container on the ancestry chain for exactly as long as its members are +/// being scanned. +fn walk_container( + db: &Scanner, + reader: R, + opts: &ScanOptions, + fmt: unpack::Format, + ft: FileType, + whole: Option<&[u8]>, + findings: &mut Vec, +) -> DeepOutcome { + let mut reader = reader; + let mut budget = scan_budget(db, opts); + // Container `CL_TYPE_*` for `Container:`-scoped signatures, so each member // inherits it. A ZIP is sub-typed as OOXML Word/Excel/PowerPoint by its part // names; read a bounded prefix (the OOXML part names sit in the leading local // file headers) and rewind, so streamed OOXML members are scanned with the // right container context (e.g. `Container:CL_TYPE_OOXML_WORD` sigs fire). - let container = if fmt == unpack::Format::Zip && db.engine.has_ooxml_container_sigs() { + let container = if let Some(w) = whole { + container_cltype(fmt, w) + } else if fmt == unpack::Format::Zip && db.engine.has_ooxml_container_sigs() { // Seek to the start FIRST: an earlier raw pass (`stream_core`) may have // consumed the reader to EOF, so reading without rewinding yields nothing. // Gated on the DB actually having `Container:CL_TYPE_OOXML_*` sigs: the @@ -1777,20 +1861,19 @@ fn scan_container_stream( // very nesting it describes. let _ag = engine::AncestryGuard::enter(container); if unpack::is_streamable(fmt) { - let mut findings = findings; - let outcome = scan_streamed_container( + return scan_streamed_container( db, reader, opts, fmt, ft, container, - &mut findings, + findings, &mut budget, 0, - &mut Sink::First, + whole, + &mut Sink::First(&db.ignored), ); - return report_of_outcome(outcome, findings, opts); } // Non-streamable Archive-walkable containers: seekable member walk. Members @@ -1809,13 +1892,11 @@ fn scan_container_stream( .and_then(|n| reader.seek(std::io::SeekFrom::Start(0)).map(|_| n)) { Ok(n) => n, - Err(e) => { - return ScanReport::unscannable(format!("container size unreadable: {e}"), findings) - } + Err(e) => return DeepOutcome::Unscannable(format!("container size unreadable: {e}")), }; let mut archive = match unpack::Archive::open(reader) { Ok(a) => a, - Err(h) => return report_for_hit(h, findings, opts), + Err(h) => return outcome_for_hit(h, opts), }; // The same context, tally and per-member checks the other walks use. A // container reached this way is a large one that was never buffered, which @@ -1833,46 +1914,55 @@ fn scan_container_stream( }; let mut tally = MemberTally::new(); let mut volumes = unpack::volume::Collector::new(budget.limits().max_buffer_bytes); - let sink = &mut Sink::First; + let sink = &mut Sink::First(&db.ignored); let mut terminal: Option = None; - loop { - let mut entry = match archive.extract_next(&mut budget) { - Ok(Some(e)) => e, - Ok(None) => break, - Err(h) => return report_for_hit(h, findings, opts), - }; - // Track this member on the location stack for the duration of its scan, - // so a detection inside it (or a nested member) reports the full path. - let _mpg = MatchPathGuard::enter(&entry.name); - let size_real = entry.data.len() as u64; - let facts = MemberFacts { - name: &entry.name, - comp_size: entry.comp_size, - size_real, - encrypted: entry.encrypted, - unsupported: entry.unsupported, - }; - if let Some(o) = member_metadata_scan(&cx, &mut tally, facts, sink) { - terminal = Some(o); - break; - } - if entry.unsupported.is_some() { - // Nothing decoded: the metadata above is all this member has. - continue; - } - // A part of a byte-split set is held and rejoined below, not scanned as - // the fragment it is. - let body = std::mem::take(&mut entry.data); - let data = match volumes.offer(&entry.name, body) { - unpack::volume::Offer::Held => continue, - unpack::volume::Offer::PassThrough { data, .. } => data, - }; - if let Some(o) = - member_content_scan(&cx, &mut tally, &data, &mut budget, &mut findings, sink) - { - terminal = Some(o); - break; + // Inclusive of the member scans, as in the other walks. + let walk = profile::timed("unpack", container_size, || { + loop { + let mut entry = match archive.extract_next(&mut budget) { + Ok(Some(e)) => e, + Ok(None) => break, + Err(h) => return Err(h), + }; + // Track this member on the location stack for the duration of its scan, + // so a detection inside it (or a nested member) reports the full path. + let _mpg = MatchPathGuard::enter(&entry.name); + let size_real = entry.data.len() as u64; + let facts = MemberFacts { + name: &entry.name, + comp_size: entry.comp_size, + size_real, + encrypted: entry.encrypted, + unsupported: entry.unsupported, + }; + if let Some(o) = member_metadata_scan(&cx, &mut tally, facts, sink) { + terminal = Some(o); + break; + } + // Nothing decoded: the metadata above is all this member has. A + // member that decoded part way is still scanned for what it has; + // its `unsupported` reason is already in the tally. + if entry.unsupported.is_some() && entry.data.is_empty() { + continue; + } + // A part of a byte-split set is held and rejoined below, not scanned as + // the fragment it is. + let body = std::mem::take(&mut entry.data); + let data = match volumes.offer(&entry.name, body) { + unpack::volume::Offer::Held => continue, + unpack::volume::Offer::PassThrough { data, .. } => data, + }; + if let Some(o) = + member_content_scan(&cx, &mut tally, &data, &mut budget, findings, sink) + { + terminal = Some(o); + break; + } } + Ok(()) + }); + if let Err(h) = walk { + return outcome_for_hit(h, opts); } // Reassembly happens only now: completeness is knowable only once no further // member can arrive. Every part held above is still scanned, rejoined or @@ -1889,12 +1979,12 @@ fn scan_container_stream( // Nothing else accounts for these bytes: the parts were charged as they // were extracted, but this is a buffer the collector made. if let Err(h) = budget.charge_scan(buf.len() as u64) { - return report_for_hit(h, findings, opts); + return outcome_for_hit(h, opts); } let _mpg = MatchPathGuard::enter(&name); - terminal = member_content_scan(&cx, &mut tally, &buf, &mut budget, &mut findings, sink); + terminal = member_content_scan(&cx, &mut tally, &buf, &mut budget, findings, sink); } - report_of_outcome(terminal.unwrap_or_else(|| tally.verdict()), findings, opts) + terminal.unwrap_or_else(|| tally.verdict()) } /// Drive the low-level reader API ([`unpack::stream_members`]): each member is a @@ -1912,6 +2002,9 @@ fn scan_streamed_container( findings: &mut Vec, budget: &mut Budget, depth: u32, + // The container's bytes when the caller already holds them, so they are + // not read a second time for the whole-object checks below. + prefetched: Option<&[u8]>, sink: &mut Sink, ) -> DeepOutcome { // Container size for `.cdb` `ContainerSize` matching (seek to end, rewind). @@ -1933,6 +2026,7 @@ fn scan_streamed_container( // Only the container-level checks below record here; everything a member // produces goes on the tally. let mut unscannable: Option = None; + let mut limits: Option = None; let mut terminal: Option = None; // The checks that need the container's own bytes rather than a member's. @@ -1949,15 +2043,21 @@ fn scan_streamed_container( // scanning less because of how a file arrived, without saying so, is the // silent clean this scanner exists to prevent. if container_size <= opts.deep_analysis_max { - let mut whole = Vec::new(); - let read = reader - .by_ref() - .take(opts.deep_analysis_max) - .read_to_end(&mut whole); - let _ = reader.seek(std::io::SeekFrom::Start(0)); + let mut owned = Vec::new(); + let read = match prefetched { + Some(bytes) => Ok(bytes), + None => { + let r = reader + .by_ref() + .take(opts.deep_analysis_max) + .read_to_end(&mut owned); + let _ = reader.seek(std::io::SeekFrom::Start(0)); + r.map(|_| &owned[..]) + } + }; match read { - Ok(_) => { - if let Some(o) = whole_buffer_heuristics(&whole, ft, opts, sink) { + Ok(whole) => { + if let Some(o) = whole_buffer_heuristics(whole, ft, opts, sink) { return o; } } @@ -1972,13 +2072,16 @@ fn scan_streamed_container( } } } else { - unscannable.get_or_insert_with(|| { - format!( - "container is {container_size} bytes, over the {}-byte \ - deep-analysis limit: whole-container heuristics were not run", - opts.deep_analysis_max - ) - }); + let reason = format!( + "container is {container_size} bytes, over the {}-byte deep-analysis \ + limit (--max-object-bytes): its members were scanned, but the full \ + signature engine and whole-container checks did not run on its own bytes", + opts.deep_analysis_max + ); + match limits_outcome(opts, unpack::LimitKind::MaxFileSize, reason) { + DeepOutcome::Limits(r) => limits = Some(r), + alert => return alert, + } } // The same context and running tally the buffered walk keeps, so the shared @@ -2138,6 +2241,9 @@ fn scan_streamed_container( if let Some(r) = unscannable { tally.unscannable = Some(r); } + if let Some(r) = limits { + tally.limits = Some(r); + } tally.verdict() } @@ -2283,7 +2389,7 @@ pub fn analyze(db: &Scanner, data: &[u8], opts: &ScanOptions) -> ScanReport { fn analyze_inner(db: &Scanner, data: &[u8], opts: &ScanOptions) -> ScanReport { if let Some((sig, off, method, unofficial)) = - scan_bytes_core(db, data, opts.filename.as_deref()) + scan_bytes_core(db, data, opts.filename.as_deref(), opts.unofficial_suffix) { return ScanReport::infected( report_name(&sig, unofficial, opts.unofficial_suffix), @@ -2303,7 +2409,7 @@ fn analyze_inner(db: &Scanner, data: &[u8], opts: &ScanOptions) -> ScanReport { None, true, &mut findings, - &mut Sink::First, + &mut Sink::First(&db.ignored), ) { DeepOutcome::Infected { signature, @@ -2322,10 +2428,9 @@ fn analyze_inner(db: &Scanner, data: &[u8], opts: &ScanOptions) -> ScanReport { // any wildcard verification was skipped because a per-buffer scan bound // ran out (the step pool or the repeated-anchor cap) — the search did // not fully complete, so we must not report `OK`. - DeepOutcome::Clean if engine::scan_was_truncated() => ScanReport::limits( - SEARCH_INCOMPLETE.into(), - findings, - ), + DeepOutcome::Clean if engine::scan_was_truncated() => { + ScanReport::limits(SEARCH_INCOMPLETE.into(), findings) + } DeepOutcome::Clean => ScanReport::clean(findings), } } @@ -2351,10 +2456,9 @@ fn report_of_outcome( // A would-be `Clean` is downgraded when wildcard verification ran out of // its per-buffer scan bounds: the search did not complete, so `OK` would // be a claim the scan cannot support. - DeepOutcome::Clean if engine::scan_was_truncated() => ScanReport::limits( - SEARCH_INCOMPLETE.into(), - findings, - ), + DeepOutcome::Clean if engine::scan_was_truncated() => { + ScanReport::limits(SEARCH_INCOMPLETE.into(), findings) + } DeepOutcome::Clean => ScanReport::clean(findings), } } @@ -2363,12 +2467,7 @@ fn report_of_outcome( /// name is on the ignore list (`.ign`/`.ign2`) or `data`'s whole-file hash is /// allowlisted (`.fp`/`.sfp`). fn is_suppressed(db: &Scanner, data: &[u8], signature: &str) -> bool { - db.ignored.contains(signature) - || (!db.allow.is_empty() - && db - .allow - .lookup(&digests_of(data), data.len() as u64) - .is_some()) + db.ignored.contains(signature) || allowlisted(db, data) } /// Clear a detection if it is suppressed (see [`is_suppressed`]). @@ -2423,9 +2522,9 @@ pub fn analyze_all_with_outcome( engine::reset_scan_truncated(); let (names, outcome) = analyze_all_raw(db, data, opts); let outcome = match outcome { - Some(DeepOutcome::Limits(r)) => AllMatchOutcome::LimitsExceeded(r), - Some(DeepOutcome::Unscannable(r)) => AllMatchOutcome::Unscannable(r), - Some(DeepOutcome::PasswordProtected(r)) => AllMatchOutcome::PasswordProtected(r), + Some(DeepOutcome::Limits(r)) => AllMatchOutcome::LimitsExceeded(one_line(r)), + Some(DeepOutcome::Unscannable(r)) => AllMatchOutcome::Unscannable(one_line(r)), + Some(DeepOutcome::PasswordProtected(r)) => AllMatchOutcome::PasswordProtected(one_line(r)), // The same cardinal rule the single-verdict paths apply, which this one // was missing: a search that did not finish must not be presented as a // complete one. There it downgrades a would-be `Clean`; here there is no @@ -2444,12 +2543,7 @@ fn analyze_all_raw( data: &[u8], opts: &ScanOptions, ) -> (Vec<(String, Method)>, Option) { - if !db.allow.is_empty() - && db - .allow - .lookup(&digests_of(data), data.len() as u64) - .is_some() - { + if allowlisted(db, data) { return (Vec::new(), None); } let mut out = Vec::new(); @@ -2650,8 +2744,11 @@ fn whole_buffer_heuristics( /// and not the other changes *which signatures and heuristics run*, silently, /// and the symptom is a clean verdict rather than an error. enum Sink<'a> { - /// Stop at the first detection — a normal scan. - First, + /// Stop at the first detection that is not on the `.ign`/`.ign2` list: a + /// normal scan. An ignored name must not end the walk, as that would leave the + /// rest of the object unscanned on the strength of a match the operator + /// asked to have ignored. + First(&'a std::collections::HashSet), /// Collect every distinct detection and keep walking — `--all-matches`. All { out: &'a mut Vec<(String, Method)>, @@ -2667,14 +2764,20 @@ impl Sink<'_> { /// Record a detection. `Some(outcome)` means this walk must stop and return /// it; `None` means carry on looking. fn hit(&mut self, signature: String, offset: u64, method: Method) -> Option { - match_loc_record(); match self { - Sink::First => Some(DeepOutcome::Infected { - signature, - offset, - method, - }), + Sink::First(ignored) => { + if ignored.contains(&signature) { + return None; + } + match_loc_record(); + Some(DeepOutcome::Infected { + signature, + offset, + method, + }) + } Sink::All { out, seen, ignored } => { + match_loc_record(); if !ignored.contains(&signature) && seen.insert(signature.clone()) { out.push((signature, method)); } @@ -2756,6 +2859,9 @@ struct MemberTally { pos: u64, unscannable: Option, password: Option, + /// A size limit cut the analysis of something short without stopping the + /// walk: the rest of the members were still scanned. + limits: Option, } impl MemberTally { @@ -2764,16 +2870,18 @@ impl MemberTally { pos: 1, unscannable: None, password: None, + limits: None, } } fn verdict(self) -> DeepOutcome { // Precedence among incomplete outcomes: PasswordProtected (actionable) - // over Unscannable over Clean. - match (self.password, self.unscannable) { - (Some(r), _) => DeepOutcome::PasswordProtected(r), - (None, Some(r)) => DeepOutcome::Unscannable(r), - (None, None) => DeepOutcome::Clean, + // over Limits over Unscannable over Clean. + match (self.password, self.limits, self.unscannable) { + (Some(r), _, _) => DeepOutcome::PasswordProtected(r), + (None, Some(r), _) => DeepOutcome::Limits(r), + (None, None, Some(r)) => DeepOutcome::Unscannable(r), + (None, None, None) => DeepOutcome::Clean, } } } @@ -2913,13 +3021,7 @@ fn member_content_scan( // inside it is a detection. Checked once, here, so an allowlist entry means // the same thing for a member of any container — the allowlist is keyed on // the member's bytes, which do not depend on how the member was produced. - if !cx.db.allow.is_empty() - && cx - .db - .allow - .lookup(&digests_of(data), data.len() as u64) - .is_some() - { + if allowlisted(cx.db, data) { return None; } // Textual content extracted from an OLE2 document is scanned in OLE @@ -3158,6 +3260,7 @@ fn deep_analyze( findings, budget, depth, + Some(data), sink, ); } @@ -3233,7 +3336,8 @@ fn deep_analyze( return Some(o); } // Nothing decoded: the metadata above is all this member has. - if e.unsupported.is_some() { + // A member that decoded part way is still scanned. + if e.unsupported.is_some() && e.data.is_empty() { return None; } // A part of a byte-split set is held, not scanned — its bytes @@ -3352,9 +3456,9 @@ fn deep_analyze( // handled by extraction, not by re-carving overlapping regions. for off in pe::embedded_archive_offsets(data) { let sub = &data[off..]; - if unpack::detect(sub).is_none() { + let Some(fmt) = unpack::detect(sub) else { continue; - } + }; if let Err(h) = budget.charge_scan(sub.len() as u64) { return limits_outcome(opts, h.kind, h.reason); } @@ -3374,6 +3478,19 @@ fn deep_analyze( // decompression bomb in a real appended archive) still matters. inf @ DeepOutcome::Infected { .. } => return inf, lim @ DeepOutcome::Limits(_) => return lim, + // An encrypted archive appended to a picture or document (a + // polyglot: archive tools open it by its directory at the end) + // is reported once its structure proves it is really there. + // Executables are left to the SFX detector. 7z and RAR magics + // are long enough that a parse reaching an encrypted member + // is proof already; ZIP's four bytes are not. + pw @ DeepOutcome::PasswordProtected(_) + if !ft.is_executable() + && (fmt != unpack::Format::Zip + || unpack::zip_directory_is_consistent(sub)) => + { + deferred = pw; + } // Clean, or a carve that could not be decoded (Unscannable / // PasswordProtected): the candidate was found by a short magic that // collides with ordinary binary data (a false `1f8b08` / `PK\x03\x04` @@ -3519,9 +3636,6 @@ fn deep_analyze( deferred } -/// The `Heuristics.Encrypted.*` name for an encrypted member of a container of -/// format `fmt` (ClamAV's `--alert-encrypted` naming), format-specific where -/// ClamAV distinguishes it, else the generic `.Archive`. /// `Heuristics.Packed.` for a member the PE-packer extractor surfaced as /// an image it could not unpack, or `None` for anything else. /// @@ -3537,6 +3651,9 @@ fn packed_heuristic_name(member: &str) -> Option { Some(format!("Heuristics.Packed.{packer}")) } +/// The `Heuristics.Encrypted.*` name for an encrypted member of a container of +/// format `fmt` (ClamAV's `--alert-encrypted` naming), format-specific where +/// ClamAV distinguishes it, else the generic `.Archive`. fn encrypted_heuristic_name(fmt: unpack::Format) -> &'static str { use unpack::Format; match fmt { @@ -3634,12 +3751,36 @@ fn phishing_scan( /// `.UNOFFICIAL` in compat mode). The name is ALWAYS clean here. type CoreHit = (String, u64, Method, bool); -fn scan_bytes_core(db: &Scanner, data: &[u8], filename: Option<&str>) -> Option { +fn scan_bytes_core( + db: &Scanner, + data: &[u8], + filename: Option<&str>, + suffix: bool, +) -> Option { // Total bytes of bytecode-extracted (unpacked) content this scan may // re-scan, across the whole recursion — bounds an extraction bomb where a // (trusted) unpacker, driven by a hostile input, emits many/large buffers. let mut extract_budget = MAX_BC_EXTRACT_TOTAL; - scan_bytes_depth(db, data, 0, &mut extract_budget, None, None, filename) + scan_bytes_depth( + db, + data, + 0, + &mut extract_budget, + None, + None, + filename, + suffix, + ) +} + +/// Whether a detection is on the `.ign`/`.ign2` list, under the name it would +/// be reported as (`suffix` is [`ScanOptions::unofficial_suffix`]). +/// +/// Checked where each matcher picks its hit rather than on the finished +/// report: a first-match scan that stopped on an ignored signature would never +/// look for a real one further on, and the report would then be cleared. +fn is_ignored(db: &Scanner, name: &str, unofficial: bool, suffix: bool) -> bool { + !db.ignored.is_empty() && db.ignored.contains(&report_name(name, unofficial, suffix)) } /// Whether a file type is text-ish (not a positively-typed binary/container). @@ -3737,6 +3878,7 @@ fn scan_bytes_member( data: &[u8], ft_override: Option, container: Option, + suffix: bool, ) -> Option { let mut extract_budget = MAX_BC_EXTRACT_TOTAL; // A container member has its own identity (not the outer file's), which exav @@ -3749,6 +3891,7 @@ fn scan_bytes_member( ft_override, container, None, + suffix, ) } @@ -3845,12 +3988,14 @@ fn core_scan_all( // `None` for the filename matches `scan_bytes_member`: a member has its own // identity rather than the outer file's, and exav does not track it, so // filename externals stay undefined on this path in both cores. - // - // The matcher already formats the full name (`YARA.` prefix and any - // `.UNOFFICIAL` suffix), so it is reported verbatim. if !db.yara.is_empty() { - if let Some(name) = profile::timed("yara", data.len() as u64, || db.yara.scan(data, None)) { - sink.hit(name, 0, Method::Yara); + let unofficial = db.yara.unofficial(); + for name in profile::timed("yara", data.len() as u64, || db.yara.scan(data, None)) { + sink.hit( + report_name(&name, unofficial, unofficial_suffix), + 0, + Method::Yara, + ); } } // Bytecode names are reported verbatim, never `.UNOFFICIAL`-suffixed. @@ -3890,7 +4035,8 @@ fn core_scan( } return None; } - let (sig, off, m, unofficial) = scan_bytes_member(db, data, ft_override, container)?; + let (sig, off, m, unofficial) = + scan_bytes_member(db, data, ft_override, container, opts.unofficial_suffix)?; sink.hit( report_name(&sig, unofficial, opts.unofficial_suffix), off, @@ -3904,6 +4050,7 @@ const MAX_BC_DEPTH: u32 = 4; /// Cap on total bytes re-scanned from bytecode-extracted buffers per scan. const MAX_BC_EXTRACT_TOTAL: u64 = 256 * 1024 * 1024; +#[allow(clippy::too_many_arguments)] fn scan_bytes_depth( db: &Scanner, data: &[u8], @@ -3912,7 +4059,9 @@ fn scan_bytes_depth( ft_override: Option, container: Option, filename: Option<&str>, + suffix: bool, ) -> Option { + let skip = |name: &str, unofficial: bool| is_ignored(db, name, unofficial, suffix); let ft = if ft_override.is_some() { ft_override } else if !db.engine.is_empty() || !db.sections.is_empty() || !db.bytecode.is_empty() { @@ -3939,7 +4088,7 @@ fn scan_bytes_depth( let icon_ctx = engine::IconCtx::new(&db.icons, &icon_metrics); if let Some((name, off, unofficial)) = profile::timed("engine", data.len() as u64, || { db.engine - .scan_with_icons(data, ft, layout.as_ref(), container, Some(&icon_ctx)) + .scan_first(data, ft, layout.as_ref(), container, Some(&icon_ctx), &skip) }) { return Some((name, off, Method::Pattern, unofficial)); } @@ -3963,7 +4112,8 @@ fn scan_bytes_depth( let norm = profile::timed("normalize", data.len() as u64, norm); if let Some((name, off, unofficial)) = profile::timed("engine", norm.len() as u64, || { - db.engine.scan_with_layout(&norm, ft, None, container) + db.engine + .scan_first(&norm, ft, None, container, None, &skip) }) { return Some((name, off, Method::Pattern, unofficial)); @@ -3975,7 +4125,7 @@ fn scan_bytes_depth( let (det, extracted) = profile::timed("bytecode", data.len() as u64, || { db.bytecode.scan(data, ft, layout.as_ref()) }); - if let Some((name, _idx)) = det { + if let Some((name, _idx)) = det.filter(|(name, _)| !skip(name, false)) { return Some((name, 0, Method::Bytecode, false)); } // Unpacker programs surface embedded files; re-scan them recursively @@ -3996,17 +4146,23 @@ fn scan_bytes_depth( // exactly like a target-gating bug and is not one. The unpacker // has no member name to offer, so the index is the identity. let _mpg = MatchPathGuard::enter(&format!("bytecode-unpacked-{}", bcix + 1)); - if let Some(hit) = - scan_bytes_depth(db, &buf, depth + 1, extract_budget, None, container, None) - { + if let Some(hit) = scan_bytes_depth( + db, + &buf, + depth + 1, + extract_budget, + None, + container, + None, + suffix, + ) { match_loc_record(); return Some(hit); } // A packer may surface an embedded archive (e.g. an unpacked // payload that is itself a ZIP/gzip). Unpack it too, bounded by // the same extraction budget, so the real payload is reached. - let bft = - profile::timed("filetype", buf.len() as u64, || filetype::identify(&buf)); + let bft = profile::timed("filetype", buf.len() as u64, || filetype::identify(&buf)); // This deep bytecode-extracted-buffer rescan runs only behind a // ClamAV `.cbc` unpacker firing; it isn't part of the common // diff-tested path, so it always runs at full capability. @@ -4038,6 +4194,7 @@ fn scan_bytes_depth( None, inner_container, None, + suffix, ); if hit.is_some() { match_loc_record(); @@ -4057,7 +4214,7 @@ fn scan_bytes_depth( if let Some((name, unofficial)) = profile::timed("sections", data.len() as u64, || { for (size, slice) in pe::section_slices(data) { let d = hashes::section_digests(slice, want_sha); - if let Some(hit) = db.sections.lookup(size, &d) { + if let Some(hit) = db.sections.lookup(size, &d).filter(|(n, u)| !skip(n, *u)) { return Some(hit); } } @@ -4069,17 +4226,20 @@ fn scan_bytes_depth( if !db.hashes.is_empty() && hash_matchable(data.len() as u64) { if let Some((name, unofficial)) = profile::timed("hashes", data.len() as u64, || { db.hashes.lookup(&digests_of(data), data.len() as u64) - }) { + }) + .filter(|(n, u)| !skip(n, *u)) + { return Some((name, 0, Method::Hash, unofficial)); } } - // YARA rules: the matcher already formats the full ClamAV name - // (`YARA.` prefix and any `.UNOFFICIAL` suffix), so it is reported verbatim. if !db.yara.is_empty() { + let unofficial = db.yara.unofficial(); if let Some(name) = profile::timed("yara", data.len() as u64, || db.yara.scan(data, filename)) + .into_iter() + .find(|n| !skip(n, unofficial)) { - return Some((name, 0, Method::Yara, false)); + return Some((name, 0, Method::Yara, unofficial)); } } None @@ -4087,13 +4247,19 @@ fn scan_bytes_depth( /// Streaming detection core (constant memory, any size): Aho-Corasick + /// triple hasher in one forward pass. -fn stream_core(db: &Scanner, reader: R) -> io::Result> { - // When no hash signatures are loaded, skip the (expensive) triple hash on - // every byte and just run the pattern matcher over the raw reader. - if db.hashes.is_empty() { - if let Some(mat) = db.patterns.ac().stream_find_iter(reader).next() { +/// +/// `suffix` is [`ScanOptions::unofficial_suffix`], which decides the name an +/// ignore-list entry has to match. +fn stream_core(db: &Scanner, reader: R, suffix: bool) -> io::Result> { + // The first match that is not on the ignore list. Running the iterator to + // its end consumes the whole stream, so the hashers below saw every byte. + let first_pattern = |reader: &mut dyn Read| -> io::Result> { + for mat in db.patterns.ac().stream_find_iter(reader) { let mat = mat?; let (name, unofficial) = db.patterns.name_prov(mat.pattern().as_usize()); + if is_ignored(db, name, unofficial, suffix) { + continue; + } return Ok(Some(( name.to_string(), mat.start() as u64, @@ -4101,25 +4267,26 @@ fn stream_core(db: &Scanner, reader: R) -> io::Result> unofficial, ))); } - return Ok(None); + Ok(None) + }; + // When no hash signatures are loaded, skip the (expensive) triple hash on + // every byte and just run the pattern matcher over the raw reader. + if db.hashes.is_empty() { + let mut reader = reader; + return first_pattern(&mut reader); } let mut tee = TeeHasher::new(reader); - // `next()` drives the reader until the first match or EOF; on `None` - // the whole stream has been consumed, so the hashers saw every byte. - if let Some(mat) = db.patterns.ac().stream_find_iter(&mut tee).next() { - let mat = mat?; - let (name, unofficial) = db.patterns.name_prov(mat.pattern().as_usize()); - return Ok(Some(( - name.to_string(), - mat.start() as u64, - Method::Pattern, - unofficial, - ))); + if let Some(hit) = first_pattern(&mut tee)? { + return Ok(Some(hit)); } let size = tee.bytes_read(); let digests = tee.finalize(); if hash_matchable(size) { - if let Some((name, unofficial)) = db.hashes.lookup(&digests, size) { + if let Some((name, unofficial)) = db + .hashes + .lookup(&digests, size) + .filter(|(n, u)| !is_ignored(db, n, *u, suffix)) + { return Ok(Some((name, 0, Method::Hash, unofficial))); } } @@ -4287,6 +4454,283 @@ mod tests { w.finish().unwrap().into_inner() } + /// As [`build_zip`], with every member deflated. + #[cfg(any(feature = "zip", feature = "all-formats"))] + fn build_zip_deflated(members: &[(&str, &[u8])]) -> Vec { + use zip::write::SimpleFileOptions; + let opts = + SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (name, body) in members { + w.start_file(*name, opts).unwrap(); + std::io::Write::write_all(&mut w, body).unwrap(); + } + w.finish().unwrap().into_inner() + } + + /// A database built from loose signature files, `(file name, contents)`. + fn db_from(files: &[(&str, &str)]) -> Scanner { + let mut b = loader::Builder::new(); + for (name, text) in files { + b.add_named_bytes(name, text.as_bytes(), false); + } + b.build().unwrap() + } + + fn seekable(db: &Scanner, data: &[u8], opts: &ScanOptions) -> Verdict { + scan_seekable(db, Cursor::new(data), data.len() as u64, opts) + .unwrap() + .verdict + } + + fn infected_as(v: &Verdict) -> Option<&str> { + match v { + Verdict::Infected { signature, .. } => Some(signature), + _ => None, + } + } + + /// A YARA rule is named like any other signature: `.UNOFFICIAL` only when + /// the scan asks for ClamAV's naming, and an ignored rule does not hide the + /// next one that matches. + #[test] + #[cfg(feature = "yara")] + fn yara_names_follow_the_scan_options() { + let rules = "rule first { strings: $a = \"yaramarker\" condition: $a }\n\ + rule second { strings: $a = \"yaramarker\" condition: $a }\n"; + let db = db_from(&[("r.yar", rules)]); + let data = b"xx yaramarker xx"; + let compat = ScanOptions { + unofficial_suffix: true, + ..ScanOptions::default() + }; + let opts = ScanOptions::default(); + assert_eq!( + infected_as(&analyze(&db, data, &opts).verdict), + Some("YARA.first") + ); + assert_eq!( + seekable(&db, data, &opts), + analyze(&db, data, &opts).verdict + ); + assert_eq!( + infected_as(&analyze(&db, data, &compat).verdict), + Some("YARA.first.UNOFFICIAL") + ); + let all: Vec = analyze_all(&db, data, &opts) + .into_iter() + .map(|h| h.0) + .collect(); + assert!(all.contains(&"YARA.first".to_string()), "{all:?}"); + assert!(all.contains(&"YARA.second".to_string()), "{all:?}"); + + let db = db_from(&[("r.yar", rules), ("r.ign2", "YARA.first\n")]); + assert_eq!( + infected_as(&analyze(&db, data, &opts).verdict), + Some("YARA.second") + ); + let all: Vec = analyze_all(&db, data, &opts) + .into_iter() + .map(|h| h.0) + .collect(); + assert_eq!(all, ["YARA.second"]); + } + + /// A YARA condition that runs out of evaluation steps did not decide + /// anything, so the scan is incomplete rather than clean. + #[test] + #[cfg(feature = "yara")] + fn a_yara_rule_out_of_steps_is_not_a_clean_scan() { + let rules = "rule endless { condition: for all i in (0..100000000) : ( i >= 0 ) }\n"; + let db = db_from(&[("r.yar", rules)]); + let opts = ScanOptions::default(); + let data = b"just some text"; + for v in [ + analyze(&db, data, &opts).verdict, + seekable(&db, data, &opts), + ] { + assert!(matches!(v, Verdict::LimitsExceeded { .. }), "{v:?}"); + } + } + + /// A top-level archive gets the full matching core over its own bytes, as + /// it does when nested in another archive. The wildcard signature matches + /// only a member NAME, i.e. the raw container; the hash is the container's. + #[test] + fn a_top_level_container_gets_the_full_engine_over_its_own_bytes() { + let zip = build_zip(&[("payload_marker.txt", b"nothing to see here")]); + let wild = db_from(&[("w.ndb", "Test.ZipName:0:*:7061796c6f6164??6d61726b6572\n")]); + let d = digests_of(&zip); + let hash = db_from(&[("h.hdb", &format!("{}:{}:Test.ZipHash\n", d.md5, zip.len()))]); + let opts = ScanOptions::default(); + + assert_eq!( + infected_as(&seekable(&wild, &zip, &opts)), + Some("Test.ZipName") + ); + assert_eq!( + infected_as(&seekable(&hash, &zip, &opts)), + Some("Test.ZipHash") + ); + + let dir = crate::tmpfile::TempDir::new().unwrap(); + let path = dir.path().join("t.zip"); + std::fs::write(&path, &zip).unwrap(); + let on_disk = scan_path(&wild, &path, &opts).unwrap().verdict; + assert_eq!(infected_as(&on_disk), Some("Test.ZipName")); + } + + /// An ignored signature is passed over, not stopped at: a real detection in + /// a later member must still be found, through the buffered and the + /// streamed walk alike. + #[test] + #[cfg(any(feature = "zip", feature = "all-formats"))] + fn an_ignored_signature_does_not_hide_a_later_member() { + let db = db_from(&[ + ("a.ndb", "Test.Noisy:0:*:6e6f697379626974\n"), + ("a.ign2", "Test.Noisy\n"), + ]); + // Padding makes the deflater emit Huffman-coded blocks rather than + // stored ones, which would leave the member bytes readable raw. + let pad = |core: &[u8]| [&[b'.'; 4096][..], core, &[b'.'; 4096][..]].concat(); + let noisy = pad(b"xx noisybit xx"); + let payload = pad(eicar()); + let zip = build_zip_deflated(&[("a.txt", &noisy), ("b.txt", &payload)]); + // Only the member walk can find them, not the pass over the raw bytes. + assert!(!contains_window(&zip, b"noisybit") && !contains_window(&zip, eicar())); + let opts = ScanOptions::default(); + for v in [ + analyze(&db, &zip, &opts).verdict, + seekable(&db, &zip, &opts), + ] { + assert_eq!(infected_as(&v), Some("Eicar-Test-Signature"), "{v:?}"); + } + // And on its own the ignored match is no detection at all. + let quiet = build_zip_deflated(&[("a.txt", &noisy)]); + assert_eq!(seekable(&db, &quiet, &opts), Verdict::Clean); + + // Both in one buffer: the matcher itself has to pass over the ignored + // hit, since a first-match scan would otherwise stop there. + let mut both = b"noisybit ".to_vec(); + both.extend_from_slice(eicar()); + assert_eq!( + infected_as(&analyze(&db, &both, &opts).verdict), + Some("Eicar-Test-Signature") + ); + + // The same through a detection that is not a byte match: a `.cdb` + // member-name signature on the first member. + let db = db_from(&[ + ("c.cdb", "Test.CdbName:CL_TYPE_ZIP:*:a\\.txt:*:*:*:*:*:\n"), + ("c.ign2", "Test.CdbName\n"), + ]); + for v in [ + analyze(&db, &zip, &opts).verdict, + seekable(&db, &zip, &opts), + ] { + assert_eq!(infected_as(&v), Some("Eicar-Test-Signature"), "{v:?}"); + } + } + + /// A file too large for the full engine was matched by literal patterns and + /// hashes alone. That is not a complete scan, so it is not clean, while a + /// hit the literal pass does find still wins. + #[test] + fn a_file_over_the_deep_analysis_limit_is_not_called_clean() { + let db = db_from(&[ + ("w.ndb", "Test.Wild:0:*:6576696c??7061796c6f6164\n"), + ("l.ndb", "Test.Literal:0:*:6d61726b65726d61726b6572\n"), + ]); + let opts = ScanOptions { + deep_analysis_max: 64, + ..ScanOptions::default() + }; + let mut wild = b"xx evil1payload xx".to_vec(); + wild.resize(1024, b' '); + let mut literal = b"xx markermarker xx".to_vec(); + literal.resize(1024, b' '); + + let dir = crate::tmpfile::TempDir::new().unwrap(); + let path = dir.path().join("big.txt"); + std::fs::write(&path, &wild).unwrap(); + for v in [ + seekable(&db, &wild, &opts), + scan_path(&db, &path, &opts).unwrap().verdict, + ] { + assert!( + matches!(v, Verdict::LimitsExceeded { .. }), + "a literal-only scan is not a full one: {v:?}" + ); + } + std::fs::write(&path, &literal).unwrap(); + for v in [ + seekable(&db, &literal, &opts), + scan_path(&db, &path, &opts).unwrap().verdict, + ] { + assert_eq!(infected_as(&v), Some("Test.Literal")); + } + } + + /// A size-limit reason names the flag an operator would raise. + #[test] + fn the_size_limit_reason_names_the_real_flag() { + let opts = ScanOptions { + max_scan_size: Some(4), + ..ScanOptions::default() + }; + match seekable(&Scanner::builtin(), b"more than four bytes", &opts) { + Verdict::LimitsExceeded { reason } => { + assert!(reason.contains("max-input-bytes"), "{reason}") + } + other => panic!("expected a limit, got {other:?}"), + } + } + + /// The `.fp` allowlist covers a top-level archive, not only its members. + #[test] + fn an_allowlisted_top_level_archive_is_clean() { + let zip = build_zip(&[("b.txt", eicar())]); + let d = digests_of(&zip); + let db = db_from(&[("x.fp", &format!("{}:{}:Allowed\n", d.md5, zip.len()))]); + assert_eq!(seekable(&db, &zip, &ScanOptions::default()), Verdict::Clean); + } + + /// The incomplete-search flag is per scan. Left over from an earlier scan + /// on the same thread, it turned a clean archive into LIMITS-EXCEEDED. + #[test] + #[cfg(any(feature = "zip", feature = "all-formats"))] + fn a_previous_scans_truncation_does_not_leak_into_the_next() { + let zip = build_zip(&[("a.txt", b"harmless")]); + engine::mark_scan_truncated(); + assert_eq!( + seekable(&Scanner::builtin(), &zip, &ScanOptions::default()), + Verdict::Clean + ); + } + + /// A member name is file content. Quoted in a reason, a newline in it would + /// write a line of its own into a line-framed reply. + #[test] + fn a_member_name_cannot_add_a_line_to_the_verdict() { + let mut t = tar::Builder::new(Vec::new()); + let body = vec![b'A'; 5000]; + let mut h = tar::Header::new_gnu(); + h.set_size(body.len() as u64); + h.set_mode(0o644); + t.append_data(&mut h, "x\nstream: OK\n", &body[..]).unwrap(); + let tarball = t.into_inner().unwrap(); + + let mut opts = ScanOptions::default(); + opts.limits.max_scanned_bytes = 1024; + match seekable(&Scanner::builtin(), &tarball, &opts) { + Verdict::LimitsExceeded { reason } => { + assert!(reason.contains("x_stream: OK_"), "{reason}"); + assert!(!reason.contains('\n'), "{reason:?}"); + } + other => panic!("expected a limit, got {other:?}"), + } + } + #[test] fn detects_eicar_pattern() { let db = Scanner::builtin(); @@ -4820,8 +5264,8 @@ mod tests { ); } - // Flat content larger than the deep-analysis cap is still fully scanned - // by the streaming core, so signatures are found. + // Flat content larger than the deep-analysis cap still goes through the + // streaming core, so a literal signature is found. #[test] #[cfg_attr( target_family = "wasm", @@ -4845,7 +5289,7 @@ mod tests { target_family = "wasm", ignore = "host filesystem/tempdir unavailable under WASI" )] - fn oversize_flat_text_clean_is_clean() { + fn oversize_flat_text_is_a_limit_not_clean() { let f = write_temp(&vec![b'Z'; 5000]); let db = Scanner::builtin(); let opts = ScanOptions { @@ -4853,7 +5297,7 @@ mod tests { ..Default::default() }; let r = scan_path(&db, f.path(), &opts).unwrap(); - assert_eq!(r.verdict, Verdict::Clean); + assert!(matches!(r.verdict, Verdict::LimitsExceeded { .. })); } fn zip_bytes(members: &[(&str, &[u8], bool)]) -> Vec { diff --git a/crates/exav-core/src/loader.rs b/crates/exav-core/src/loader.rs index 3ce58026..1ccf4f6b 100644 --- a/crates/exav-core/src/loader.rs +++ b/crates/exav-core/src/loader.rs @@ -124,7 +124,7 @@ pub struct Builder { /// /// Provenance — whether a signature came from an unofficial (non-`.cvd`) /// database — is now recorded *per signature*, always, regardless of this - /// flag. The `.UNOFFICIAL` name suffix / `YARA.` prefix is applied at REPORT + /// flag. The `.UNOFFICIAL` name suffix is applied at REPORT /// time (gated on `ScanOptions::unofficial_suffix`), so ONE database serves both compat and /// non-compat scans. Official `.cvd`/`.cld` sigs carry `unofficial=false` and /// are never suffixed. See `crate::report_name`. @@ -139,7 +139,7 @@ pub struct Builder { /// members. User-supplied (the official CVDs ship no `.pwdb`). passwords: Vec, /// Optional per-shard memory budget for the signature-automaton build - /// (`--max-build-memory`). `None` = build each partition as a single + /// (`--build-shard-bytes`). `None` = build each partition as a single /// automaton (fastest). A budget shards large partitions so a huge set /// (`main+daily`) can build on a small host. Affects `--build-db` builds. max_build_mem: Option, @@ -165,7 +165,7 @@ impl Builder { self.engine.set_detect_pua(on); } - /// Set the per-shard automaton-build memory budget (`--max-build-memory`). + /// Set the per-shard automaton-build memory budget (`--build-shard-bytes`). /// `None` (default) builds each partition as one automaton. pub fn set_max_build_mem(&mut self, bytes: Option) { self.max_build_mem = bytes; @@ -276,7 +276,7 @@ impl Builder { let ext = name.rsplit('.').next().unwrap_or("").to_ascii_lowercase(); // Provenance: a signature from any database that is NOT a signed // `.cvd`/`.cld` container is "unofficial" — ClamAV suffixes such - // detections with `.UNOFFICIAL` (and YARA rules with a `YARA.` prefix). + // detections with `.UNOFFICIAL`. // We record this bit per signature, ALWAYS (independent of any compat // flag), and store the CLEAN name. The suffix is applied at report time // (see `crate::report_name`), gated on `ScanOptions::unofficial_suffix`, so a single @@ -294,12 +294,28 @@ impl Builder { "ndb" | "ndu" => { // Literal subset for streaming; full text for the engine. let (mut pats, _unsup) = PatternSet::parse_ndb_prov(text, unofficial); + // The engine drops `PUA.*` unless PUA detection is on; the + // streaming set has to agree or a large file reports a PUA + // the same content in a small file would not. + if !self.detect_pua { + pats.retain(|p| !p.name.starts_with("PUA.")); + } self.patterns.append(&mut pats); self.engine.add_ndb(text, unofficial); } + // Literal `Name=HEX` signatures. The engine is what in-memory scans + // match with, so they go there too. In the streaming set alone they + // were only ever seen by a file too large to buffer. "db" => { - if let Ok(mut pats) = PatternSet::parse_simple(text) { - self.patterns.append(&mut pats); + if let Ok(pats) = PatternSet::parse_simple(text) { + for mut p in pats { + if p.name.starts_with("PUA.") && !self.detect_pua { + continue; + } + p.unofficial = unofficial; + self.engine.add_literal_prov(&p.name, &p.bytes, unofficial); + self.patterns.push(p); + } } } "hdb" | "hsb" | "hdu" | "hsu" => self.hashes.extend_from_text_prov(text, unofficial), @@ -488,11 +504,11 @@ pub fn load_with_pua(path: &Path, detect_pua: bool) -> Result assert_eq!(signature, "Test.Db"), + other => panic!("expected the .db signature, got {other:?}"), + } + } + + /// `PUA.*` stays off unless asked for on every path, the streaming literal + /// pass included. + #[test] + #[cfg_attr( + target_family = "wasm", + ignore = "host filesystem/tempdir unavailable under WASI" + )] + fn pua_literals_stay_off_the_streaming_path_by_default() { + use crate::{scan_stream, Verdict}; + let dir = crate::tmpfile::TempDir::new().unwrap(); + std::fs::write( + dir.path().join("p.ndb"), + "PUA.Test.X:0:*:70756170756170756170\n", + ) + .unwrap(); + std::fs::write(dir.path().join("a.ndb"), "Demo.X:0:*:6e6f7065\n").unwrap(); + let data = b"xx puapuapuap xx"; + let db = load(dir.path()).unwrap(); + assert_eq!(scan_stream(&db, &data[..]).unwrap().verdict, Verdict::Clean); + + let mut b = Builder::new(); + b.set_detect_pua(true); + b.add_path(dir.path()).unwrap(); + let pua = b.build().unwrap(); + assert!(matches!( + scan_stream(&pua, &data[..]).unwrap().verdict, + Verdict::Infected { .. } + )); + } + + /// A literal `.ndb` signature is held twice, by the engine and by the + /// streaming set, and must be counted once. + #[test] + #[cfg_attr( + target_family = "wasm", + ignore = "host filesystem/tempdir unavailable under WASI" + )] + fn a_literal_signature_is_counted_once() { + let dir = crate::tmpfile::TempDir::new().unwrap(); + std::fs::write(dir.path().join("a.ndb"), "Demo.X:0:*:6e6f7065\n").unwrap(); + let db = load(dir.path()).unwrap(); + // It and the always-present EICAR test signature. + assert_eq!(db.signature_count(), 2); + } } diff --git a/crates/exav-core/src/profile.rs b/crates/exav-core/src/profile.rs index ed2593f4..3a26fb92 100644 --- a/crates/exav-core/src/profile.rs +++ b/crates/exav-core/src/profile.rs @@ -4,4 +4,4 @@ //! scan infra at the bottom of the dependency stack, so the packer emulator in //! `exav-unpack` and the matchers here record into the same thread-local //! profile. Re-exported so existing paths keep working. -pub use exav_unpack::profile::{Profile, Stat, enable, take, timed}; +pub use exav_unpack::profile::{enable, take, timed, Profile, Stat}; diff --git a/crates/exav-core/src/yara/mod.rs b/crates/exav-core/src/yara/mod.rs index 1e7b7af8..01f661df 100644 --- a/crates/exav-core/src/yara/mod.rs +++ b/crates/exav-core/src/yara/mod.rs @@ -153,7 +153,8 @@ impl YaraDb { self.sources.is_empty() } /// Add one rule file's source text. `unofficial` marks rules loaded from a - /// non-`.cvd` database (the usual case), so matches carry `.UNOFFICIAL`. + /// non-`.cvd` database (the usual case), so compat scans name matches + /// `.UNOFFICIAL`. pub fn extend_from_text(&mut self, text: &str, unofficial: bool) { self.sources.push(text.to_string()); self.unofficial |= unofficial; @@ -235,9 +236,15 @@ impl YaraDb { .as_ref() } - /// Scan `data`; return the name of the first matching rule, formatted as - /// ClamAV reports YARA detections: a `YARA.` prefix and, for rules from an - /// unofficial database, a `.UNOFFICIAL` suffix. + /// Whether any rule came from an unofficial database, for the report + /// layer's `.UNOFFICIAL` suffix. + pub fn unofficial(&self) -> bool { + self.unofficial + } + + /// Scan `data`; return every matching rule, in rule order, named as ClamAV + /// reports YARA detections (`YARA.`). The `.UNOFFICIAL` suffix is the + /// report layer's, see [`Self::unofficial`]. /// /// `filename` is the identity of the object being scanned (a file path). /// When present, the standard THOR/signature-base external variables are set @@ -252,8 +259,10 @@ impl YaraDb { /// `filename` is `None`, all externals stay undefined and rules referencing /// them simply do not match. #[cfg(feature = "yara")] - pub fn scan(&self, data: &[u8], filename: Option<&str>) -> Option { - let rules = self.rules()?; + pub fn scan(&self, data: &[u8], filename: Option<&str>) -> Vec { + let Some(rules) = self.rules() else { + return Vec::new(); + }; let mut scanner = Scanner::new(rules); if let Some(path) = filename { scanner.set_global("filepath", path); @@ -263,16 +272,24 @@ impl YaraDb { scanner.set_global("extension", ext); } } - let results = scanner.scan(data).ok()?; - let id = results.matching_rules().next()?.identifier().to_string(); - let suffix = if self.unofficial { ".UNOFFICIAL" } else { "" }; - Some(format!("YARA.{id}{suffix}")) + let Ok(results) = scanner.scan(data) else { + return Vec::new(); + }; + // A condition that ran out of steps evaluated to undefined, so a rule + // may have failed to match for that reason alone. + if results.budget_exhausted() { + crate::engine::mark_scan_truncated(); + } + results + .matching_rules() + .map(|r| format!("YARA.{}", r.identifier())) + .collect() } /// Built without the `yara` feature: rules are stored but never matched. #[cfg(not(feature = "yara"))] - pub fn scan(&self, _data: &[u8], _filename: Option<&str>) -> Option { - None + pub fn scan(&self, _data: &[u8], _filename: Option<&str>) -> Vec { + Vec::new() } } @@ -372,6 +389,10 @@ fn preview(rejected: &[String]) -> String { mod tests { use super::*; + fn first(db: &YaraDb, data: &[u8], filename: Option<&str>) -> Option { + db.scan(data, filename).into_iter().next() + } + #[test] fn matches_a_simple_rule() { let mut db = YaraDb::new(); @@ -384,14 +405,15 @@ mod tests { $a } "#, - true, // unofficial -> YARA..UNOFFICIAL + true, ); assert_eq!(db.len(), 1); + assert!(db.unofficial()); assert_eq!( - db.scan(b"....UNIQUE_EXAV_YARA_MARKER....", None).as_deref(), - Some("YARA.evil_marker.UNOFFICIAL") + first(&db, b"....UNIQUE_EXAV_YARA_MARKER....", None).as_deref(), + Some("YARA.evil_marker") ); - assert!(db.scan(b"nothing to see here", None).is_none()); + assert!(first(&db, b"nothing to see here", None).is_none()); } #[test] @@ -407,13 +429,14 @@ mod tests { $mz at 0 and $s } "#, - false, // official -> YARA. prefix only, no .UNOFFICIAL + false, ); + assert!(!db.unofficial()); assert_eq!( - db.scan(b"MZ\x90\x00 ... evilfn ...", None).as_deref(), + first(&db, b"MZ\x90\x00 ... evilfn ...", None).as_deref(), Some("YARA.pe_with_two") ); - assert!(db.scan(b"xxMZ evilfn", None).is_none()); + assert!(first(&db, b"xxMZ evilfn", None).is_none()); } #[test] @@ -436,7 +459,7 @@ mod tests { db.finalize(); assert_eq!(db.rejected_count(), 0, "rejected: {:?}", db.rejected()); assert_eq!( - db.scan(b"MZ .... evilfn ....", None).as_deref(), + first(&db, b"MZ .... evilfn ....", None).as_deref(), Some("YARA.uses_pe") ); } @@ -475,11 +498,9 @@ mod tests { let hit = b"a needle and version42 here"; let miss = b"a needle but no ver marker"; for d in [db, reloaded] { - assert_eq!( - d.scan(hit, None).as_deref(), - Some("YARA.blob_demo.UNOFFICIAL") - ); - assert!(d.scan(miss, None).is_none()); + assert_eq!(first(&d, hit, None).as_deref(), Some("YARA.blob_demo")); + assert!(d.unofficial()); + assert!(first(&d, miss, None).is_none()); } } @@ -494,7 +515,7 @@ mod tests { // Simulate a blob written by an incompatible engine version. db.blob_version = YARA_BLOB_VERSION.wrapping_add(1); // Still detects (recompiled from the retained sources). - assert_eq!(db.scan(b"..marker..", None).as_deref(), Some("YARA.v")); + assert_eq!(first(&db, b"..marker..", None).as_deref(), Some("YARA.v")); } #[test] @@ -521,20 +542,20 @@ mod tests { let data = b"....payload...."; // Right name + extension -> match. assert_eq!( - db.scan(data, Some("/tmp/Invoice_2026.JS")).as_deref(), - Some("YARA.js_dropper.UNOFFICIAL") + first(&db, data, Some("/tmp/Invoice_2026.JS")).as_deref(), + Some("YARA.js_dropper") ); // Windows path, basename still resolves the extension. assert_eq!( - db.scan(data, Some("C:\\Users\\x\\invoice.js")).as_deref(), - Some("YARA.js_dropper.UNOFFICIAL") + first(&db, data, Some("C:\\Users\\x\\invoice.js")).as_deref(), + Some("YARA.js_dropper") ); // Wrong extension -> no match. - assert!(db.scan(data, Some("/tmp/invoice.txt")).is_none()); + assert!(first(&db, data, Some("/tmp/invoice.txt")).is_none()); // Wrong filename -> no match. - assert!(db.scan(data, Some("/tmp/report.js")).is_none()); + assert!(first(&db, data, Some("/tmp/report.js")).is_none()); // No filename at all -> externals undefined -> no match. - assert!(db.scan(data, None).is_none()); + assert!(first(&db, data, None).is_none()); } #[test] @@ -564,8 +585,8 @@ mod tests { assert!(db.rejected()[0].contains("bad_rule")); // The good rule still matches despite its neighbour being rejected. assert_eq!( - db.scan(b"....keepme....", None).as_deref(), - Some("YARA.good_rule.UNOFFICIAL") + first(&db, b"....keepme....", None).as_deref(), + Some("YARA.good_rule") ); } } diff --git a/crates/exav-core/tests/suites/encrypted_archives.rs b/crates/exav-core/tests/suites/encrypted_archives.rs index 55ea10ae..da232959 100644 --- a/crates/exav-core/tests/suites/encrypted_archives.rs +++ b/crates/exav-core/tests/suites/encrypted_archives.rs @@ -118,3 +118,33 @@ fn pwdb_pool_supplies_decryption_password() { other => panic!("expected Infected from .pwdb pool, got {other:?}"), } } + +/// A picture with an encrypted ZIP appended opens as that ZIP in the usual +/// archive tools, so it is reported like the ZIP would be. +#[test] +fn an_encrypted_zip_appended_to_a_picture_is_reported() { + let db = builtin_db(); + let zip = fixture("zip_aes256_store.zip"); + let mut jpeg = b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00".to_vec(); + jpeg.resize(4096, 0x42); + jpeg.extend_from_slice(b"\xff\xd9"); + let polyglot = [&jpeg[..], &zip[..]].concat(); + + match analyze(&db, &polyglot, &ScanOptions::default()).verdict { + Verdict::PasswordProtected { .. } => {} + other => panic!("expected PasswordProtected, got {other:?}"), + } + match analyze(&db, &polyglot, &with_pw("secret")).verdict { + Verdict::Infected { .. } => {} + other => panic!("expected Infected with the password, got {other:?}"), + } + + // Without its central directory the local header alone does not prove an + // archive is there: a chance `PK\x03\x04` looks the same. + let cd = zip.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let headless = [&jpeg[..], &zip[..cd]].concat(); + assert_eq!( + analyze(&db, &headless, &ScanOptions::default()).verdict, + Verdict::Clean + ); +} diff --git a/crates/exav-core/tests/suites/heuristic_alerts.rs b/crates/exav-core/tests/suites/heuristic_alerts.rs index 958d19b2..7ac2c45a 100644 --- a/crates/exav-core/tests/suites/heuristic_alerts.rs +++ b/crates/exav-core/tests/suites/heuristic_alerts.rs @@ -78,6 +78,32 @@ fn alert_encrypted_with_password_still_finds_payload() { } } +#[test] +fn a_decrypted_7z_member_is_still_reported_as_encrypted() { + // Decrypting with a pool password must not erase the fact that the member + // was encrypted: under all-match both the payload and the heuristic are + // reported, as they are for ZIP. + let db = builtin_db(); + let blob = fixture("aes256_hdr.7z"); + let mut opts = ScanOptions::default(); + opts.alert_encrypted = true; + opts.passwords = vec!["password".to_string()]; + let names: Vec = exav_core::analyze_all(&db, &blob, &opts) + .into_iter() + .map(|(n, _)| n) + .collect(); + assert!( + names + .iter() + .any(|n| n.to_ascii_uppercase().contains("EICAR")), + "{names:?}" + ); + assert!( + names.iter().any(|n| n == "Heuristics.Encrypted.7Zip"), + "{names:?}" + ); +} + // --- alert-macros ---------------------------------------------------------- /// Build a minimal, valid OLE2 (Compound File Binary) document carrying a VBA diff --git a/crates/exav-core/tests/suites/limits_alerts.rs b/crates/exav-core/tests/suites/limits_alerts.rs index fdf8717e..e7e1bb30 100644 --- a/crates/exav-core/tests/suites/limits_alerts.rs +++ b/crates/exav-core/tests/suites/limits_alerts.rs @@ -105,6 +105,66 @@ fn the_recursion_budget_reports_its_own_name() { } } +/// A PE shaped like a packed file (entry point in a writable section, a +/// destination section with no raw data) whose stub is `jmp $`. +#[cfg(any(feature = "pe-emu", feature = "all-formats"))] +fn spinning_packed_pe() -> Vec { + let (pe, opt_size, raw) = (0x80usize, 0xe0usize, 0x400usize); + let mut d = vec![0u8; raw + 0x1000]; + d[..2].copy_from_slice(b"MZ"); + d[0x3c..0x40].copy_from_slice(&(pe as u32).to_le_bytes()); + d[pe..pe + 4].copy_from_slice(b"PE\0\0"); + let coff = pe + 4; + d[coff..coff + 2].copy_from_slice(&0x14cu16.to_le_bytes()); + d[coff + 2..coff + 4].copy_from_slice(&2u16.to_le_bytes()); + d[coff + 16..coff + 18].copy_from_slice(&(opt_size as u16).to_le_bytes()); + let opt = coff + 20; + d[opt..opt + 2].copy_from_slice(&0x10bu16.to_le_bytes()); + d[opt + 16..opt + 20].copy_from_slice(&0xe000u32.to_le_bytes()); + d[opt + 28..opt + 32].copy_from_slice(&0x40_0000u32.to_le_bytes()); + d[opt + 32..opt + 36].copy_from_slice(&0x1000u32.to_le_bytes()); + d[opt + 36..opt + 40].copy_from_slice(&0x200u32.to_le_bytes()); + d[opt + 56..opt + 60].copy_from_slice(&0x2_0000u32.to_le_bytes()); + d[opt + 60..opt + 64].copy_from_slice(&0x400u32.to_le_bytes()); + let s0 = opt + opt_size; + d[s0..s0 + 5].copy_from_slice(b".text"); + d[s0 + 8..s0 + 12].copy_from_slice(&0xd000u32.to_le_bytes()); + d[s0 + 12..s0 + 16].copy_from_slice(&0x1000u32.to_le_bytes()); + d[s0 + 36..s0 + 40].copy_from_slice(&0xe000_0020u32.to_le_bytes()); + let s1 = s0 + 40; + d[s1..s1 + 5].copy_from_slice(b".data"); + d[s1 + 8..s1 + 12].copy_from_slice(&0x1000u32.to_le_bytes()); + d[s1 + 12..s1 + 16].copy_from_slice(&0xe000u32.to_le_bytes()); + d[s1 + 16..s1 + 20].copy_from_slice(&0x1000u32.to_le_bytes()); + d[s1 + 20..s1 + 24].copy_from_slice(&(raw as u32).to_le_bytes()); + d[s1 + 36..s1 + 40].copy_from_slice(&0xe000_0060u32.to_le_bytes()); + d[raw..raw + 2].copy_from_slice(&[0xeb, 0xfe]); + d +} + +#[cfg(any(feature = "pe-emu", feature = "all-formats"))] +#[test] +fn the_emulation_budget_reports_its_own_name() { + let db = scanner(); + let blob = spinning_packed_pe(); + let mut opts = ScanOptions::default(); + opts.limits.max_pe_emulation_steps = 10_000; + match analyze(&db, &blob, &opts).verdict { + Verdict::LimitsExceeded { reason } => assert!( + reason.contains("--max-pe-emulation-steps"), + "the reason names the flag to raise: {reason}" + ), + other => panic!("expected LIMITS-EXCEEDED, got {other:?}"), + } + opts.alert_exceeds_max = true; + match analyze(&db, &blob, &opts).verdict { + Verdict::Infected { signature, .. } => { + assert_eq!(signature, "Heuristics.Limits.Exceeded.MaxScanTime") + } + other => panic!("expected MaxScanTime, got {other:?}"), + } +} + #[test] fn a_real_detection_still_beats_the_limit_alert() { // Precedence matters: a file that both trips a budget and carries malware diff --git a/crates/exav-core/tests/suites/mod.rs b/crates/exav-core/tests/suites/mod.rs index 1cc6ed3c..4ab64d7b 100644 --- a/crates/exav-core/tests/suites/mod.rs +++ b/crates/exav-core/tests/suites/mod.rs @@ -35,6 +35,8 @@ mod matryoshka; #[cfg(feature = "all-formats")] mod multi_volume; #[cfg(feature = "all-formats")] +mod one_scan; +#[cfg(feature = "all-formats")] mod ooxml_container; #[cfg(feature = "all-formats")] mod oversize_container; diff --git a/crates/exav-core/tests/suites/one_scan.rs b/crates/exav-core/tests/suites/one_scan.rs new file mode 100644 index 00000000..dcda19fd --- /dev/null +++ b/crates/exav-core/tests/suites/one_scan.rs @@ -0,0 +1,94 @@ +//! A file path and the same bytes from any other source get one answer: both +//! entry points are the same scan, at every size threshold. + +use std::io::Write; + +use exav_core::{scan_path, scan_seekable, ScanOptions, Scanner, Verdict}; + +fn eicar() -> &'static [u8] { + exav_core::unpack::eicar() +} + +/// The verdicts of `scan_path` and `scan_seekable` on `blob`. +fn both(blob: &[u8], opts: &ScanOptions) -> [(&'static str, Verdict); 2] { + let db = Scanner::builtin(); + let dir = crate::tmpfile::TempDir::new().unwrap(); + let path = dir.path().join("input"); + std::fs::write(&path, blob).unwrap(); + let file = scan_path(&db, &path, opts).unwrap().verdict; + let seekable = scan_seekable(&db, std::io::Cursor::new(blob), blob.len() as u64, opts) + .unwrap() + .verdict; + [("scan_path", file), ("scan_seekable", seekable)] +} + +fn zip_with(member: &[u8], comment: &str) -> Vec { + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + z.set_comment(comment); + z.start_file( + "member.bin", + zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated), + ) + .unwrap(); + z.write_all(member).unwrap(); + z.finish().unwrap().into_inner() +} + +/// Over `--max-input-bytes` the first bytes get the whole scan, not only the +/// literal pass: EICAR deflated inside a ZIP is not in the file's bytes, and +/// only extraction finds it. +#[test] +fn the_start_of_an_oversize_input_gets_the_whole_scan() { + // Compressible text first, so the deflater codes EICAR rather than + // storing it verbatim. + let mut member = vec![b'a'; 4096]; + member.extend_from_slice(eicar()); + let zip = zip_with(&member, ""); + assert!(!zip.windows(eicar().len()).any(|w| w == eicar())); + let mut blob = zip.clone(); + blob.extend(vec![0u8; 64 * 1024]); + let mut opts = ScanOptions::default(); + opts.max_scan_size = Some(zip.len() as u64 + 512); + for (entry, v) in both(&blob, &opts) { + assert!(matches!(v, Verdict::Infected { .. }), "{entry}: {v:?}"); + } + + // A payload past the limit is not scanned, and the input is not clean. + let mut blob = vec![b'.'; 64 * 1024]; + blob.extend_from_slice(eicar()); + opts.max_scan_size = Some(1024); + for (entry, v) in both(&blob, &opts) { + assert!( + matches!(v, Verdict::LimitsExceeded { .. }), + "{entry}: {v:?}" + ); + } +} + +/// A container over the deep-analysis limit is walked member by member, and +/// its own bytes get the constant-memory pass on every entry point. +#[test] +fn an_oversize_container_gets_the_same_passes_everywhere() { + let mut member = Vec::new(); + for i in 0..4000u32 { + member.extend_from_slice(format!("{i} clean line\n").as_bytes()); + } + let comment = String::from_utf8(eicar().to_vec()).unwrap(); + let blob = zip_with(&member, &comment); + let mut opts = ScanOptions::default(); + opts.deep_analysis_max = 4096; + assert!(blob.len() > 4096); + for (entry, v) in both(&blob, &opts) { + assert!(matches!(v, Verdict::Infected { .. }), "{entry}: {v:?}"); + } + + // The counterweight: nothing in its bytes, and it is a limit, not clean. + let blob = zip_with(&member, ""); + for (entry, v) in both(&blob, &opts) { + assert!( + matches!(v, Verdict::LimitsExceeded { .. }), + "{entry}: {v:?}" + ); + } +} diff --git a/crates/exav-core/tests/suites/oversize_container.rs b/crates/exav-core/tests/suites/oversize_container.rs index 2166c7ba..534f5c3e 100644 --- a/crates/exav-core/tests/suites/oversize_container.rs +++ b/crates/exav-core/tests/suites/oversize_container.rs @@ -1,12 +1,11 @@ -//! An oversize container is never clean, whichever entry point sees it. +//! A file past `deep_analysis_max` is never clean, whichever entry point sees it. //! -//! Past `deep_analysis_max` the structural walk does not run: the flat -//! pattern+hash core reads the bytes and that is all. For flat content that is -//! a complete scan and `Clean` is the truth. For a container it is not — the -//! members were never opened, so nothing that only structural parsing finds -//! could have fired, and reporting `Clean` is the never-a-silent-clean -//! invariant broken in the one case where the file is large enough to be worth -//! hiding something in. +//! Past that size the full engine does not run: the streaming core reads the +//! bytes, and it matches literal signatures and whole-file hashes only. That is +//! not a complete scan of any file, and for a container the members were never +//! opened either, so reporting `Clean` would break the never-a-silent-clean +//! invariant in the one case where the file is large enough to be worth hiding +//! something in. //! //! [`scan_path`] and [`scan_seekable`] reach that decision by separate routes — //! one buffers from a `File`, the other from a `Read + Seek` that may be an @@ -75,18 +74,66 @@ fn scan_path_refuses_the_same_container() { } } +/// A ZIP is walked member by member at any size, but past the cap neither its +/// own bytes nor a member that large get the full engine. That is the same +/// size limit, and it is named as one. #[test] -fn oversize_flat_content_stays_clean() { - // The counterweight: the refusal above has to be about containers, not - // about size. A large text file was fully examined by the flat core, and - // downgrading it would make every big log file unscannable. +fn an_oversize_streamed_container_or_member_is_a_limit() { + let db = scanner(); + let zip_of = |members: &[(String, Vec)]| { + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let stored = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + for (name, body) in members { + z.start_file(name.as_str(), stored).unwrap(); + z.write_all(body).unwrap(); + } + z.finish().unwrap().into_inner() + }; + let many_small: Vec<_> = (0..64) + .map(|i| (format!("m{i}.txt"), vec![b'a'; 200])) + .collect(); + let one_large = vec![("big.txt".to_string(), vec![b'a'; 64 * 1024])]; + let flat = vec![b'a'; 64 * 1024]; + for blob in [zip_of(&many_small), zip_of(&one_large), flat] { + let size = blob.len() as u64; + let report = scan_seekable( + &db, + std::io::Cursor::new(blob.clone()), + size, + &tiny_deep_analysis(), + ) + .expect("scan"); + match report.verdict { + Verdict::LimitsExceeded { reason } => { + assert!(reason.contains("--max-object-bytes"), "{reason}") + } + other => panic!("expected LIMITS-EXCEEDED, got {other:?}"), + } + // Under ClamAV's naming it is the size alert ClamAV raises. + let mut alert = tiny_deep_analysis(); + alert.alert_exceeds_max = true; + let report = scan_seekable(&db, std::io::Cursor::new(blob), size, &alert).expect("scan"); + match report.verdict { + Verdict::Infected { signature, .. } => { + assert_eq!(signature, "Heuristics.Limits.Exceeded.MaxFileSize") + } + other => panic!("expected the MaxFileSize alert, got {other:?}"), + } + } +} + +#[test] +fn oversize_flat_content_is_a_limit_too() { + // Flat content past the cap got the literal-and-hash pass only, so it is + // not fully scanned either. let db = scanner(); let blob = vec![b'a'; 64 * 1024]; let size = blob.len() as u64; let report = scan_seekable(&db, std::io::Cursor::new(blob), size, &tiny_deep_analysis()).expect("scan"); match report.verdict { - Verdict::Clean => {} - other => panic!("flat content past the cap is fully scanned; got {other:?}"), + Verdict::LimitsExceeded { .. } => {} + other => panic!("flat content past the cap had a literal-only scan; got {other:?}"), } } diff --git a/crates/exav-core/tests/suites/salvage.rs b/crates/exav-core/tests/suites/salvage.rs index 85a14184..c5271b75 100644 --- a/crates/exav-core/tests/suites/salvage.rs +++ b/crates/exav-core/tests/suites/salvage.rs @@ -226,6 +226,210 @@ fn a_malformed_archive_does_not_hide_a_detection_beside_it() { } } +/// Damage part way through a compressed stream, in each format that decodes +/// one. Needs those formats compiled in. +#[cfg(feature = "all-formats")] +mod damaged { + use super::*; + + /// Deflate data that decodes `prefix`, then hits a block of the reserved type + /// with `rest` behind it: content present in the file that no decoder reaches. + fn deflate_broken_after(prefix: &[u8], rest: &[u8]) -> Vec { + let mut e = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(prefix).unwrap(); + // A sync flush ends on a byte boundary with no final block. + e.flush().unwrap(); + let mut raw = e.get_ref().clone(); + // BFINAL 0, BTYPE 11. + raw.push(0x06); + raw.extend_from_slice(rest); + raw + } + + /// Long enough that a decoder hands some of it over before it reaches the + /// damage, which is the case a salvage exists for. + fn clean_prefix() -> Vec { + (0..4000u32) + .flat_map(|i| format!("line {i} of a clean prefix\n").into_bytes()) + .collect() + } + + const UNREACHED: &[u8] = &[0x5a; 4096]; + + fn gzip_header() -> Vec { + vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 0xff] + } + + fn gzip_broken() -> Vec { + let mut g = gzip_header(); + g.extend(deflate_broken_after(&clean_prefix(), UNREACHED)); + g + } + + /// A gzip that decodes in full and then fails its CRC-32. + fn gzip_bad_crc() -> Vec { + let mut g = gzip(&clean_prefix(), flate2::Compression::default()); + let n = g.len(); + g[n - 8] ^= 0xff; + g + } + + fn zip_one(name: &str, body: &[u8]) -> Vec { + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + z.start_file( + name, + zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated), + ) + .unwrap(); + z.write_all(body).unwrap(); + z.finish().unwrap().into_inner() + } + + /// A ZIP whose deflated member breaks part way. + fn zip_broken() -> Vec { + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + z.start_file( + "m.bin", + zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored), + ) + .unwrap(); + z.write_all(&deflate_broken_after(&clean_prefix(), UNREACHED)) + .unwrap(); + let mut blob = z.finish().unwrap().into_inner(); + // Relabel the stored member as deflated, in its local header and its + // directory record. + blob[8..10].copy_from_slice(&8u16.to_le_bytes()); + let cd = blob.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + blob[cd + 10..cd + 12].copy_from_slice(&8u16.to_le_bytes()); + blob + } + + /// A ZIP whose member decodes in full and then fails its CRC-32. + fn zip_bad_crc() -> Vec { + let mut blob = zip_one("m.bin", &clean_prefix()); + blob[14] ^= 0xff; + let cd = blob.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + blob[cd + 16] ^= 0xff; + blob + } + + fn in_zip(name: &str, data: &[u8]) -> Vec { + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + z.start_file( + name, + zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored), + ) + .unwrap(); + z.write_all(data).unwrap(); + z.finish().unwrap().into_inner() + } + + fn in_tar(name: &str, data: &[u8]) -> Vec { + let mut ar = tar::Builder::new(Vec::new()); + let mut h = tar::Header::new_gnu(); + h.set_size(data.len() as u64); + h.set_mode(0o644); + h.set_cksum(); + ar.append_data(&mut h, name, data).unwrap(); + ar.into_inner().unwrap() + } + + /// A PDF with one FlateDecode stream. + fn pdf_with_stream(zlib: &[u8]) -> Vec { + let mut p = format!( + "%PDF-1.4\n1 0 obj\n<< /Length {} /Filter /FlateDecode >>\nstream\n", + zlib.len() + ) + .into_bytes(); + p.extend_from_slice(zlib); + p.extend_from_slice(b"\nendstream\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n"); + p + } + + fn zlib(payload: &[u8]) -> Vec { + let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(payload).unwrap(); + e.finish().unwrap() + } + + fn verdict(blob: &[u8]) -> Verdict { + analyze(&Scanner::builtin(), blob, &ScanOptions::default()).verdict + } + + /// Corruption part way through a compressed stream leaves content in the + /// file that was never decoded, so a clean prefix is not a clean file. The + /// same answer whether the stream is the file or sits inside another one. + #[test] + fn content_left_undecoded_is_not_clean() { + let mut pdf_broken = vec![0x78, 0x9c]; + pdf_broken.extend(deflate_broken_after(&clean_prefix(), UNREACHED)); + for (what, blob) in [ + ("gzip", gzip_broken()), + ("gzip in tar", in_tar("x.gz", &gzip_broken())), + ("gzip in zip", in_zip("x.gz", &gzip_broken())), + ("zip", zip_broken()), + ("zip in tar", in_tar("x.zip", &zip_broken())), + ("zip in zip", in_zip("x.zip", &zip_broken())), + ("pdf", pdf_with_stream(&pdf_broken)), + ] { + match verdict(&blob) { + Verdict::Unscannable { .. } => {} + other => panic!("{what}: expected Unscannable, got {other:?}"), + } + } + } + + /// A stream that decoded in full and then failed its checksum hides nothing: + /// every byte it holds was scanned. exav is not an integrity checker. + #[test] + fn a_bad_checksum_after_a_full_decode_is_clean() { + let mut pdf_adler = zlib(&clean_prefix()); + let n = pdf_adler.len(); + pdf_adler[n - 1] ^= 0xff; + for (what, blob) in [ + ("gzip", gzip_bad_crc()), + ("gzip in tar", in_tar("x.gz", &gzip_bad_crc())), + ("gzip in zip", in_zip("x.gz", &gzip_bad_crc())), + ("zip", zip_bad_crc()), + ("zip in tar", in_tar("x.zip", &zip_bad_crc())), + ("zip in zip", in_zip("x.zip", &zip_bad_crc())), + ("pdf", pdf_with_stream(&pdf_adler)), + ] { + match verdict(&blob) { + Verdict::Clean => {} + other => panic!("{what}: expected Clean, got {other:?}"), + } + } + } + + /// The counterweight: each broken stream still has its prefix scanned, and a + /// detection there wins over the damage after it. EICAR sits right before the + /// damage, in the output of the very read that fails. + #[test] + fn a_detection_before_the_damage_is_still_found() { + let mut prefix = clean_prefix(); + prefix.extend_from_slice(eicar()); + let mut gz = gzip_header(); + gz.extend(deflate_broken_after(&prefix, UNREACHED)); + let mut pdf = vec![0x78, 0x9c]; + pdf.extend(deflate_broken_after(&prefix, UNREACHED)); + for (what, blob) in [ + ("gzip", gz.clone()), + ("gzip in tar", in_tar("x.gz", &gz)), + ("gzip in zip", in_zip("x.gz", &gz)), + ("pdf", pdf_with_stream(&pdf)), + ] { + match verdict(&blob) { + Verdict::Infected { .. } => {} + other => panic!("{what}: expected Infected, got {other:?}"), + } + } + } +} + #[test] fn intact_gzip_still_works() { // Sanity: the salvage path doesn't regress the normal (untruncated) case. diff --git a/crates/exav-pe-emu/src/win.rs b/crates/exav-pe-emu/src/win.rs index 579a8cbe..ccd6f963 100644 --- a/crates/exav-pe-emu/src/win.rs +++ b/crates/exav-pe-emu/src/win.rs @@ -80,6 +80,12 @@ const MAPPING_HANDLE_BASE: u32 = 0x0000_0800; const SAMPLE_PATH: &str = "C:\\sample.exe"; const PROCESS_HEAP: u32 = 0x0052_0000; +/// TLS indices a process has: `TLS_MINIMUM_AVAILABLE` (64) plus the 1024 +/// expansion slots. `TlsSetValue`/`TlsGetValue` reject anything past it. +const TLS_SLOTS: usize = 1088; +const TLS_OUT_OF_INDEXES: u32 = 0xffff_ffff; +const ERROR_INVALID_PARAMETER: u32 = 87; + /// A synthetic loaded module. #[derive(Debug, Clone)] pub struct Module { @@ -136,6 +142,7 @@ enum Api { TlsAlloc, TlsSetValue, TlsGetValue, + TlsFree, Sleep, CloseHandle, CreateFile { @@ -264,7 +271,7 @@ const APIS: &[ApiSpec] = &[ k("TlsAlloc", 0, Api::TlsAlloc), k("TlsSetValue", 2, Api::TlsSetValue), k("TlsGetValue", 1, Api::TlsGetValue), - k("TlsFree", 1, Api::Const(1)), + k("TlsFree", 1, Api::TlsFree), k("Sleep", 1, Api::Sleep), k("CloseHandle", 1, Api::CloseHandle), k("CreateFileA", 7, Api::CreateFile { wide: false }), @@ -723,8 +730,10 @@ pub struct Env<'a> { last_error: u32, tick: u32, perf: u64, - tls: FxHashMap, - tls_next: u32, + /// Per-index TLS values and which indices `TlsAlloc` handed out, both + /// `TLS_SLOTS` long so a guest cannot grow them. + tls: Vec, + tls_used: Vec, /// Exports that were called but not implemented, for diagnostics. pub missing_apis: Vec, /// Record every serviced call. Off on the scan path; the triage tool turns @@ -764,13 +773,15 @@ impl<'a> Env<'a> { last_error: 0, tick: 0x0001_0000, perf: 0x0010_0000, - tls: FxHashMap::default(), - tls_next: 1, + tls: vec![0; TLS_SLOTS], + tls_used: vec![false; TLS_SLOTS], missing_apis: Vec::new(), trace: false, api_log: Vec::new(), bulk_bytes: 0, }; + // Slot 0 starts taken: the first `TlsAlloc` returns 1. + env.tls_used[0] = true; map(mem, STACK_BASE, STACK_SIZE)?; map(mem, ENV_BASE, PAGE_SIZE as u32)?; map(mem, LDR_BASE, 0x4000)?; @@ -1355,20 +1366,50 @@ impl<'a> Env<'a> { mem.write_u32(p + 4, 0x01c9_0000).map_err(Stop::Fault)?; 0 } - Api::TlsAlloc => { - let i = self.tls_next; - self.tls_next += 1; - i - } + Api::TlsAlloc => match self.tls_used.iter().position(|&u| !u) { + Some(i) => { + self.tls_used[i] = true; + self.tls[i] = 0; + i as u32 + } + None => TLS_OUT_OF_INDEXES, + }, + // Like Windows, Set/Get only range-check the index; whether it was + // allocated is checked by `TlsFree` alone. Api::TlsSetValue => { let i = arg(cpu, mem, 0)?; let v = arg(cpu, mem, 1)?; - self.tls.insert(i, v); - 1 + match self.tls.get_mut(i as usize) { + Some(slot) => { + *slot = v; + 1 + } + None => { + self.last_error = ERROR_INVALID_PARAMETER; + 0 + } + } } Api::TlsGetValue => { let i = arg(cpu, mem, 0)?; - self.tls.get(&i).copied().unwrap_or(0) + match self.tls.get(i as usize) { + Some(&v) => v, + None => { + self.last_error = ERROR_INVALID_PARAMETER; + 0 + } + } + } + Api::TlsFree => { + let i = arg(cpu, mem, 0)? as usize; + if self.tls_used.get(i).copied().unwrap_or(false) { + self.tls_used[i] = false; + self.tls[i] = 0; + 1 + } else { + self.last_error = ERROR_INVALID_PARAMETER; + 0 + } } Api::Sleep => 0, Api::CloseHandle => 1, @@ -2184,6 +2225,65 @@ mod tests { ); } + /// Call a stdcall kernel32 export with `args` and return EAX. + fn call_k32(env: &mut Env, mem: &mut Mem, name: &str, args: &[u32]) -> u32 { + let mut cpu = Cpu::new(); + cpu.regs[ESP] = INITIAL_ESP; + for &a in args.iter().rev() { + cpu.push32(mem, a).unwrap(); + } + cpu.push32(mem, 0xdead_0000).unwrap(); // return address + cpu.eip = trap_for(env, "kernel32.dll", name); + assert_eq!(env.call(&mut cpu, mem).unwrap(), ApiEffect::Continue); + cpu.regs[EAX] + } + + /// TLS state is bounded by the index range, whatever indices a guest + /// passes, and `TlsFree` only releases what `TlsAlloc` handed out. + #[test] + fn tls_indices_are_bounded_and_freed() { + let (mut mem, mut env) = env(); + + let i = call_k32(&mut env, &mut mem, "TlsAlloc", &[]); + assert_eq!(i, 1); + assert_eq!(call_k32(&mut env, &mut mem, "TlsSetValue", &[i, 0x1234]), 1); + assert_eq!(call_k32(&mut env, &mut mem, "TlsGetValue", &[i]), 0x1234); + + for bad in [TLS_SLOTS as u32, 0x7fff_ffff, u32::MAX] { + assert_eq!(call_k32(&mut env, &mut mem, "TlsSetValue", &[bad, 1]), 0); + assert_eq!(env.last_error, ERROR_INVALID_PARAMETER); + assert_eq!(call_k32(&mut env, &mut mem, "TlsGetValue", &[bad]), 0); + } + assert_eq!(env.tls.len(), TLS_SLOTS); + + assert_eq!(call_k32(&mut env, &mut mem, "TlsFree", &[i]), 1); + assert_eq!(call_k32(&mut env, &mut mem, "TlsGetValue", &[i]), 0); + assert_eq!( + call_k32(&mut env, &mut mem, "TlsFree", &[i]), + 0, + "double free" + ); + assert_eq!( + call_k32(&mut env, &mut mem, "TlsFree", &[5]), + 0, + "never allocated" + ); + assert_eq!(call_k32(&mut env, &mut mem, "TlsFree", &[u32::MAX]), 0); + + // The freed index is reused, then the pool runs out. + assert_eq!(call_k32(&mut env, &mut mem, "TlsAlloc", &[]), 1); + for _ in 2..TLS_SLOTS { + assert_ne!( + call_k32(&mut env, &mut mem, "TlsAlloc", &[]), + TLS_OUT_OF_INDEXES + ); + } + assert_eq!( + call_k32(&mut env, &mut mem, "TlsAlloc", &[]), + TLS_OUT_OF_INDEXES + ); + } + fn trap_for(env: &Env, module: &str, name: &str) -> u32 { env.traps .iter() diff --git a/crates/exav-unpack/Cargo.toml b/crates/exav-unpack/Cargo.toml index 2bc04a87..3ecb7b59 100644 --- a/crates/exav-unpack/Cargo.toml +++ b/crates/exav-unpack/Cargo.toml @@ -221,7 +221,7 @@ ntfs = ["dep:lznt1"] upx = ["dep:flate2", "dep:lzma-rust2", "dep:base64"] # Structured documents. -ole = ["dep:cfb"] +ole = ["dep:cfb", "dep:flate2"] pdf = ["dep:flate2"] email = ["dep:mail-parser", "dep:base64"] diff --git a/crates/exav-unpack/src/formats/cab.rs b/crates/exav-unpack/src/formats/cab.rs index badef0d5..3a691c4f 100644 --- a/crates/exav-unpack/src/formats/cab.rs +++ b/crates/exav-unpack/src/formats/cab.rs @@ -35,8 +35,24 @@ pub(crate) fn stream_cab( ) .map_err(|e| LimitHit::new(format!("cab folder: {e}")))?; let mut pos = 0u64; + // Set once the folder stream fails: past that, offsets no longer + // match what the reader would return. + let mut stalled = false; for f in folder_files { let want = f.data_offset as u64; + if stalled { + budget.count_entry()?; + let m = MemberMeta { + name: f.name().to_string(), + comp_size: f.uncompressed_size as u64, + encrypted: false, + unsupported: Some("CAB folder could not be read up to this member"), + }; + if let Some(t) = visit(&m, None, budget) { + return Ok(Some(t)); + } + continue; + } // Both of these are members the cabinet's own directory names, so // they exist; this reader just cannot reach them. Reporting is the // whole difference between "no malware here" and "did not look". @@ -56,15 +72,20 @@ pub(crate) fn stream_cab( } continue; } - let skipped = skip_forward(&mut reader, want - pos); + let (skipped, failed) = skip_forward(&mut reader, want - pos); pos += skipped; + stalled = failed; if pos < want { budget.count_entry()?; let m = MemberMeta { name: f.name().to_string(), comp_size: f.uncompressed_size as u64, encrypted: false, - unsupported: Some("CAB folder ended before this member's offset"), + unsupported: Some(if failed { + "CAB folder could not be read up to this member" + } else { + "CAB folder ended before this member's offset" + }), }; if let Some(t) = visit(&m, None, budget) { return Ok(Some(t)); @@ -83,7 +104,7 @@ pub(crate) fn stream_cab( let out = visit_member(&meta_m, &mut window, budget, visit)?; // Drain any bytes the visitor left so `pos` advances by the full // size and the next file lands at the right offset. - let _ = std::io::copy(&mut window, &mut std::io::sink()); + stalled = std::io::copy(&mut window, &mut std::io::sink()).is_err(); out }; pos = want + f.uncompressed_size as u64; @@ -95,8 +116,9 @@ pub(crate) fn stream_cab( Ok(None) } -/// Read and discard up to `n` bytes; returns how many were skipped. -fn skip_forward(r: &mut R, n: u64) -> u64 { +/// Read and discard up to `n` bytes; returns how many were skipped, and whether +/// the reader failed rather than ended. +fn skip_forward(r: &mut R, n: u64) -> (u64, bool) { let mut skipped = 0u64; let mut buf = [0u8; 8192]; while skipped < n { @@ -104,10 +126,11 @@ fn skip_forward(r: &mut R, n: u64) -> u64 { match r.read(&mut buf[..want]) { Ok(0) => break, Ok(k) => skipped += k as u64, - Err(_) => break, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(_) => return (skipped, true), } } - skipped + (skipped, false) } pub(crate) fn extract_cab( @@ -142,16 +165,24 @@ pub(crate) fn extract_cab( continue; } }; - let buf = tolerant_read(reader, cap); + let (buf, failed) = tolerant_read(reader, cap); budget.commit(buf.len() as u64); - if let Some(r) = visit(Entry::new(name, buf), budget) { + let mut e = Entry::new(name, buf); + if failed { + e.unsupported = Some( + "CAB member data failed to decode part way; the bytes before \ + the failure were scanned", + ); + } + if let Some(r) = visit(e, budget) { return Ok(Some(r)); } } Ok(None) } -fn tolerant_read(mut r: R, cap: u64) -> Vec { +/// Up to `cap` bytes, and whether the reader failed before it ended. +fn tolerant_read(mut r: R, cap: u64) -> (Vec, bool) { let mut buf = Vec::new(); let mut chunk = [0u8; 8192]; while (buf.len() as u64) < cap { @@ -161,10 +192,11 @@ fn tolerant_read(mut r: R, cap: u64) -> Vec { let room = (cap - buf.len() as u64) as usize; buf.extend_from_slice(&chunk[..n.min(room)]); } - Err(_) => break, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(_) => return (buf, true), } } - buf + (buf, false) } fn repair_cab_size(data: &[u8]) -> Option> { diff --git a/crates/exav-unpack/src/formats/cpio.rs b/crates/exav-unpack/src/formats/cpio.rs index 20c0ca4a..2e5ad8ca 100644 --- a/crates/exav-unpack/src/formats/cpio.rs +++ b/crates/exav-unpack/src/formats/cpio.rs @@ -17,29 +17,11 @@ fn align4_u64(n: u64) -> u64 { } /// Read a member name of `namesize` bytes at `off` (capped, `\0`-trimmed). -fn read_name(source: &mut R, off: u64, namesize: u64) -> String { - let n = namesize.min(MAX_NAME) as usize; - let mut buf = vec![0u8; n]; - if source.seek(SeekFrom::Start(off)).is_err() { - return String::new(); - } - let got = fill(source, &mut buf); - String::from_utf8_lossy(&buf[..got]) +fn read_name(source: &mut R, off: u64, namesize: u64) -> Result { + let buf = crate::read_at(source, off, namesize.min(MAX_NAME) as usize)?; + Ok(String::from_utf8_lossy(&buf) .trim_end_matches('\0') - .to_string() -} - -/// Read up to `buf.len()` bytes, tolerating short reads; returns bytes filled. -fn fill(source: &mut R, buf: &mut [u8]) -> usize { - let mut n = 0; - while n < buf.len() { - match source.read(&mut buf[n..]) { - Ok(0) => break, - Ok(k) => n += k, - Err(_) => break, - } - } - n + .to_string()) } /// Parse member offsets from a seekable source (reader-based streaming): walk the @@ -81,11 +63,8 @@ fn stream_walk( let mut out = Vec::new(); let mut pos = 0u64; loop { - source - .seek(SeekFrom::Start(pos)) - .map_err(|e| LimitHit::corrupt(format!("cpio seek: {e}")))?; - let mut h = vec![0u8; hdr_len]; - if source.read_exact(&mut h).is_err() { + let h = crate::read_at(source, pos, hdr_len)?; + if h.len() < hdr_len { break; } let (namesize, filesize, magic_ok) = match &fields { @@ -115,7 +94,7 @@ fn stream_walk( break; } let name_start = pos + hdr_len as u64; - let name = read_name(source, name_start, namesize); + let name = read_name(source, name_start, namesize)?; if name == TRAILER { break; } diff --git a/crates/exav-unpack/src/formats/gzip.rs b/crates/exav-unpack/src/formats/gzip.rs index 2213f731..6db59ecb 100644 --- a/crates/exav-unpack/src/formats/gzip.rs +++ b/crates/exav-unpack/src/formats/gzip.rs @@ -7,25 +7,46 @@ pub(crate) fn extract_gzip( budget: &mut Budget, visit: Sink, ) -> Result, LimitHit> { - // `MultiGzDecoder`, not `GzDecoder`: a gzip file may be several concatenated - // members (RFC 1952 §2.2) and the payload can live in a later one. `GzDecoder` - // stops after the first member — a real FN source (observed: a 2-member gz - // whose first member is a 1 KiB header and whose second holds the malware). - // gzip/zcat/ClamAV all concatenate every member. - use flate2::read::MultiGzDecoder; budget.count_entry()?; let cap = budget.reserve()?; - // Scan-everything default: salvage the decompressed bytes even if flate2 - // reports a trailing CRC-32/ISIZE mismatch, so a corrupted checksum can't - // hide the payload from scanning (unless checksum verification is enabled). - let salvage = !budget.should_verify_checksums(); - let (out, truncated) = - bounded_read_salvage(MultiGzDecoder::new(Cursor::new(data)), cap, salvage) - .map_err(|e| LimitHit::new(format!("gzip: {e}")))?; - if truncated { + let s = gunzip(data, cap, budget).map_err(|e| LimitHit::new(format!("gzip: {e}")))?; + if s.over_cap { return Err(LimitHit::new("gzip member exceeds budget".to_string())); } - ratio_guard(data.len() as u64, out.len() as u64, budget)?; - budget.commit(out.len() as u64); - Ok(visit(Entry::new("gzip-content".to_string(), out), budget)) + ratio_guard(data.len() as u64, s.data.len() as u64, budget)?; + budget.commit(s.data.len() as u64); + Ok(visit(gzip_entry(s), budget)) +} + +/// Decode every member of a gzip file, up to `cap` bytes. Every member, not +/// the first: a gzip file may be several concatenated members (RFC 1952 +/// section 2.2) and the payload can live in a later one (observed: a 2-member +/// gz whose first member is a 1 KiB header and whose second holds the +/// malware). gzip, zcat and ClamAV all read them all. +/// +/// Unless checksum verification is on, a decode error keeps what was decoded +/// before it, so a damaged or mis-summed file still has its content scanned. +pub(crate) fn gunzip( + src: R, + cap: u64, + budget: &Budget, +) -> std::io::Result { + bounded_read_salvage( + crate::inflate::Gunzip::new(src), + cap, + !budget.should_verify_checksums(), + ) +} + +/// The entry for decoded gzip content, flagged when damage left part of it +/// undecoded. +pub(crate) fn gzip_entry(s: Salvaged) -> Entry { + let mut e = Entry::new("gzip-content".to_string(), s.data); + if s.undecoded { + e.unsupported = Some( + "gzip data failed to decode part way; the bytes before the failure \ + were scanned", + ); + } + e } diff --git a/crates/exav-unpack/src/formats/iso.rs b/crates/exav-unpack/src/formats/iso.rs index 758ccf72..134e9bf7 100644 --- a/crates/exav-unpack/src/formats/iso.rs +++ b/crates/exav-unpack/src/formats/iso.rs @@ -78,21 +78,14 @@ pub(crate) fn stream_offsets( let total_len = source .seek(std::io::SeekFrom::End(0)) .map_err(|e| LimitHit::corrupt(format!("iso: {e}")))?; + // `vd_roots` is shared with the in-memory walk and reads infallibly, so a + // failure is kept here and reported after the read that hit it. + let failed = std::cell::Cell::new(None); let mut read_at = |off: u64, len: usize| -> Vec { - let mut buf = vec![0u8; len]; - if source.seek(std::io::SeekFrom::Start(off)).is_err() { - return Vec::new(); - } - let mut n = 0; - while n < len { - match source.read(&mut buf[n..]) { - Ok(0) => break, - Ok(k) => n += k, - Err(_) => break, - } - } - buf.truncate(n); - buf + crate::read_at(source, off, len).unwrap_or_else(|e| { + failed.set(Some(e)); + Vec::new() + }) }; let le32 = |b: &[u8], o: usize| -> u64 { b.get(o..o + 4) @@ -100,6 +93,9 @@ pub(crate) fn stream_offsets( .unwrap_or(0) }; let roots = vd_roots(&mut read_at); + if let Some(e) = failed.take() { + return Err(e); + } if roots.is_empty() { return Ok(Vec::new()); } @@ -123,6 +119,9 @@ pub(crate) fn stream_offsets( let avail = total_len.saturating_sub(start); let read_len = len.min(avail).min(max_buffer) as usize; let dir = read_at(start, read_len); + if let Some(e) = failed.take() { + return Err(e); + } let mut p = 0usize; while p < dir.len() { let rec_len = dir[p] as usize; diff --git a/crates/exav-unpack/src/formats/machofat.rs b/crates/exav-unpack/src/formats/machofat.rs index e267725b..50c86458 100644 --- a/crates/exav-unpack/src/formats/machofat.rs +++ b/crates/exav-unpack/src/formats/machofat.rs @@ -137,14 +137,7 @@ pub(crate) fn stream_offsets( .map_err(|e| LimitHit::corrupt(format!("machofat: {e}")))?; // The largest possible arch table: 8-byte header + MAX_ARCH * 32-byte records. let mut header = vec![0u8; 8 + MAX_ARCH as usize * 32]; - let mut n = 0; - while n < header.len() { - match source.read(&mut header[n..]) { - Ok(0) => break, - Ok(k) => n += k, - Err(_) => break, - } - } + let n = crate::read_full(source, &mut header)?; header.truncate(n); let Some(archs) = parse_archs_len(&header, total_len) else { return Ok(Vec::new()); diff --git a/crates/exav-unpack/src/formats/mod.rs b/crates/exav-unpack/src/formats/mod.rs index ac0e2adb..e759e80e 100644 --- a/crates/exav-unpack/src/formats/mod.rs +++ b/crates/exav-unpack/src/formats/mod.rs @@ -190,7 +190,7 @@ pub(crate) use dmg::{extract_dmg, is_dmg}; #[cfg(feature = "email")] pub(crate) use email::extract_email; #[cfg(feature = "gzip")] -pub(crate) use gzip::extract_gzip; +pub(crate) use gzip::{extract_gzip, gunzip, gzip_entry}; #[cfg(feature = "iso")] pub(crate) use iso::extract_iso; #[cfg(feature = "javaclass")] diff --git a/crates/exav-unpack/src/formats/ole.rs b/crates/exav-unpack/src/formats/ole.rs index b8343df2..5cd30ee8 100644 --- a/crates/exav-unpack/src/formats/ole.rs +++ b/crates/exav-unpack/src/formats/ole.rs @@ -294,7 +294,7 @@ fn append_ppt_embedded_storages(entries: &mut Vec, budget: &mut Budget) { if !leaf.eq_ignore_ascii_case("PowerPoint Document") { continue; } - for (i, blob) in ppt_embedded_storages(&e.data, budget) + for (i, (blob, why)) in ppt_embedded_storages(&e.data, budget) .into_iter() .enumerate() { @@ -302,7 +302,9 @@ fn append_ppt_embedded_storages(entries: &mut Vec, budget: &mut Budget) { break; } budget.commit(blob.len() as u64); - carved.push(Entry::new(format!("ppt-embedded-{i}"), blob)); + let mut entry = Entry::new(format!("ppt-embedded-{i}"), blob); + entry.unsupported = why; + carved.push(entry); } } entries.extend(carved); @@ -317,7 +319,10 @@ fn append_ppt_embedded_storages(entries: &mut Vec, budget: &mut Budget) { /// how a record nested three levels down is reached at all. The instance (the /// high twelve bits) selects the storage form: `0` is stored, `1` is /// `[uncompressedSize:u32]` followed by a zlib stream. -fn ppt_embedded_storages(stream: &[u8], budget: &mut Budget) -> Vec> { +fn ppt_embedded_storages( + stream: &[u8], + budget: &mut Budget, +) -> Vec<(Vec, Option<&'static str>)> { /// `RT_ExternalOleObjectStg`. const EXT_OLE_OBJ_STG: u16 = 0x1011; /// A record tree deep enough to need more steps than this is malformed, and @@ -355,25 +360,44 @@ fn ppt_embedded_storages(stream: &[u8], budget: &mut Budget) -> Vec> { out } -/// One storage record's bytes: stored as-is, or inflated, bounded by the budget. -fn ppt_storage_payload(instance: u16, body: &[u8], budget: &mut Budget) -> Option> { +/// One storage record's bytes: stored as-is, or inflated, bounded by the +/// budget, and why they are not the whole storage when they are not. +fn ppt_storage_payload( + instance: u16, + body: &[u8], + budget: &mut Budget, +) -> Option<(Vec, Option<&'static str>)> { + const OVER: &str = "embedded PowerPoint storage exceeds the per-member budget"; + const BROKEN: &str = "embedded PowerPoint storage could not be decompressed"; + const PART_WAY: &str = "embedded PowerPoint storage failed to decompress part way; \ + the bytes before the failure were scanned"; let cap = budget.reserve().ok()?; if instance == 0 { // Stored. The record is the compound file. - return (body.len() as u64 <= cap).then(|| body.to_vec()); + return Some(if body.len() as u64 <= cap { + (body.to_vec(), None) + } else { + (Vec::new(), Some(OVER)) + }); } // Compressed: a declared size then a zlib stream. The declared size is a // hint from the file, so it is not trusted for allocation — the read is // bounded by the budget and salvages a truncated tail, which is how a // deliberately-truncated object still gets scanned rather than dropped. let deflated = body.get(4..)?; - let (data, _truncated) = bounded_read_salvage( - flate2::read::ZlibDecoder::new(Cursor::new(deflated)), - cap, - true, - ) - .ok()?; - (!data.is_empty()).then_some(data) + let Some(zlib) = crate::inflate::zlib_body(deflated) else { + return Some((Vec::new(), Some(BROKEN))); + }; + let s = bounded_read_salvage(zlib, cap, true).ok()?; + let why = match (s.over_cap, s.undecoded, s.data.is_empty()) { + (true, _, _) => Some(OVER), + (_, true, true) => Some(BROKEN), + (_, true, false) => Some(PART_WAY), + // Nothing decoded and nothing left undecoded: an empty storage. + (_, false, true) => return None, + (_, false, false) => None, + }; + Some((s.data, why)) } /// The payload bytes inside an `Ole10Native` stream, or `None` when the header @@ -1022,6 +1046,53 @@ mod macro_artifact_tests { ); } + /// A storage record hands over its bytes, and says so whenever they are + /// not all of it. + #[test] + fn a_ppt_storage_says_when_it_is_not_whole() { + use std::io::Write; + let compressed = |payload: &[u8]| { + let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(payload).unwrap(); + let mut body = (payload.len() as u32).to_le_bytes().to_vec(); + body.extend(e.finish().unwrap()); + body + }; + let mut b = Budget::new(Limits::default()); + + let intact = compressed(b"storage"); + let got = ppt_storage_payload(1, &intact, &mut b).unwrap(); + assert_eq!(got, (b"storage".to_vec(), None)); + + let mut bad_adler = intact.clone(); + let n = bad_adler.len(); + bad_adler[n - 1] ^= 1; + let got = ppt_storage_payload(1, &bad_adler, &mut b).unwrap(); + assert_eq!(got, (b"storage".to_vec(), None)); + + // Damaged after a long run of content: that run, and a reason. + let content: Vec = (0..20_000u32).flat_map(|i| i.to_le_bytes()).collect(); + let mut e = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(&content).unwrap(); + e.flush().unwrap(); + let mut damaged = vec![0, 0, 0, 0, 0x78, 0x9c]; + damaged.extend_from_slice(e.get_ref()); + damaged.extend_from_slice(&[0x06, 0x5a, 0x5a]); + let (data, why) = ppt_storage_payload(1, &damaged, &mut b).unwrap(); + assert_eq!(data, content); + assert!(why.is_some_and(|w| w.contains("part way")), "{why:?}"); + + let (data, why) = ppt_storage_payload(1, b"\0\0\0\0not zlib", &mut b).unwrap(); + assert!(data.is_empty() && why.is_some()); + + let mut small = Budget::new(Limits { + max_buffer_bytes: 4, + ..Limits::default() + }); + let (data, why) = ppt_storage_payload(0, b"stored storage", &mut small).unwrap(); + assert!(data.is_empty() && why.is_some()); + } + #[test] fn an_absent_macro_is_not_reported() { // No macro of this kind in the document: nothing to scan, nothing to say. diff --git a/crates/exav-unpack/src/formats/onenote.rs b/crates/exav-unpack/src/formats/onenote.rs index adb88d56..270467c3 100644 --- a/crates/exav-unpack/src/formats/onenote.rs +++ b/crates/exav-unpack/src/formats/onenote.rs @@ -65,19 +65,7 @@ pub(crate) fn stream_offsets( .seek(SeekFrom::End(0)) .map_err(|e| LimitHit::corrupt(format!("onenote: {e}")))?; let scan = max_buffer.min(file_len) as usize; - let mut prefix = vec![0u8; scan]; - if source.seek(SeekFrom::Start(0)).is_err() { - return Ok(Vec::new()); - } - let mut got = 0usize; - while got < prefix.len() { - match source.read(&mut prefix[got..]) { - Ok(0) => break, - Ok(k) => got += k, - Err(_) => break, - } - } - prefix.truncate(got); + let prefix = crate::read_at(source, 0, scan)?; let mut out = Vec::new(); for (idx, guid_at) in memchr::memmem::Finder::new(&FILE_DATA_STORE_GUID) diff --git a/crates/exav-unpack/src/formats/partition.rs b/crates/exav-unpack/src/formats/partition.rs index deac9c67..38b337aa 100644 --- a/crates/exav-unpack/src/formats/partition.rs +++ b/crates/exav-unpack/src/formats/partition.rs @@ -63,24 +63,6 @@ pub(crate) fn is_partition(data: &[u8]) -> bool { is_gpt(data) || is_apm(data) || is_mbr(data) } -/// Read `len` bytes at `off` from a seekable source (tolerant of short reads). -fn read_at(source: &mut R, off: u64, len: usize) -> Vec { - let mut buf = vec![0u8; len]; - if source.seek(std::io::SeekFrom::Start(off)).is_err() { - return Vec::new(); - } - let mut n = 0; - while n < len { - match source.read(&mut buf[n..]) { - Ok(0) => break, - Ok(k) => n += k, - Err(_) => break, - } - } - buf.truncate(n); - buf -} - /// One partition as a byte range `[first_lba*512, end_sector*512)` clamped to /// `total_len`; `None` for a degenerate/out-of-range slice. fn lba_range( @@ -155,7 +137,7 @@ pub(crate) fn stream_offsets( let total_len = source .seek(std::io::SeekFrom::End(0)) .map_err(|e| LimitHit::corrupt(format!("partition: {e}")))?; - let head = read_at(source, 0, 2 * SECTOR); // covers GPT sig, APM ER/PM, MBR table + let head = crate::read_at(source, 0, 2 * SECTOR)?; // covers GPT sig, APM ER/PM, MBR table let mut out = Vec::new(); if is_gpt(&head) { let entries_lba = le_u64(&head, SECTOR + 72); @@ -173,11 +155,11 @@ pub(crate) fn stream_offsets( return Ok(out); } let base = entries_lba.saturating_mul(SECTOR as u64); - let table = read_at( + let table = crate::read_at( source, base, (num_entries as usize).saturating_mul(entry_size as usize), - ); + )?; let mut emitted = 0usize; for i in 0..num_entries as usize { if emitted >= MAX_PARTS { @@ -213,7 +195,7 @@ pub(crate) fn stream_offsets( if emitted >= MAX_PARTS { break; } - let sector = read_at(source, ((i + 1) * SECTOR) as u64, SECTOR); + let sector = crate::read_at(source, ((i + 1) * SECTOR) as u64, SECTOR)?; if sector.get(0..2) != Some(b"PM".as_slice()) { break; } diff --git a/crates/exav-unpack/src/formats/pdf.rs b/crates/exav-unpack/src/formats/pdf.rs index 03d243a3..b9adf122 100644 --- a/crates/exav-unpack/src/formats/pdf.rs +++ b/crates/exav-unpack/src/formats/pdf.rs @@ -213,7 +213,14 @@ pub(crate) fn extract_pdf( "PDF stream could not be decoded", ) } else { - Entry::new(format!("pdf-obj-{obj_id}-{gen}"), buf) + let mut e = Entry::new(format!("pdf-obj-{obj_id}-{gen}"), buf); + if outcome.part_way { + e.unsupported = Some( + "PDF stream failed to decode part way; the bytes before the \ + failure were scanned", + ); + } + e }; if let Some(r) = visit(entry, budget) { return Ok(Some(r)); @@ -323,6 +330,22 @@ struct FilterOutcome { /// which is a successful decode of an empty stream (legal — e.g. an empty /// page-content stream) and must NOT be reported undecodable. decode_error: bool, + /// A filter decoded part of its input and failed on the rest: `buf` holds + /// what came before the failure. + part_way: bool, +} + +/// A zlib stream decoded up to any damage, without its Adler-32 check. +fn salvage_zlib(data: &[u8], cap: u64) -> Salvaged { + let failed = Salvaged { + data: Vec::new(), + over_cap: false, + undecoded: true, + }; + match crate::inflate::zlib_body(data) { + Some(body) => bounded_read_salvage(body, cap, true).unwrap_or(failed), + None => failed, + } } /// Decode a stream's raw bytes by applying its `/Filter` chain left to right, @@ -344,10 +367,12 @@ fn apply_filters( truncated: stream_data.len() as u64 > cap, // No decoding involved: raw bytes pass through, nothing to fail. decode_error: false, + part_way: false, }); } let parms = decode_parms_list(info, names.len()); let mut buf = stream_data.to_vec(); + let mut part_way = false; // Whether an implemented filter failed with nothing recovered. Sticky // across the chain: only a successful decode producing content clears the // question, and the caller reports `unsupported` solely on @@ -369,28 +394,17 @@ fn apply_filters( match bounded_read(flate2::read::ZlibDecoder::new(Cursor::new(&buf)), cap) { Ok((out, truncated)) => (out, truncated, false), Err(_) => { - let (mut out, mut truncated) = bounded_read_salvage( - flate2::read::ZlibDecoder::new(Cursor::new(&buf)), - cap, - true, - ) - .unwrap_or((Vec::new(), false)); - if out.is_empty() { + let mut s = salvage_zlib(&buf, cap); + if s.data.is_empty() { if let Some(z) = buf.iter().position(|&b| b == 0x78) { - let r = bounded_read_salvage( - flate2::read::ZlibDecoder::new(Cursor::new(&buf[z..])), - cap, - true, - ) - .unwrap_or((Vec::new(), false)); - out = r.0; - truncated = r.1; + s = salvage_zlib(&buf[z..], cap); } } // Empty input decodes vacuously (nothing to fail on); // only non-empty input decoding to nothing is an error. - let err = out.is_empty() && !buf.is_empty(); - (out, truncated, err) + let err = s.data.is_empty() && !buf.is_empty(); + part_way |= s.undecoded && !err; + (s.data, s.over_cap, err) } } } @@ -430,6 +444,7 @@ fn apply_filters( buf: out, truncated: true, decode_error, + part_way, }); } buf = out; @@ -439,6 +454,7 @@ fn apply_filters( buf, truncated, decode_error, + part_way, }) } diff --git a/crates/exav-unpack/src/formats/pepack.rs b/crates/exav-unpack/src/formats/pepack.rs index bcc809d7..fc77e7ca 100644 --- a/crates/exav-unpack/src/formats/pepack.rs +++ b/crates/exav-unpack/src/formats/pepack.rs @@ -864,14 +864,29 @@ pub(crate) fn emulated_unpack( ) -> Result, LimitHit> { budget.count_entry()?; let cap = budget.reserve()?.min(MAX_INNER as u64) as usize; + let room = budget.pe_emulation_room(); + if room == 0 { + return Err(budget.pe_emulation_exhausted()); + } let limits = x86::run::EmuLimits { - max_ticks: MAX_EMU_TICKS, + max_ticks: MAX_EMU_TICKS.min(room), max_dump: cap.min(MAX_EMU_DUMP), max_pages: cap.min(MAX_EMU_MEMORY) / exav_pe_emu::PAGE_SIZE, ..Default::default() }; let report = crate::profile::timed("emu", data.len() as u64, || x86::run::unpack(data, &limits)); + budget.charge_pe_emulation(report.ticks); + // The per-stub cap ending a run is the emulator's normal behavior; the + // scan-wide one ending it means this stub was not given its full share. + let cut_short = limits.max_ticks < MAX_EMU_TICKS && report.ticks >= limits.max_ticks; + let ended = |budget: &Budget, done: Recovered| { + if cut_short { + Err(budget.pe_emulation_exhausted()) + } else { + Ok(done) + } + }; // Payloads the stub built in memory it allocated. Emitted whether or not it // also rebuilt its own image: a loader that unfolds the original program @@ -891,12 +906,13 @@ pub(crate) fn emulated_unpack( } } + let fallback = if emitted { + Recovered::Emitted + } else { + Recovered::Nothing + }; let Some(unpacked) = report.unpacked else { - return Ok(if emitted { - Recovered::Emitted - } else { - Recovered::Nothing - }); + return ended(budget, fallback); }; // The same output gate the static path uses: only a buffer that reads back // as a PE image is emitted. A run that ended mid-decompression leaves the @@ -904,11 +920,7 @@ pub(crate) fn emulated_unpack( // rejects is a dump whose headers the stub overwrote with something else, // where there is no way to tell code from rubble. if !looks_like_pe(&unpacked.data) { - return Ok(if emitted { - Recovered::Emitted - } else { - Recovered::Nothing - }); + return ended(budget, fallback); } let name = if unpacked.reached_oep { format!("{label}-emulated") @@ -919,7 +931,7 @@ pub(crate) fn emulated_unpack( if let Some(r) = visit(Entry::new(name, unpacked.data), budget) { return Ok(Recovered::Halt(r)); } - Ok(Recovered::Emitted) + ended(budget, Recovered::Emitted) } fn packer_name(p: Packer) -> &'static str { diff --git a/crates/exav-unpack/src/formats/sevenz/entry.rs b/crates/exav-unpack/src/formats/sevenz/entry.rs index e6cd48f3..9c3829a4 100644 --- a/crates/exav-unpack/src/formats/sevenz/entry.rs +++ b/crates/exav-unpack/src/formats/sevenz/entry.rs @@ -255,11 +255,11 @@ fn decode_block( Ok(out) } -/// Read and discard up to `n` decompressed bytes from `r`, returning how many -/// were actually skipped (fewer than `n` means the stream ended first). Constant -/// memory — used to advance a block reader to a file's offset within a solid -/// block without buffering the skipped prefix. -fn skip_reader(r: &mut dyn Read, n: u64) -> u64 { +/// Read and discard up to `n` decompressed bytes from `r`. Returns how many +/// were skipped (fewer than `n` means the stream ended or failed first) and +/// whether it failed. Constant memory: used to advance a block reader to a +/// file's offset within a solid block without buffering the skipped prefix. +fn skip_reader(r: &mut dyn Read, n: u64) -> (u64, bool) { let mut skipped = 0u64; let mut buf = [0u8; 8192]; while skipped < n { @@ -267,10 +267,11 @@ fn skip_reader(r: &mut dyn Read, n: u64) -> u64 { match r.read(&mut buf[..want]) { Ok(0) => break, Ok(k) => skipped += k as u64, - Err(_) => break, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(_) => return (skipped, true), } } - skipped + (skipped, false) } /// Streaming 7z extraction (pattern A): each file's solid block is built as a @@ -422,11 +423,13 @@ pub(crate) fn stream_sevenz( } } let r = match file_data { + // Decrypted, and still reported as encrypted: the plaintext is + // scanned, and cracking the password does not erase the fact. Some(fd) => { let meta = MemberMeta { name, comp_size: file.size, - encrypted: false, + encrypted: true, unsupported: None, }; let mut cur = Cursor::new(fd); @@ -480,7 +483,8 @@ pub(crate) fn stream_sevenz( } else { decode_block_reader(block, pack_data, block_total, None, max_buffer)? }; - if skip_reader(reader.as_mut(), bytes_to_skip) < bytes_to_skip { + let (skipped, failed) = skip_reader(reader.as_mut(), bytes_to_skip); + if skipped < bytes_to_skip { // The block ended before this file's offset: the sub-stream table // claims more content than the block decodes to. Every later member // of a solid block is in the same position, so leaving these out @@ -489,7 +493,11 @@ pub(crate) fn stream_sevenz( name, comp_size: file.size, encrypted: false, - unsupported: Some("7z: solid block ended before this member"), + unsupported: Some(if failed { + "7z: solid block failed to decode before this member" + } else { + "7z: solid block ended before this member" + }), }; if let Some(t) = visit(&meta, None, budget) { return Ok(Some(t)); diff --git a/crates/exav-unpack/src/formats/sfx.rs b/crates/exav-unpack/src/formats/sfx.rs index eda29d61..6341e2de 100644 --- a/crates/exav-unpack/src/formats/sfx.rs +++ b/crates/exav-unpack/src/formats/sfx.rs @@ -120,14 +120,7 @@ pub(crate) fn stream_offsets( .map_err(|e| LimitHit::corrupt(format!("sfx: {e}")))?; let scan = max_buffer.min(len) as usize; let mut prefix = vec![0u8; scan]; - let mut n = 0; - while n < prefix.len() { - match source.read(&mut prefix[n..]) { - Ok(0) => break, - Ok(k) => n += k, - Err(_) => break, - } - } + let n = crate::read_full(source, &mut prefix)?; prefix.truncate(n); let Some((off, _)) = find_embedded_archive(&prefix) else { return Ok(Vec::new()); diff --git a/crates/exav-unpack/src/formats/tnef.rs b/crates/exav-unpack/src/formats/tnef.rs index e1f7f8d8..6d03a6c5 100644 --- a/crates/exav-unpack/src/formats/tnef.rs +++ b/crates/exav-unpack/src/formats/tnef.rs @@ -42,32 +42,16 @@ pub(crate) fn stream_offsets( let len = source .seek(SeekFrom::End(0)) .map_err(|e| LimitHit::corrupt(format!("tnef: {e}")))?; - let mut sig = [0u8; 6]; - if source - .seek(SeekFrom::Start(0)) - .and_then(|_| source.read_exact(&mut sig)) - .is_err() - || sig[0..4] != TNEF_SIGNATURE_LE - { + let sig = crate::read_at(source, 0, 6)?; + if sig.len() < 6 || sig[0..4] != TNEF_SIGNATURE_LE { return Ok(Vec::new()); } let mut out = Vec::new(); let mut name: Option = None; let mut pos = 6u64; while pos < len { - if source.seek(SeekFrom::Start(pos)).is_err() { - break; - } - let mut hdr = [0u8; 9]; // level(1) + type_and_id(4) + length(4) - let mut got = 0; - while got < 9 { - match source.read(&mut hdr[got..]) { - Ok(0) => break, - Ok(k) => got += k, - Err(_) => break, - } - } - if got < 9 || hdr[0] == 0 { + let hdr = crate::read_at(source, pos, 9)?; // level(1) + type_and_id(4) + length(4) + if hdr.len() < 9 || hdr[0] == 0 { break; } let level = hdr[0]; @@ -84,18 +68,7 @@ pub(crate) fn stream_offsets( match tag { ATT_ATTACHTITLE => { let n = clamped.min(max_buffer) as usize; - let mut buf = vec![0u8; n]; - if source.seek(SeekFrom::Start(data_start)).is_ok() { - let mut g = 0; - while g < buf.len() { - match source.read(&mut buf[g..]) { - Ok(0) => break, - Ok(k) => g += k, - Err(_) => break, - } - } - buf.truncate(g); - } + let buf = crate::read_at(source, data_start, n)?; name = Some(attachment_name(&buf)); } ATT_ATTACHDATA | ATT_ATTACHMENT => { diff --git a/crates/exav-unpack/src/formats/zip.rs b/crates/exav-unpack/src/formats/zip.rs index 4225d20c..c9fbaf3c 100644 --- a/crates/exav-unpack/src/formats/zip.rs +++ b/crates/exav-unpack/src/formats/zip.rs @@ -521,6 +521,73 @@ pub fn overlapping_local_records(data: &[u8]) -> usize { overlapping } +/// Whether `data` holds a ZIP with a consistent central directory: an +/// end-of-central-directory record, and every directory entry pointing at a +/// local header with the same name. Offsets may count from the archive or from +/// an earlier start, as they do in an archive appended to another file. ZIP64 +/// is not followed, so answers `false`. +pub fn directory_is_consistent(data: &[u8]) -> bool { + const EOCD_LEN: usize = 22; + const CDH_LEN: usize = 46; + let u16_at = |p: usize| { + data.get(p..p + 2) + .map(|b| usize::from(u16::from_le_bytes([b[0], b[1]]))) + }; + let u32_at = |p: usize| { + data.get(p..p + 4) + .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize) + }; + let tail = data.len().saturating_sub(EOCD_LEN + usize::from(u16::MAX)); + let Some(eocd) = data[tail..] + .windows(4) + .rposition(|w| w == b"PK\x05\x06") + .map(|p| tail + p) + else { + return false; + }; + let (Some(entries), Some(cd_size), Some(cd_off)) = + (u16_at(eocd + 10), u32_at(eocd + 12), u32_at(eocd + 16)) + else { + return false; + }; + if entries == 0 || entries == 0xffff || cd_size == 0xffff_ffff || cd_off == 0xffff_ffff { + return false; + } + let Some(cd_start) = eocd.checked_sub(cd_size) else { + return false; + }; + // Where offset 0 of the archive's own numbering sits in `data`. + let base = cd_start as i64 - cd_off as i64; + let mut p = cd_start; + for _ in 0..entries { + if data.get(p..p + 4) != Some(b"PK\x01\x02") { + return false; + } + let (Some(n), Some(e), Some(c), Some(local)) = ( + u16_at(p + 28), + u16_at(p + 30), + u16_at(p + 32), + u32_at(p + 42), + ) else { + return false; + }; + let Some(name) = data.get(p + CDH_LEN..p + CDH_LEN + n) else { + return false; + }; + let Ok(lo) = usize::try_from(base + local as i64) else { + return false; + }; + if data.get(lo..lo + 4) != Some(b"PK\x03\x04") + || u16_at(lo + 26) != Some(n) + || data.get(lo + LFH_LEN..lo + LFH_LEN + n) != Some(name) + { + return false; + } + p += CDH_LEN + n + e + c; + } + p == eocd +} + /// Does the `PK\x03\x04` at `off` look like a genuine local file header rather /// than a chance byte sequence? Checks the fields a real writer must fill in /// consistently; deliberately strict, because every false positive here becomes @@ -632,21 +699,10 @@ fn encryption_flag_is_a_lie( let plain = match method { 0 => raw.to_vec(), 8 => { - let mut out = Vec::new(); - if bounded_read_salvage( - flate2::read::DeflateDecoder::new(Cursor::new(raw)), - PROBE_MAX as u64, - true, - ) - .map(|(o, truncated)| { - out = o; - truncated - }) - .unwrap_or(true) - { - return false; + match bounded_read_salvage(crate::inflate::Inflate::new(raw), PROBE_MAX as u64, true) { + Ok(s) if !s.over_cap => s.data, + _ => return false, } - out } // Anything else we would not decode even in the clear. _ => return false, @@ -775,7 +831,7 @@ fn parse_local_member( }; budget.count_entry()?; let cap = budget.reserve()?; - let out = match method { + let (out, part_way) = match method { // Stored. The deflate arm below already refuses to hand back a prefix; // this one clamped to the cap and returned it as a complete member, so // an oversized stored orphan was silently truncated. Same treatment. @@ -787,23 +843,19 @@ fn parse_local_member( "orphan zip member exceeds size budget", ))) } - 0 => raw.get(..comp).unwrap_or(raw).to_vec(), + 0 => (raw.get(..comp).unwrap_or(raw).to_vec(), false), 8 => { // Salvage the bytes decoded before any corruption rather than // dropping the whole member: this is a best-effort recovery of a // malformed archive, and the payload a signature matches may sit in // the valid prefix (matching clamd, which scans partial inflate). - let (o, truncated) = bounded_read_salvage( - flate2::read::DeflateDecoder::new(Cursor::new(raw)), - cap, - true, - ) - .map_err(|e| LimitHit::corrupt(format!("orphan zip inflate: {e}")))?; + let s = bounded_read_salvage(crate::inflate::Inflate::new(raw), cap, true) + .map_err(|e| LimitHit::corrupt(format!("orphan zip inflate: {e}")))?; // Over the per-member cap. Yield metadata-only rather than the // prefix (which would read as a complete member) and rather than // aborting, so the remaining orphans are still scanned — the same // shape the central-directory path uses for an oversized member. - if truncated { + if s.over_cap { return Ok(Some(Entry::unsupported( name, comp as u64, @@ -811,10 +863,10 @@ fn parse_local_member( "orphan zip member exceeds size budget", ))); } - o + (s.data, s.undecoded) } _ => match decode_zip_raw(method, raw, usz, cap) { - Some((o, false)) => o, + Some((o, false)) => (o, false), // Either a codec exav has no decoder for, or one whose stream was // truncated. Both leave content unexamined. _ => { @@ -832,12 +884,39 @@ fn parse_local_member( Ok(Some(Entry { comp_size: comp as u64, encrypted: false, - unsupported: None, + unsupported: part_way.then_some(PART_WAY), name, data: out, })) } +/// Why a ZIP member's entry holds only part of its content. +const PART_WAY: &str = + "zip member failed to decode part way; the bytes before the failure were scanned"; + +/// A cleartext member's content. Stored and deflated members are decoded here +/// rather than by the crate: its deflate reader drops what it decoded in the +/// call that meets damage, and its CRC-32 failure is not told apart from +/// damage. Other codecs go through the crate as before. +pub(crate) fn member_reader<'a, R: Read + Seek>( + zip: &'a mut ::zip::ZipArchive, + i: usize, +) -> ::zip::result::ZipResult> { + use crate::inflate::{CrcCheck, Inflate}; + use ::zip::CompressionMethod as C; + let method = zip.by_index_raw(i)?.compression(); + if !matches!(method, C::Stored | C::Deflated) { + return Ok(Box::new(zip.by_index(i)?)); + } + let raw = zip.by_index_raw(i)?; + let crc = raw.crc32(); + Ok(if method == C::Stored { + Box::new(CrcCheck::new(raw, crc)) + } else { + Box::new(CrcCheck::new(Inflate::new(BufReader::new(raw)), crc)) + }) +} + /// Recover the compressed length of a member whose local header deferred its /// sizes to a trailing data descriptor (general-purpose flag bit 3). /// @@ -1076,12 +1155,12 @@ pub fn extract_zip_from( // unsupported method. A decode we can't do (or an unknown codec) yields a // metadata-only unsupported member and the archive keeps going — one bad // member never aborts the whole ZIP. - let (buf, truncated) = if method != 0 && method != 8 { + let (buf, truncated, part_way) = if method != 0 && method != 8 { let (raw, _) = bounded_read(&mut file, budget.limits.max_buffer_bytes) .map_err(|e| LimitHit::new(format!("zip raw read: {e}")))?; drop(file); match decode_zip_raw(method, &raw, usz, cap) { - Some(out) => out, + Some((out, truncated)) => (out, truncated, false), None => { if let Some(r) = oversized(budget, "archive member: unsupported ZIP codec") { return Ok(Some(r)); @@ -1090,10 +1169,10 @@ pub fn extract_zip_from( } } } else { - // Store/Deflate: re-open with the decompressing reader (the raw reader + // Store/Deflate: re-open with the decoding reader (the raw reader // returns still-compressed bytes). drop(file); - let mut file = match zip.by_index(i) { + let mut file = match member_reader(&mut zip, i) { Ok(f) => f, Err(_) => { if let Some(r) = oversized(budget, "archive member: ZIP decode failed") { @@ -1102,10 +1181,10 @@ pub fn extract_zip_from( continue; } }; - let r = bounded_read_salvage(&mut file, cap, !budget.should_verify_checksums()) + let s = bounded_read_salvage(&mut file, cap, !budget.should_verify_checksums()) .map_err(|e| LimitHit::new(format!("zip read: {e}")))?; drop(file); - r + (s.data, s.over_cap, s.undecoded) }; if truncated { if let Some(r) = oversized(budget, "archive member exceeds size budget") { @@ -1121,7 +1200,7 @@ pub fn extract_zip_from( let entry = Entry { comp_size: comp, encrypted: false, - unsupported: None, + unsupported: part_way.then_some(PART_WAY), name, data: buf, }; @@ -1324,6 +1403,42 @@ fn read_spans( /// gigabytes unclaimed would otherwise turn opening it into a full read. const MAX_ORPHAN_SCAN: u64 = 16 * 1024 * 1024; +/// The members `zip`'s central directory omits, decoded, for a walk that has +/// already been through the ones it lists. +/// +/// The streaming counterpart of the buffered path's orphan scan, over the same +/// bounded search [`ZipMembers`] uses: only the space the directory leaves +/// unaccounted for is read, up to [`MAX_ORPHAN_SCAN`], and a search that stops +/// there yields a member saying so. +pub(crate) fn hidden_members( + mut zip: ::zip::ZipArchive, + budget: &mut Budget, +) -> Result, LimitHit> { + let (gaps, known) = unclaimed_spans(&mut zip); + if gaps.is_empty() { + return Ok(Vec::new()); + } + let mut reader = zip.into_inner(); + let (unclaimed, truncated) = read_spans(&mut reader, &gaps, MAX_ORPHAN_SCAN)?; + let mut out = Vec::new(); + for (span, off) in find_orphans(&unclaimed, &known) { + if let Some(entry) = parse_local_member(&unclaimed[span].1, off, budget)? { + out.push(entry); + } + } + if truncated { + budget.count_entry()?; + out.push(Entry::unsupported( + ZipMembers::::TRUNCATED_MARKER.to_string(), + 0, + false, + "the search for members hidden from the central directory \ + reached its size limit with space left unexamined", + )); + } + Ok(out) +} + impl ZipMembers { /// Open a seekable ZIP (reads only the central directory, and whatever /// space that directory leaves unaccounted for). @@ -1594,7 +1709,7 @@ impl ZipMembers { // Cleartext member: re-open decompressing so `bounded_read` yields // the *decompressed* content. drop(file); - let mut file = self.zip.by_index(i).map_err(|e| zip_entry_error(i, &e))?; + let mut file = member_reader(&mut self.zip, i).map_err(|e| zip_entry_error(i, &e))?; // A member too large for the budget yields a metadata-only member // (Unscannable, empty data) rather than aborting the archive walk, // so name/size `.cdb` sigs still match and later members are scanned. @@ -1609,10 +1724,9 @@ impl ZipMembers { ))) } }; - let (buf, truncated) = - bounded_read_salvage(&mut file, cap, !budget.should_verify_checksums()) - .map_err(|e| LimitHit::new(format!("zip read: {e}")))?; - if truncated { + let s = bounded_read_salvage(&mut file, cap, !budget.should_verify_checksums()) + .map_err(|e| LimitHit::new(format!("zip read: {e}")))?; + if s.over_cap { return Ok(Some(Entry::unsupported( name, comp, @@ -1620,14 +1734,14 @@ impl ZipMembers { "archive member exceeds size budget", ))); } - ratio_guard(comp, buf.len() as u64, budget)?; - budget.commit(buf.len() as u64); + ratio_guard(comp, s.data.len() as u64, budget)?; + budget.commit(s.data.len() as u64); Ok(Some(Entry { comp_size: comp, encrypted: false, - unsupported: None, + unsupported: s.undecoded.then_some(PART_WAY), name, - data: buf, + data: s.data, })) } } @@ -1671,6 +1785,63 @@ mod orphan_scan_tests { } } +#[cfg(test)] +mod directory_tests { + use super::*; + use std::io::Write; + + fn two_member_zip() -> Vec { + let mut z = ::zip::ZipWriter::new(Cursor::new(Vec::new())); + for name in ["a.txt", "b.txt"] { + z.start_file(name, ::zip::write::SimpleFileOptions::default()) + .unwrap(); + z.write_all(b"hello").unwrap(); + } + z.finish().unwrap().into_inner() + } + + /// Add `by` to the directory offset and to every entry's local offset, + /// as `zip -A` does after prepending `by` bytes. + fn shift_offsets(zip: &mut [u8], by: u32) { + let eocd = zip.windows(4).rposition(|w| w == b"PK\x05\x06").unwrap(); + let entries: Vec = zip + .windows(4) + .enumerate() + .filter(|(_, w)| *w == b"PK\x01\x02") + .map(|(p, _)| p + 42) + .collect(); + for p in std::iter::once(eocd + 16).chain(entries) { + let v = u32::from_le_bytes(zip[p..p + 4].try_into().unwrap()) + by; + zip[p..p + 4].copy_from_slice(&v.to_le_bytes()); + } + } + + #[test] + fn an_appended_archive_is_consistent_under_either_numbering() { + let zip = two_member_zip(); + assert!(directory_is_consistent(&zip)); + let prefix = [0x42u8; 100]; + assert!(directory_is_consistent(&[&prefix[..], &zip[..]].concat())); + let mut shifted = zip.clone(); + shift_offsets(&mut shifted, 100); + assert!(directory_is_consistent( + &[&prefix[..], &shifted[..]].concat() + )); + } + + #[test] + fn a_directory_that_does_not_match_its_members_is_not_consistent() { + let zip = two_member_zip(); + let cd = zip.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + assert!(!directory_is_consistent(&zip[..cd])); + let mut renamed = zip.clone(); + renamed[cd + 46] = b'z'; + assert!(!directory_is_consistent(&renamed)); + assert!(!directory_is_consistent(b"xxPK\x03\x04garbage")); + assert!(!directory_is_consistent(b"")); + } +} + #[cfg(test)] mod zip_ppmd_tests { use super::*; diff --git a/crates/exav-unpack/src/inflate.rs b/crates/exav-unpack/src/inflate.rs new file mode 100644 index 00000000..d61e29c1 --- /dev/null +++ b/crates/exav-unpack/src/inflate.rs @@ -0,0 +1,415 @@ +//! Deflate decoding that keeps what it decoded before an error, and tells a +//! failed checksum from damaged data. +//! +//! flate2's `Read` decoders drop the output of the call that fails, which can +//! be a whole read buffer of content sitting right before the damage. + +// Each format uses a part of this module; see `cap_prealloc` for why the +// feature lists are not spelled out. +#![allow(dead_code)] + +use std::io::{self, BufRead, Read}; + +use flate2::{Crc, Decompress, FlushDecompress, Status}; + +use crate::checksum_mismatch; + +/// Raw deflate over a buffered source. +pub(crate) struct Inflate { + src: R, + state: Decompress, + /// Decoded bytes not yet handed over, and how many of them were. + out: Vec, + given: usize, + /// Why decoding stopped, once the bytes decoded before it are handed over. + broken: Option, + ended: bool, +} + +/// Room for a whole deflate window per call. miniz_oxide decodes into its +/// 32 KiB window and, on failure, copies out only what the caller's buffer +/// holds; every later call fails at once, so the rest would be lost. +const OUT: usize = 64 * 1024; + +impl Inflate { + pub(crate) fn new(src: R) -> Self { + Self { + src, + state: Decompress::new(false), + out: Vec::new(), + given: 0, + broken: None, + ended: false, + } + } + + /// The source, positioned right after the deflate data once it has ended. + fn source(&mut self) -> &mut R { + &mut self.src + } + + /// Start over on a new deflate stream from the current source position. + fn restart(&mut self) { + self.state.reset(false); + self.out.clear(); + self.given = 0; + self.ended = false; + } +} + +impl Inflate { + /// Decode the next run of output into `self.out`. `Ok(false)` once the + /// stream has ended. + fn fill(&mut self) -> io::Result { + if let Some(why) = &self.broken { + return Err(io::Error::new(io::ErrorKind::InvalidInput, why.clone())); + } + if self.ended { + return Ok(false); + } + self.out.resize(OUT, 0); + loop { + let input = self.src.fill_buf()?; + let eof = input.is_empty(); + let (in0, out0) = (self.state.total_in(), self.state.total_out()); + let flush = if eof { + FlushDecompress::Finish + } else { + FlushDecompress::None + }; + let res = self.state.decompress(input, &mut self.out, flush); + let used = (self.state.total_in() - in0) as usize; + let made = (self.state.total_out() - out0) as usize; + self.src.consume(used); + self.out.truncate(made); + self.given = 0; + match res { + Ok(Status::StreamEnd) => { + self.ended = true; + return Ok(made > 0); + } + Ok(_) if made > 0 => return Ok(true), + Ok(_) if eof => { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "deflate stream ends before its last block", + )) + } + Ok(_) => self.out.resize(OUT, 0), + Err(e) => { + let why = format!("corrupt deflate stream: {e}"); + self.broken = Some(why.clone()); + if made > 0 { + return Ok(true); + } + return Err(io::Error::new(io::ErrorKind::InvalidInput, why)); + } + } + } + } +} + +impl Read for Inflate { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + if self.given == self.out.len() && !self.fill()? { + return Ok(0); + } + let n = buf.len().min(self.out.len() - self.given); + buf[..n].copy_from_slice(&self.out[self.given..self.given + n]); + self.given += n; + Ok(n) + } +} + +/// The deflate data of a zlib stream (RFC 1950), or `None` when its header is +/// not one. The trailing Adler-32 is not checked: a mismatch after a full +/// decode hides nothing. +pub(crate) fn zlib_body(data: &[u8]) -> Option> { + let [cmf, flg, ..] = *data else { + return None; + }; + let deflate = cmf & 0x0f == 8 && cmf >> 4 <= 7; + let preset_dictionary = flg & 0x20 != 0; + let check = (u16::from(cmf) << 8 | u16::from(flg)) % 31 == 0; + (deflate && check && !preset_dictionary).then(|| Inflate::new(&data[2..])) +} + +/// A gzip file (RFC 1952), every member in turn, with each trailer checked +/// here so a CRC-32 or size mismatch after a full decode is reported as a +/// checksum mismatch rather than as damage. It is reported once every member +/// has been decoded, so one bad trailer does not cost the members after it. +/// Bytes after the last member that do not start another one are ignored, as +/// `gzip -d` does. +pub(crate) struct Gunzip { + body: Inflate, + crc: Crc, + state: GzState, + members: u32, + bad_trailer: bool, +} + +#[derive(Clone, Copy, PartialEq)] +enum GzState { + Header, + Body, + Done, +} + +impl Gunzip { + pub(crate) fn new(src: R) -> Self { + Self { + body: Inflate::new(src), + crc: Crc::new(), + state: GzState::Header, + members: 0, + bad_trailer: false, + } + } + + /// Read a member header. `false` when the source holds no further member. + fn header(&mut self) -> io::Result { + let first = self.members == 0; + let src = self.body.source(); + for magic in [0x1f, 0x8b] { + let next = src.fill_buf()?.first().copied(); + match next { + Some(b) if b == magic => src.consume(1), + _ if !first => return Ok(false), + None => return Err(io::ErrorKind::UnexpectedEof.into()), + Some(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "not a gzip stream", + )) + } + } + } + let mut fixed = [0u8; 8]; + src.read_exact(&mut fixed)?; + let [method, flags, ..] = fixed; + if method != 8 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("gzip member uses compression method {method}"), + )); + } + if flags & 0x04 != 0 { + let mut len = [0u8; 2]; + src.read_exact(&mut len)?; + skip(src, u16::from_le_bytes(len).into())?; + } + for field in [0x08, 0x10] { + if flags & field != 0 { + skip_string(src)?; + } + } + if flags & 0x02 != 0 { + skip(src, 2)?; + } + Ok(true) + } + + fn trailer(&mut self) -> io::Result<()> { + let mut t = [0u8; 8]; + self.body.source().read_exact(&mut t)?; + let crc = u32::from_le_bytes([t[0], t[1], t[2], t[3]]); + let size = u32::from_le_bytes([t[4], t[5], t[6], t[7]]); + if crc != self.crc.sum() || size != self.crc.amount() { + self.bad_trailer = true; + } + Ok(()) + } +} + +impl Read for Gunzip { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + loop { + match self.state { + GzState::Done => return Ok(0), + GzState::Header => { + if self.header()? { + self.body.restart(); + self.crc.reset(); + self.members += 1; + self.state = GzState::Body; + } else { + self.state = GzState::Done; + if self.bad_trailer { + return Err(checksum_mismatch("gzip CRC-32 or size")); + } + } + } + GzState::Body => { + let n = self.body.read(buf)?; + if n > 0 { + self.crc.update(&buf[..n]); + return Ok(n); + } + // `Inflate` returns 0 only once its stream has ended. + self.trailer()?; + self.state = GzState::Header; + } + } + } + } +} + +/// Wraps a member's decoded bytes, which must have CRC-32 `expected`; a +/// mismatch at the end is a checksum mismatch. +pub(crate) struct CrcCheck { + inner: R, + crc: Crc, + expected: u32, + reported: bool, +} + +impl CrcCheck { + pub(crate) fn new(inner: R, expected: u32) -> Self { + Self { + inner, + crc: Crc::new(), + expected, + reported: false, + } + } +} + +impl Read for CrcCheck { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let n = self.inner.read(buf)?; + if n == 0 && !buf.is_empty() && !self.reported && self.crc.sum() != self.expected { + self.reported = true; + return Err(checksum_mismatch("zip CRC-32")); + } + self.crc.update(&buf[..n]); + Ok(n) + } +} + +fn skip(src: &mut R, n: u64) -> io::Result<()> { + if io::copy(&mut src.take(n), &mut io::sink())? < n { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + Ok(()) +} + +/// Skip a NUL-terminated header field without holding it. +fn skip_string(src: &mut R) -> io::Result<()> { + loop { + let buf = src.fill_buf()?; + if buf.is_empty() { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + match buf.iter().position(|&b| b == 0) { + Some(i) => { + src.consume(i + 1); + return Ok(()); + } + None => { + let n = buf.len(); + src.consume(n); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn deflate_broken_after(prefix: &[u8]) -> Vec { + let mut e = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(prefix).unwrap(); + e.flush().unwrap(); + let mut raw = e.get_ref().clone(); + raw.push(0x06); + raw.extend_from_slice(&[0x5a; 64]); + raw + } + + fn gzip(payload: &[u8]) -> Vec { + let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(payload).unwrap(); + e.finish().unwrap() + } + + fn read_all(mut r: impl Read) -> (Vec, Option) { + let mut out = Vec::new(); + let mut chunk = [0u8; 8192]; + loop { + match r.read(&mut chunk) { + Ok(0) => return (out, None), + Ok(n) => out.extend_from_slice(&chunk[..n]), + Err(e) => return (out, Some(e)), + } + } + } + + /// Everything decoded before the damage is handed over, then the error. + #[test] + fn the_bytes_before_the_damage_are_kept() { + let prefix: Vec = (0..20_000u32).flat_map(|i| i.to_le_bytes()).collect(); + let (out, err) = read_all(Inflate::new(&deflate_broken_after(&prefix)[..])); + assert_eq!(out, prefix); + assert!(crate::decode_error_hides_content(&err.unwrap())); + } + + #[test] + fn a_truncated_stream_is_not_damage() { + let full = gzip(&[7u8; 50_000]); + let (_, err) = read_all(Gunzip::new(&full[..full.len() / 2])); + assert!(!crate::decode_error_hides_content(&err.unwrap())); + } + + /// Every member is decoded before a bad trailer is reported, and the + /// report is a checksum mismatch. + #[test] + fn a_bad_trailer_costs_no_member() { + let mut first = gzip(b"first "); + let n = first.len(); + first[n - 8] ^= 1; + let mut file = first; + file.extend(gzip(b"second")); + file.extend_from_slice(&[0; 32]); + let (out, err) = read_all(Gunzip::new(&file[..])); + assert_eq!(out, b"first second"); + assert!(!crate::decode_error_hides_content(&err.unwrap())); + + let (out, err) = read_all(Gunzip::new(&gzip(b"intact")[..])); + assert_eq!(out, b"intact"); + assert!(err.is_none()); + } + + #[test] + fn header_fields_are_skipped() { + let mut e = flate2::GzBuilder::new() + .filename("name.txt") + .comment("a comment") + .extra(vec![1, 2, 3]) + .write(Vec::new(), flate2::Compression::default()); + e.write_all(b"payload").unwrap(); + let (out, err) = read_all(Gunzip::new(&e.finish().unwrap()[..])); + assert_eq!(out, b"payload"); + assert!(err.is_none()); + } + + #[test] + fn a_zlib_body_decodes_without_its_checksum() { + let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(b"zlib payload").unwrap(); + let mut z = e.finish().unwrap(); + let n = z.len(); + z[n - 1] ^= 1; + let (out, err) = read_all(zlib_body(&z).unwrap()); + assert_eq!(out, b"zlib payload"); + assert!(err.is_none()); + assert!(zlib_body(b"not zlib").is_none()); + } +} diff --git a/crates/exav-unpack/src/lib.rs b/crates/exav-unpack/src/lib.rs index 30c21aef..fa6c4cc7 100644 --- a/crates/exav-unpack/src/lib.rs +++ b/crates/exav-unpack/src/lib.rs @@ -65,8 +65,10 @@ pub(crate) mod formats; #[allow(unused_imports)] use formats::*; -mod stream; +#[cfg(any(feature = "gzip", feature = "zip", feature = "pdf", feature = "ole"))] +mod inflate; pub mod profile; +mod stream; pub mod volume; #[cfg(feature = "pdf")] pub use formats::has_obfuscated_name_object; @@ -86,6 +88,15 @@ pub fn overlapping_local_records(_data: &[u8]) -> usize { 0 } +#[cfg(feature = "zip")] +pub use formats::zip::directory_is_consistent as zip_directory_is_consistent; + +/// Without the `zip` feature no ZIP is ever confirmed. +#[cfg(not(feature = "zip"))] +pub fn zip_directory_is_consistent(_data: &[u8]) -> bool { + false +} + /// The dictionary size an XZ stream declares, and the largest exav will /// allocate. A declaration above the cap is ClamAV's /// `Heuristics.XZ.DicSizeLimit`: memory a decoder must commit before producing @@ -194,6 +205,13 @@ pub struct Limits { /// large file. Deterministic, so it trips identically on every machine /// (unlike a wall-clock deadline). pub max_scanned_bytes: u64, + /// Cap on the instructions the PE stub emulator may run across the whole + /// recursive analysis of one top-level file. + /// + /// Each emulation already has its own instruction budget; this bounds their + /// sum, which otherwise grows with the number of packed executables an + /// archive carries. A run cut short by it is `LimitsExceeded`. + pub max_pe_emulation_steps: u64, /// Formats this scan will open, or `None` for every format compiled in. /// /// A compile-time feature decides what a *binary* can do; this decides what @@ -235,6 +253,9 @@ impl Default for Limits { // and runaway scan time, and lets a multi-gigabyte member be fully // scanned. max_scanned_bytes: 10 * 1024 * 1024 * 1024, + // A few seconds to tens of seconds of emulation per top-level file, + // against the per-stub budget of `formats::pepack`. + max_pe_emulation_steps: 1_000_000_000, // Every format the build was compiled with. Narrowing this is a // deployment decision, and a default that narrowed it would hide // content from callers who never asked for that. @@ -258,6 +279,8 @@ pub struct Budget { pub(crate) total_out: u64, /// Cumulative bytes fed to the matching core (see [`Limits::max_scanned_bytes`]). pub(crate) scanned: u64, + /// Emulator instructions spent so far (see [`Limits::max_pe_emulation_steps`]). + pub(crate) pe_emulation_steps: u64, /// Candidate passwords tried (in order) when decrypting an encrypted member /// (ZIP ZipCrypto/AES today). Empty by default — an encrypted member with no /// password yields an `Entry::unsupported(encrypted=true, …)` so the scanner @@ -286,6 +309,7 @@ impl Budget { files: 0, total_out: 0, scanned: 0, + pe_emulation_steps: 0, passwords: Vec::new(), verify_checksums: false, } @@ -374,6 +398,30 @@ impl Budget { pub fn commit(&mut self, n: u64) { self.total_out = self.total_out.saturating_add(n); } + + /// Emulator instructions still available to this scan. + #[cfg_attr(not(feature = "pe-emu"), allow(dead_code))] + pub(crate) fn pe_emulation_room(&self) -> u64 { + self.limits + .max_pe_emulation_steps + .saturating_sub(self.pe_emulation_steps) + } + + #[cfg_attr(not(feature = "pe-emu"), allow(dead_code))] + pub(crate) fn charge_pe_emulation(&mut self, steps: u64) { + self.pe_emulation_steps = self.pe_emulation_steps.saturating_add(steps); + } + + #[cfg_attr(not(feature = "pe-emu"), allow(dead_code))] + pub(crate) fn pe_emulation_exhausted(&self) -> LimitHit { + LimitHit::of_kind( + LimitKind::MaxScanTime, + format!( + "PE emulation steps > {} (--max-pe-emulation-steps)", + self.limits.max_pe_emulation_steps + ), + ) + } } /// An extraction stopped early. [`LimitHit::is_corrupt`] picks the verdict: @@ -408,6 +456,8 @@ pub enum LimitKind { MaxFiles, /// Container nesting depth (`max_recursion`). MaxRecursion, + /// CPU work: emulator instructions across the scan (`max_pe_emulation_steps`). + MaxScanTime, /// Not a budget — the input was malformed. Kept in the same enum so every /// `LimitHit` has a kind and nothing has to guess. Corrupt, @@ -443,6 +493,41 @@ impl LimitHit { } } +/// Read until `buf` is full or the source ends, and return how much was read. +/// +/// A read error is reported, never taken for the end: from a source that can +/// fail part way (a network range reader) it would pass for a short, complete +/// container, and every member past it would go unseen. +#[allow(dead_code)] +pub(crate) fn read_full(src: &mut R, buf: &mut [u8]) -> Result { + let mut n = 0; + while n < buf.len() { + match src.read(&mut buf[n..]) { + Ok(0) => break, + Ok(k) => n += k, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return Err(LimitHit::corrupt(format!("read failed after {n} B: {e}"))), + } + } + Ok(n) +} + +/// Up to `len` bytes at `off`; fewer only where the source ends. See +/// [`read_full`] for why a failure is an error. +#[allow(dead_code)] +pub(crate) fn read_at( + src: &mut R, + off: u64, + len: usize, +) -> Result, LimitHit> { + src.seek(std::io::SeekFrom::Start(off)) + .map_err(|e| LimitHit::corrupt(format!("seek to {off} failed: {e}")))?; + let mut buf = vec![0u8; len]; + let n = read_full(src, &mut buf)?; + buf.truncate(n); + Ok(buf) +} + /// One extracted member. #[derive(Debug, Clone, PartialEq)] pub struct Entry { @@ -1881,12 +1966,52 @@ pub(crate) fn bounded_read(mut r: R, cap: u64) -> Result<(Vec, bool Ok((buf, truncated)) } +/// A decode error raised after the input was decoded in full, when only its +/// checksum disagreed. Carried inside the `io::Error` so a caller can tell it +/// from damage, which leaves content undecoded. +#[derive(Debug)] +struct ChecksumMismatch(&'static str); + +impl std::fmt::Display for ChecksumMismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} mismatch after a full decode", self.0) + } +} + +impl std::error::Error for ChecksumMismatch {} + +#[allow(dead_code)] // see `bounded_read_salvage` +pub(crate) fn checksum_mismatch(what: &'static str) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidData, ChecksumMismatch(what)) +} + +/// Whether a decode error left compressed content in the input that was never +/// decoded. A stream that ends early (the rest is absent) or that decoded in +/// full and failed its checksum hides nothing; damage part way does. +pub fn decode_error_hides_content(e: &std::io::Error) -> bool { + e.kind() != std::io::ErrorKind::UnexpectedEof + && !e + .get_ref() + .is_some_and(|inner| inner.is::()) +} + +/// What [`bounded_read_salvage`] recovered. +#[allow(dead_code)] // see `bounded_read_salvage` +pub(crate) struct Salvaged { + pub(crate) data: Vec, + /// The source had more than `cap` bytes. + pub(crate) over_cap: bool, + /// A decode error stopped the read with content left undecoded + /// ([`decode_error_hides_content`]). + pub(crate) undecoded: bool, +} + /// Like [`bounded_read`], but when `salvage` is set, a read error does not -/// discard the bytes decoded so far — it returns them. This is the -/// scan-everything default: a trailing checksum/integrity error from a -/// decompressor (e.g. a gzip CRC-32 or ISIZE mismatch, a ZIP CRC) must not throw -/// away already-decompressed content that a scanner still needs to inspect. With -/// `salvage` false it is exactly [`bounded_read`] (errors propagate). +/// discard the bytes decoded so far: it returns them, and says whether the +/// error left content undecoded. This is the scan-everything default: the +/// prefix is scanned, and a clean prefix of a damaged stream is not reported +/// as a clean member. With `salvage` false errors propagate, as in +/// [`bounded_read`]. // Dead only in a build with none of the formats that salvage a partial member; // see `cap_prealloc` for why the feature list is not spelled out. #[allow(dead_code)] @@ -1894,28 +2019,39 @@ pub(crate) fn bounded_read_salvage( mut r: R, cap: u64, salvage: bool, -) -> Result<(Vec, bool), std::io::Error> { +) -> Result { if !salvage { - return bounded_read(r, cap); + let (data, over_cap) = bounded_read(r, cap)?; + return Ok(Salvaged { + data, + over_cap, + undecoded: false, + }); } let limit = cap.saturating_add(1); let mut buf = Vec::new(); let mut chunk = [0u8; 8192]; + let mut undecoded = false; while (buf.len() as u64) < limit { match r.read(&mut chunk) { Ok(0) => break, Ok(n) => buf.extend_from_slice(&chunk[..n]), Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, - // Salvage: keep the content decoded before the (likely checksum) - // error rather than dropping the whole member. - Err(_) => break, + Err(e) => { + undecoded = decode_error_hides_content(&e); + break; + } } } - let truncated = buf.len() as u64 > cap; - if truncated { + let over_cap = buf.len() as u64 > cap; + if over_cap { buf.truncate(cap as usize); } - Ok((buf, truncated)) + Ok(Salvaged { + data: buf, + over_cap, + undecoded, + }) } // --------------------------------------------------------------------------- @@ -2280,19 +2416,14 @@ impl Archive { .ok_or_else(|| LimitHit::corrupt("gzip: already extracted".into()))?; budget.count_entry()?; let cap = budget.reserve()?; - use flate2::read::MultiGzDecoder; - let (out, truncated) = bounded_read_salvage( - MultiGzDecoder::new(r), - cap, - !budget.should_verify_checksums(), - ) - .map_err(|e| LimitHit::corrupt(format!("gzip: {e}")))?; - if truncated { + let s = gunzip(std::io::BufReader::new(r), cap, budget) + .map_err(|e| LimitHit::corrupt(format!("gzip: {e}")))?; + if s.over_cap { return Err(LimitHit::new("gzip member exceeds budget".to_string())); } - budget.commit(out.len() as u64); + budget.commit(s.data.len() as u64); *done = true; - Ok(Some(Entry::new("gzip-content".to_string(), out))) + Ok(Some(gzip_entry(s))) } #[cfg(feature = "tar")] ArchiveInner::Tar { @@ -2412,19 +2543,14 @@ impl Archive { .ok_or_else(|| LimitHit::corrupt("gzip: already extracted".into()))?; budget.count_entry()?; let cap = budget.reserve()?; - use flate2::read::MultiGzDecoder; - let (out, truncated) = bounded_read_salvage( - MultiGzDecoder::new(r), - cap, - !budget.should_verify_checksums(), - ) - .map_err(|e| LimitHit::corrupt(format!("gzip: {e}")))?; - if truncated { + let s = gunzip(std::io::BufReader::new(r), cap, budget) + .map_err(|e| LimitHit::corrupt(format!("gzip: {e}")))?; + if s.over_cap { return Err(LimitHit::new("gzip member exceeds budget".to_string())); } - budget.commit(out.len() as u64); + budget.commit(s.data.len() as u64); *done = true; - Ok(Entry::new("gzip-content".to_string(), out)) + Ok(gzip_entry(s)) } #[cfg(feature = "tar")] ArchiveInner::Tar { diff --git a/crates/exav-unpack/src/stream.rs b/crates/exav-unpack/src/stream.rs index 96fde45b..8bde686f 100644 --- a/crates/exav-unpack/src/stream.rs +++ b/crates/exav-unpack/src/stream.rs @@ -307,10 +307,22 @@ fn dispatch_stream( .map(|d| Box::new(d) as Box) .map_err(|e| LimitHit::corrupt(format!("zstd: {e}"))) }), + // `LzipReader` takes any failure to read a member header, an I/O error + // included, for the end of the stream, so errors are caught underneath. #[cfg(feature = "lzip")] - Format::Lzip => stream_single(&mut source, budget, visit, "lzip-content", |r| { - Ok(Box::new(lzma_rust2::LzipReader::new(r)) as Box) - }), + Format::Lzip => { + let mut src = ErrorLatch { + inner: &mut source, + failed: None, + }; + let out = stream_single(&mut src, budget, visit, "lzip-content", |r| { + Ok(Box::new(lzma_rust2::LzipReader::new(r)) as Box) + })?; + match (out, src.failed) { + (None, Some(e)) => Err(LimitHit::corrupt(format!("lzip: read failed: {e}"))), + (out, _) => Ok(out), + } + } #[cfg(feature = "lha")] Format::Lha => stream_lha(&mut source, budget, visit), #[cfg(feature = "ar")] @@ -401,6 +413,32 @@ fn stream_single( visit_member(&meta, &mut *dec, budget, visit) } +/// Passes reads and seeks through, keeping the first read error for a caller +/// whose decoder does not report it. +#[cfg(feature = "lzip")] +struct ErrorLatch { + inner: R, + failed: Option, +} + +#[cfg(feature = "lzip")] +impl Read for ErrorLatch { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf).inspect_err(|e| { + if e.kind() != io::ErrorKind::Interrupted && self.failed.is_none() { + self.failed = Some(io::Error::new(e.kind(), e.to_string())); + } + }) + } +} + +#[cfg(feature = "lzip")] +impl Seek for ErrorLatch { + fn seek(&mut self, to: io::SeekFrom) -> io::Result { + self.inner.seek(to) + } +} + /// Run one member's reader through the visitor under a [`BudgetReader`], mapping /// an over-budget overflow to a [`LimitHit`]. /// @@ -438,12 +476,11 @@ fn stream_gzip( budget: &mut Budget, visit: StreamVisit, ) -> Result, LimitHit> { - use flate2::read::MultiGzDecoder; source .seek(io::SeekFrom::Start(0)) .map_err(|e| LimitHit::corrupt(format!("gzip seek: {e}")))?; budget.count_entry()?; - let mut dec = MultiGzDecoder::new(source); + let mut dec = crate::inflate::Gunzip::new(io::BufReader::new(source)); let meta = MemberMeta { name: "gzip-content".to_string(), comp_size: 0, @@ -505,25 +542,8 @@ fn stream_zip( None }, )?; - for entry in entries { - budget.count_entry()?; - let meta = MemberMeta { - name: entry.name, - comp_size: entry.comp_size, - encrypted: entry.encrypted, - unsupported: entry.unsupported, - }; - let hit = if entry.unsupported.is_some() { - visit(&meta, None, budget) - } else { - let mut cur = io::Cursor::new(&entry.data[..]); - visit_member(&meta, &mut cur, budget, visit)? - }; - if let Some(t) = hit { - return Ok(Some(t)); - } - } - return Ok(None); + // `extract_zip` already counted each of these toward `max_members`. + return visit_entries(entries, budget, visit); } }; for i in 0..zip.len() { @@ -588,21 +608,10 @@ fn stream_zip( // Probe first so the borrow of `zip` ends before the fallback needs it. let decodable = zip.by_index(i).is_ok(); let mut file = match decodable { - true => zip - .by_index(i) + true => crate::formats::zip::member_reader(&mut zip, i) .map_err(|e| crate::formats::zip::zip_entry_error(i, &e))?, false => { - if let Some(t) = stream_zip_raw_decode(&mut zip, i, &name, comp, budget, visit)? { - return Ok(Some(t)); - } - // Nothing we can decode either: surface it, never drop it. - let meta = MemberMeta { - name, - comp_size: comp, - encrypted: false, - unsupported: Some("unsupported zip compression method"), - }; - if let Some(t) = visit(&meta, None, budget) { + if let Some(t) = stream_zip_raw_decode(&mut zip, i, name, comp, budget, visit)? { return Ok(Some(t)); } continue; @@ -618,48 +627,92 @@ fn stream_zip( return Ok(Some(t)); } } + // Members left out of the central directory while their local headers and + // data stay in the file, which is what the target extracts. The buffered + // extractor has always looked for them; this is the walk every scanned ZIP + // takes, so without it a hidden member is never read. + let hidden = crate::formats::zip::hidden_members(zip, budget)?; + visit_entries(hidden, budget, visit) +} + +/// Hand already-decoded members to a streaming visitor, stopping when it does. +#[cfg(feature = "zip")] +fn visit_entries( + entries: Vec, + budget: &mut Budget, + visit: StreamVisit, +) -> Result, LimitHit> { + for entry in entries { + let meta = MemberMeta { + name: entry.name, + comp_size: entry.comp_size, + encrypted: entry.encrypted, + unsupported: entry.unsupported, + }; + let hit = if entry.unsupported.is_some() { + visit(&meta, None, budget) + } else { + let mut cur = io::Cursor::new(&entry.data[..]); + visit_member(&meta, &mut cur, budget, visit)? + }; + if let Some(t) = hit { + return Ok(Some(t)); + } + } Ok(None) } /// Decode a member the `zip` crate refused, from its RAW bytes, using exav's own -/// codec set. Returns `Ok(None)` when we cannot decode it either. +/// codec set. The member is visited once: decoded, or as a marker saying why +/// it could not be. Too large for the per-object buffer is a limit. #[cfg(feature = "zip")] fn stream_zip_raw_decode( zip: &mut ::zip::ZipArchive, i: usize, - name: &str, + name: String, comp: u64, budget: &mut Budget, visit: StreamVisit, ) -> Result, LimitHit> { let cap = budget.limits.max_buffer_bytes; - let (method, usz, raw) = { - let Ok(f) = zip.by_index_raw(i) else { - return Ok(None); - }; - let method = crate::formats::zip::zip_method_code(&f.compression()); - let usz = f.size(); - let mut raw = Vec::new(); - if f.take(cap).read_to_end(&mut raw).is_err() { - return Ok(None); - } - (method, usz, raw) - }; - let Some((out, truncated)) = crate::formats::zip::decode_zip_raw(method, &raw, usz, cap) else { - return Ok(None); - }; - if truncated { - return Ok(None); - } - budget.commit(out.len() as u64); - let meta = MemberMeta { - name: name.to_string(), + let mut meta = MemberMeta { + name, comp_size: comp, encrypted: false, unsupported: None, }; - let mut cur = io::Cursor::new(out); - Ok(visit(&meta, Some(&mut cur), budget)) + let too_big = |name: &str| { + LimitHit::new(format!( + "zip member '{name}' is larger than --max-object-bytes {cap}" + )) + }; + let decoded = match zip.by_index_raw(i) { + Err(_) => Err("ZIP member header will not parse"), + Ok(f) => { + if f.compressed_size() > cap { + return Err(too_big(&meta.name)); + } + let method = crate::formats::zip::zip_method_code(&f.compression()); + let usz = f.size(); + let mut raw = Vec::new(); + match f.take(cap).read_to_end(&mut raw) { + Err(_) => Err("ZIP member data could not be read"), + Ok(_) => crate::formats::zip::decode_zip_raw(method, &raw, usz, cap) + .ok_or("unsupported zip compression method"), + } + } + }; + match decoded { + Err(reason) => { + meta.unsupported = Some(reason); + Ok(visit(&meta, None, budget)) + } + Ok((_, true)) => Err(too_big(&meta.name)), + Ok((out, false)) => { + budget.commit(out.len() as u64); + visit_member(&meta, &mut io::Cursor::new(out), budget, visit) + } + } } /// Handle one encrypted ZIP member: try the password pool, present the decrypted @@ -908,14 +961,7 @@ fn stream_szdd( .seek(io::SeekFrom::Start(0)) .map_err(|e| LimitHit::corrupt(format!("szdd: {e}")))?; let mut hdr = [0u8; 14]; - let mut n = 0; - while n < hdr.len() { - match source.read(&mut hdr[n..]) { - Ok(0) => break, - Ok(k) => n += k, - Err(_) => break, - } - } + let n = crate::read_full(source, &mut hdr)?; if n >= 8 && &hdr[0..8] == SZDD_MAGIC { if n < 14 { return Err(LimitHit::corrupt("szdd: truncated header".to_string())); @@ -981,14 +1027,7 @@ fn stream_swf( .seek(io::SeekFrom::Start(0)) .map_err(|e| LimitHit::corrupt(format!("swf: {e}")))?; let mut hdr = [0u8; 17]; - let mut n = 0; - while n < hdr.len() { - match source.read(&mut hdr[n..]) { - Ok(0) => break, - Ok(k) => n += k, - Err(_) => break, - } - } + let n = crate::read_full(source, &mut hdr)?; if n < 8 || (&hdr[0..3] != b"CWS" && &hdr[0..3] != b"ZWS") { return Ok(None); // FWS / non-SWF: covered by the raw-container scan } @@ -1235,6 +1274,85 @@ mod tests { ); } + /// A ZIP whose central directory will not parse is salvaged from its local + /// headers, and each salvaged member counts once toward `max_members`. + #[cfg(feature = "zip")] + #[test] + fn a_salvaged_zip_counts_each_member_once() { + use std::io::Write; + let mut buf = Cursor::new(Vec::new()); + { + let mut zip = ::zip::ZipWriter::new(&mut buf); + let opts = ::zip::write::SimpleFileOptions::default() + .compression_method(::zip::CompressionMethod::Stored); + for name in ["a.txt", "b.txt"] { + zip.start_file(name, opts).unwrap(); + zip.write_all(b"member").unwrap(); + } + zip.finish().unwrap(); + } + let mut blob = buf.into_inner(); + // Point the end-of-central-directory record's directory offset past + // the end of the file, so only the local headers remain usable. + let eocd = blob.len() - 22; + blob[eocd + 16..eocd + 20].copy_from_slice(&u32::MAX.to_le_bytes()); + + let mut budget = Budget::new(Limits { + max_members: 2, + ..Limits::default() + }); + let mut seen = 0; + let mut visit = |_: &MemberMeta, _: Option<&mut dyn Read>, _: &mut Budget| -> Option<()> { + seen += 1; + None + }; + let walk = stream_members(Format::Zip, Cursor::new(blob), &mut budget, &mut visit); + assert!(walk.is_ok(), "{:?}", walk.err()); + assert_eq!(seen, 2); + } + + /// A member whose local header and data are in the file but whose entry is + /// missing from the central directory is still visited: the target extracts + /// it, so the scan has to read it. + #[cfg(feature = "zip")] + #[test] + fn a_member_hidden_from_the_central_directory_is_visited() { + use std::io::Write; + let zip_of = |name: &str, data: &[u8]| { + let mut buf = Cursor::new(Vec::new()); + { + let mut zip = ::zip::ZipWriter::new(&mut buf); + let opts = ::zip::write::SimpleFileOptions::default() + .compression_method(::zip::CompressionMethod::Stored); + zip.start_file(name, opts).unwrap(); + zip.write_all(data).unwrap(); + zip.finish().unwrap(); + } + buf.into_inner() + }; + let cd_offset = |z: &[u8]| { + let eocd = z.len() - 22; + u32::from_le_bytes(z[eocd + 16..eocd + 20].try_into().unwrap()) as usize + }; + let listed = zip_of("listed.txt", b"listed"); + let hidden = zip_of("hidden.txt", b"hidden"); + // Splice `hidden.txt`'s local header and data in ahead of the listed + // archive's central directory, and move the directory offset past it. + let orphan = &hidden[..cd_offset(&hidden)]; + let cd = cd_offset(&listed); + let mut blob = listed[..cd].to_vec(); + blob.extend_from_slice(orphan); + blob.extend_from_slice(&listed[cd..]); + let eocd = blob.len() - 22; + blob[eocd + 16..eocd + 20].copy_from_slice(&((cd + orphan.len()) as u32).to_le_bytes()); + + let names: Vec = streamed_members(Format::Zip, &blob) + .into_iter() + .map(|(n, _)| n) + .collect(); + assert_eq!(names, ["listed.txt", "hidden.txt"]); + } + // A member at exactly the cap reads fully, with no overflow. #[test] fn budget_reader_delivers_up_to_cap() { diff --git a/crates/exav-unpack/src/volume.rs b/crates/exav-unpack/src/volume.rs index 565bb5a6..bd864d85 100644 --- a/crates/exav-unpack/src/volume.rs +++ b/crates/exav-unpack/src/volume.rs @@ -305,6 +305,10 @@ struct Pending { key: VolumeName, /// `(volume index, name, bytes)`, in arrival order. parts: Vec<(usize, String, Vec)>, + /// A part of this set was passed through rather than held, so the set can + /// never be joined whole. A lost last part leaves no gap in the indices, so + /// this is the only record of it. + lost_part: bool, } impl Collector { @@ -333,13 +337,7 @@ impl Collector { data, }; } - if self.held_bytes.saturating_add(data.len() as u64) > self.max_held_bytes { - return Offer::PassThrough { - name: name.to_string(), - data, - }; - } - + let over_budget = self.held_bytes.saturating_add(data.len() as u64) > self.max_held_bytes; let slot = self.pending.iter_mut().find(|p| p.key.same_set_as(&vn)); let set = match slot { Some(p) => p, @@ -347,13 +345,15 @@ impl Collector { self.pending.push(Pending { key: vn.clone(), parts: Vec::new(), + lost_part: false, }); self.pending.last_mut().expect("just pushed") } }; - // Two members claiming the same position: the set's shape is ambiguous, + // Two members claiming the same position make the set's shape ambiguous, // so joining would splice bytes in an order nothing wrote. - if set.parts.iter().any(|(i, _, _)| *i == vn.index) { + if over_budget || set.parts.iter().any(|(i, _, _)| *i == vn.index) { + set.lost_part = true; return Offer::PassThrough { name: name.to_string(), data, @@ -373,6 +373,19 @@ impl Collector { let mut out = Finished::default(); for mut set in self.pending { set.parts.sort_by_key(|(i, _, _)| *i); + if set.lost_part { + for (_, name, data) in set.parts { + out.unjoined.push(Unjoined { + name, + data, + incomplete_set: Some( + "part of a multi-volume set one of whose volumes could not be \ + held for rejoining", + ), + }); + } + continue; + } // One numbered file on its own is not a set, however much its name // looks like one. Hand it straight back unflagged. if set.parts.len() < 2 { @@ -694,6 +707,30 @@ mod tests { let _ = left; } + /// A part the collector could not hold is scanned on its own, so the set it + /// belongs to is missing it. Joining the parts that were held would scan a + /// truncated prefix as though it were the archive. + #[test] + fn a_set_that_lost_a_part_to_the_budget_does_not_join() { + let mut c = Collector::new(8); + assert!(matches!( + c.offer("s.tar.gz.001", b"AAA".to_vec()), + Offer::Held + )); + assert!(matches!( + c.offer("s.tar.gz.002", b"BBB".to_vec()), + Offer::Held + )); + assert!(matches!( + c.offer("s.tar.gz.003", b"CCCCCC".to_vec()), + Offer::PassThrough { .. } + )); + let done = c.finish(); + assert!(done.joined.is_empty(), "joined a set missing its last part"); + assert_eq!(done.unjoined.len(), 2); + assert!(done.unjoined.iter().all(|u| u.incomplete_set.is_some())); + } + #[test] fn the_held_budget_bounds_memory() { // A container claiming a huge volume set must not turn this into diff --git a/crates/exav-unpack/tests/suites/mod.rs b/crates/exav-unpack/tests/suites/mod.rs index 5b0c9b58..cf8f7ba6 100644 --- a/crates/exav-unpack/tests/suites/mod.rs +++ b/crates/exav-unpack/tests/suites/mod.rs @@ -87,6 +87,8 @@ mod rar_solid; #[cfg(feature = "sevenz")] mod sevenz_bcj2; mod silent_skips; +#[cfg(feature = "all-formats")] +mod stream_read_errors; #[cfg(feature = "iso")] mod udf; #[cfg(feature = "upx")] diff --git a/crates/exav-unpack/tests/suites/pe_emulation.rs b/crates/exav-unpack/tests/suites/pe_emulation.rs index 9c7be3f2..eb2ab755 100644 --- a/crates/exav-unpack/tests/suites/pe_emulation.rs +++ b/crates/exav-unpack/tests/suites/pe_emulation.rs @@ -157,6 +157,38 @@ fn the_recovered_image_is_a_pe_the_scanner_can_walk() { ); } +/// The emulator's work is bounded per top-level file, not only per stub: an +/// archive of many packed executables must not multiply it without limit. +#[test] +fn emulation_is_bounded_across_the_whole_scan() { + let steps = |n: u64| { + let mut l = Limits::default(); + l.max_pe_emulation_steps = n; + Budget::new(l) + }; + let looping = packed_pe(&[0xeb, 0xfe], b"payload"); + let mut b = steps(10_000); + match extract(Format::PePacked, &looping, &mut b) { + Err(hit) => assert!(!hit.is_corrupt(), "a budget stop, not a decode failure"), + Ok(entries) => panic!("the scan-wide emulation budget did not stop the run: {entries:?}"), + } + + // Shared, not per stub: a stub that fits alone does not get a fresh budget + // after another one spent it. The stub runs four instructions per byte. + let plain = payload_of(b"EXAV_SHARED_EMULATION_BUDGET", 0xc000); + let cipher: Vec = plain.iter().map(|b| b ^ 0x5a).collect(); + let small = packed_pe(&decrypt_stub(cipher.len() as u32, 0x5a), &cipher); + let mut b = steps(300_000); + assert!( + extract(Format::PePacked, &small, &mut b).is_ok(), + "fits once" + ); + assert!( + extract(Format::PePacked, &small, &mut b).is_err(), + "not twice" + ); +} + #[test] fn a_stub_that_loops_forever_is_bounded_and_yields_nothing() { // `jmp $`: the file never unpacks. The run must end on its own budget and diff --git a/crates/exav-unpack/tests/suites/stream_read_errors.rs b/crates/exav-unpack/tests/suites/stream_read_errors.rs new file mode 100644 index 00000000..d2264172 --- /dev/null +++ b/crates/exav-unpack/tests/suites/stream_read_errors.rs @@ -0,0 +1,80 @@ +//! A source that fails part way is reported, never read as a short file. +//! +//! The streaming walk reads its container through `Read + Seek`, and for a +//! network range reader a failed read is an ordinary event. Taking it for end +//! of file drops every member past that point without a word. + +use exav_unpack::{is_streamable, stream_members, Budget, Format, Limits, MemberMeta}; +use std::io::{self, Read, Seek, SeekFrom}; + +/// Seeks anywhere in `len` bytes; every read fails. +struct Unreadable { + pos: u64, + len: u64, +} + +impl Read for Unreadable { + fn read(&mut self, _buf: &mut [u8]) -> io::Result { + Err(io::Error::other("link down")) + } +} + +impl Seek for Unreadable { + fn seek(&mut self, to: SeekFrom) -> io::Result { + self.pos = match to { + SeekFrom::Start(p) => p, + SeekFrom::End(d) => self.len.saturating_add_signed(d), + SeekFrom::Current(d) => self.pos.saturating_add_signed(d), + }; + Ok(self.pos) + } +} + +#[test] +fn a_failing_source_is_reported_by_every_streamed_format() { + let formats = [ + Format::Gzip, + Format::Tar, + Format::Bzip2, + Format::Xz, + Format::SevenZip, + Format::Cab, + Format::Zip, + Format::Zstd, + Format::Lzip, + Format::Lha, + Format::Ar, + Format::Cpio, + Format::Machofat, + Format::Pyc, + Format::Sfx, + Format::Tnef, + Format::Partition, + Format::Iso, + Format::OneNote, + Format::Swf, + Format::Szdd, + ]; + let mut silent = Vec::new(); + for fmt in formats.into_iter().filter(|f| is_streamable(*f)) { + let mut budget = Budget::new(Limits::default()); + // A member's bytes are read by the visitor, which sees their errors. + let mut visit = |_: &MemberMeta, r: Option<&mut dyn Read>, _: &mut Budget| { + io::copy(r?, &mut io::sink()).err().map(|e| e.to_string()) + }; + let src = Unreadable { + pos: 0, + len: 1 << 20, + }; + match stream_members(fmt, src, &mut budget, &mut visit) { + Err(hit) if hit.reason.contains("link down") => {} + Ok(Some(seen)) if seen.contains("link down") => {} + other => silent.push(format!("{fmt:?}: {other:?}")), + } + } + assert!( + silent.is_empty(), + "read errors not reported:\n{}", + silent.join("\n") + ); +} diff --git a/crates/exav-unpack/tests/suites/zip_codecs.rs b/crates/exav-unpack/tests/suites/zip_codecs.rs index 750d0ac9..ed96782b 100644 --- a/crates/exav-unpack/tests/suites/zip_codecs.rs +++ b/crates/exav-unpack/tests/suites/zip_codecs.rs @@ -30,6 +30,59 @@ fn zip_lzma_member_is_decoded() { ); } +/// The same member on the streamed walk the top-level scan takes: decoded once, +/// and not also reported as a codec nothing could decode. +#[test] +#[cfg(feature = "lzip")] +fn zip_lzma_member_is_decoded_once_when_streamed() { + use exav_unpack::{stream_members, MemberMeta}; + use std::io::Read; + let blob = fixture("eicar_lzma.zip"); + let mut budget = Budget::new(Limits::default()); + let mut seen = Vec::new(); + let mut visit = |m: &MemberMeta, r: Option<&mut dyn Read>, _: &mut Budget| { + let mut data = Vec::new(); + if let Some(r) = r { + r.read_to_end(&mut data).unwrap(); + } + seen.push((m.name.clone(), m.unsupported, data)); + None::<()> + }; + stream_members( + Format::Zip, + std::io::Cursor::new(blob), + &mut budget, + &mut visit, + ) + .unwrap(); + assert_eq!( + seen.len(), + 1, + "{:?}", + seen.iter().map(|s| (&s.0, s.1)).collect::>() + ); + assert_eq!(seen[0].1, None); + assert!(seen[0].2.windows(EICAR.len()).any(|w| w == EICAR)); + + // Too large to hold is a limit, and says which one. + let mut limits = Limits::default(); + limits.max_buffer_bytes = 16; + let mut budget = Budget::new(limits); + let blob = fixture("eicar_lzma.zip"); + let mut visit = |_: &MemberMeta, _: Option<&mut dyn Read>, _: &mut Budget| None::<()>; + let hit = stream_members( + Format::Zip, + std::io::Cursor::new(blob), + &mut budget, + &mut visit, + ) + .unwrap_err(); + assert!( + !hit.is_corrupt() && hit.reason.contains("--max-object-bytes"), + "{hit:?}" + ); +} + /// Dual indexing: a member present only as a Local File Header (not in the /// central directory) must still be extracted — this defeats central/local /// mismatch hiding. diff --git a/crates/exav-update/src/lib.rs b/crates/exav-update/src/lib.rs index c65d6294..88cf2952 100644 --- a/crates/exav-update/src/lib.rs +++ b/crates/exav-update/src/lib.rs @@ -108,9 +108,34 @@ fn scheme_of(url: &str) -> String { .map_or_else(String::new, |(s, _)| s.to_ascii_lowercase()) } +/// Undo a URL's percent-encoding. A `%` not followed by two hex digits is kept. +fn percent_decode(s: &str) -> Vec { + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + let hex = b + .get(i + 1..i + 3) + .and_then(|h| std::str::from_utf8(h).ok()) + .and_then(|h| u8::from_str_radix(h, 16).ok()); + match (b[i], hex) { + (b'%', Some(v)) => { + out.push(v); + i += 3; + } + (c, _) => { + out.push(c); + i += 1; + } + } + } + out +} + /// Split `user:pass@` credentials out of a URL's authority. Returns the URL with /// the userinfo removed and, when present, a ready-to-send `Basic` Authorization -/// header value (kept out of the request line; RFC 7617). +/// header value (kept out of the request line; RFC 7617), built from the +/// decoded credentials. fn split_basic_auth(url: &str) -> (String, Option) { let Some(scheme) = url.find("://") else { return (url.to_string(), None); @@ -124,7 +149,10 @@ fn split_basic_auth(url: &str) -> (String, Option) { let creds = &authority[..at]; let host = &authority[at + 1..]; let clean = format!("{}{host}{}", &url[..after], &url[auth_end..]); - (clean, Some(format!("Basic {}", base64(creds.as_bytes())))) + ( + clean, + Some(format!("Basic {}", base64(&percent_decode(creds)))), + ) } /// The change-detection validator advertised by a response: `ETag` if present, @@ -628,6 +656,21 @@ mod tests { let (url, auth) = split_basic_auth("http://bob:s3cr3t@host.tld:8080"); assert_eq!(url, "http://host.tld:8080"); assert_eq!(auth.as_deref(), Some("Basic Ym9iOnMzY3IzdA==")); + + // A URL carries reserved characters percent-encoded; the server is + // owed the characters themselves. + let (url, auth) = split_basic_auth("https://a%40b.c:p%2Fw%3Ad%25@host.tld/db"); + assert_eq!(url, "https://host.tld/db"); + assert_eq!( + auth.as_deref(), + Some(format!("Basic {}", base64(b"a@b.c:p/w:d%")).as_str()) + ); + // A `%` not followed by two hex digits is taken as written. + let (_, auth) = split_basic_auth("https://u:50%zz@host.tld/db"); + assert_eq!( + auth.as_deref(), + Some(format!("Basic {}", base64(b"u:50%zz")).as_str()) + ); } #[test] diff --git a/crates/exav/README.md b/crates/exav/README.md index e6cf01d4..f6ad28a3 100644 --- a/crates/exav/README.md +++ b/crates/exav/README.md @@ -15,13 +15,13 @@ cargo install exav exav file.bin # scan one file exav /var/www # scan a directory, recursively -cat file.zip | exav - # scan stdin (constant memory, any size) +cat file.zip | exav - # scan stdin exav -d /var/lib/clamav /data # use an existing ClamAV signature directory exav --listen 0.0.0.0:3310 -d /var/lib/exav # serve the clamd protocol ``` Two things it does that ClamAV does not. **A file too large to scan is never -reported clean** — anything that stops a full scan comes back as status +reported clean**: anything that stops a full scan comes back as status `PARTIAL` under `LIMITS-EXCEEDED`, `UNSCANNABLE` or `PASSWORD-PROTECTED`, and exits 3, where `clamscan` reports `OK` and exits 0. Exit `2` keeps ClamAV's meaning, the scanner itself failing, so `3` is an addition rather than a @@ -29,7 +29,7 @@ reused code. And **one process serves both protocols**: naming a `clamd://` and an `icap://` address on `--listen` replaces a `c-icap` + `clamav` container pair over one loaded database. -exav ships **no signatures** — the ClamAV database is GPL. Fetch them with +exav ships **no signatures** (the ClamAV database is GPL). Fetch them with `freshclam` or `cvd`, or let `--auto-update` keep a directory current. Full documentation, including the ClamAV flag matrix and the migration guide, is diff --git a/crates/exav/src/daemon.rs b/crates/exav/src/daemon.rs index 0e4a7c36..dc04ffd4 100644 --- a/crates/exav/src/daemon.rs +++ b/crates/exav/src/daemon.rs @@ -39,10 +39,12 @@ //! sanitised, capped ~512 chars) //! {"v":1,"status":"PARTIAL","category":C[,"reason":R]} //! (C ∈ LIMITS-EXCEEDED / UNSCANNABLE / -//! PASSWORD-PROTECTED / TRUNCATED — a stream that -//! was not fully scanned is PARTIAL, NEVER OK) +//! PASSWORD-PROTECTED; a stream that was not +//! fully scanned is PARTIAL unless `--partial-as` +//! says otherwise) //! {"v":1,"status":"ERROR","reason":R} //! (transient/infra failure the client may retry, +//! a stream that ended before its terminator, //! or a PARTIAL under `--partial-as error`) //! `status` is the same four-word vocabulary the CLI //! prints and `--json` emits, and each word names the @@ -83,8 +85,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use exav_core::{ - analyze_all_with_outcome, scan_path, scan_seekable_located, scan_stream, AllMatchOutcome, - ScanOptions, ScanReport, Scanner, VerdictCategory, + analyze_all_with_outcome, scan_path, scan_seekable_located, AllMatchOutcome, ScanOptions, + ScanReport, Scanner, Verdict, VerdictCategory, }; use walkdir::WalkDir; @@ -215,8 +217,9 @@ impl<'a> AncillaryReader<'a> { /// One `recvmsg` into `buf`, draining any passed fds into `self.fds`. fn recv(&mut self, buf: &mut [u8]) -> io::Result { use std::os::fd::{AsRawFd, RawFd}; - // Control buffer sized for a handful of fds. - let mut cmsg = [0u8; 256]; + // Control buffer sized for a handful of fds. `u64` rather than `u8` for + // the alignment `cmsghdr` needs, as in `send_fd_command`. + let mut cmsg = [0u64; 32]; let mut iov = libc::iovec { iov_base: buf.as_mut_ptr() as *mut libc::c_void, iov_len: buf.len(), @@ -228,7 +231,7 @@ impl<'a> AncillaryReader<'a> { msg.msg_iov = &mut iov; msg.msg_iovlen = 1; msg.msg_control = cmsg.as_mut_ptr() as *mut libc::c_void; - msg.msg_controllen = cmsg.len() as _; + msg.msg_controllen = std::mem::size_of_val(&cmsg) as _; let n = libc::recvmsg(self.stream.as_raw_fd(), &mut msg, 0); if n < 0 { return Err(io::Error::last_os_error()); @@ -590,6 +593,24 @@ enum BoundListener { Tcp(TcpListener), } +#[cfg(unix)] +impl BoundListener { + fn as_raw_fd(&self) -> std::os::fd::RawFd { + use std::os::fd::AsRawFd; + match self { + BoundListener::Unix(l) => l.as_raw_fd(), + BoundListener::Tcp(l) => l.as_raw_fd(), + } + } + + fn set_nonblocking(&self) -> io::Result<()> { + match self { + BoundListener::Unix(l) => l.set_nonblocking(true), + BoundListener::Tcp(l) => l.set_nonblocking(true), + } + } +} + #[cfg(unix)] fn bind_listener(addr: &ListenAddr) -> io::Result { match addr { @@ -653,18 +674,74 @@ static RELOAD_REQUESTED: std::sync::atomic::AtomicBool = std::sync::atomic::Atom #[cfg(unix)] extern "C" fn on_shutdown(_sig: libc::c_int) { SHUTDOWN.store(true, std::sync::atomic::Ordering::Relaxed); + wake(); } #[cfg(unix)] extern "C" fn on_sighup(_sig: libc::c_int) { RELOAD_REQUESTED.store(true, std::sync::atomic::Ordering::Relaxed); + wake(); +} + +/// Wakes the supervisor so a worker exit is reaped at once. +#[cfg(unix)] +extern "C" fn on_sigchld(_sig: libc::c_int) { + wake(); +} + +/// The supervisor's self-pipe: the signal handlers write to it and [`nap`] +/// polls it. Interrupting a sleep is not enough, because a signal that lands +/// while the supervisor is awake would then wait out the next full nap. +#[cfg(unix)] +static WAKE_READ: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(-1); +#[cfg(unix)] +static WAKE_WRITE: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(-1); +/// A byte is in the pipe. Keeps it to one, so the handler's `write` never +/// fails and never changes `errno` under the code it interrupted. +#[cfg(unix)] +static WAKE_PENDING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Create the self-pipe. Idempotent; call before installing the handlers. +#[cfg(unix)] +fn arm_wakeup() -> io::Result<()> { + use std::sync::atomic::Ordering; + if WAKE_WRITE.load(Ordering::Relaxed) >= 0 { + return Ok(()); + } + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: `pipe` writes two descriptors into `fds`. + if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { + return Err(io::Error::last_os_error()); + } + for fd in fds { + // SAFETY: sets flags on descriptors this process just created. + unsafe { + libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC); + libc::fcntl(fd, libc::F_SETFL, libc::O_NONBLOCK); + } + } + WAKE_READ.store(fds[0], Ordering::Relaxed); + WAKE_WRITE.store(fds[1], Ordering::Relaxed); + Ok(()) } -/// Empty handler whose only job is to interrupt the supervisor's `nanosleep` -/// (handlers are installed without `SA_RESTART`) so a worker exit is reaped -/// promptly rather than after the full poll interval. +/// Make the next [`nap`] return at once. Async-signal-safe: atomics and one +/// `write`. #[cfg(unix)] -extern "C" fn on_sigchld(_sig: libc::c_int) {} +fn wake() { + use std::sync::atomic::Ordering; + if WAKE_PENDING.swap(true, Ordering::SeqCst) { + return; + } + let fd = WAKE_WRITE.load(Ordering::Relaxed); + if fd >= 0 { + // SAFETY: writes one byte from a static to a descriptor this process + // owns. The pipe is empty here, so this neither blocks nor fails. + unsafe { + libc::write(fd, b"!".as_ptr() as *const libc::c_void, 1); + } + } +} /// Public so the CLI's updater thread can raise a reload after writing new /// signatures to the data dir (equivalent to sending `RELOAD` over the socket). @@ -693,6 +770,41 @@ pub fn request_reload() { #[cfg(unix)] const SUPERVISOR_TICK: std::time::Duration = std::time::Duration::from_secs(10); +/// A child of a retired generation, finishing the job it had when a reload +/// replaced it. +#[cfg(unix)] +struct Retiring { + /// The side listener's child rather than a worker. + side: bool, + since: std::time::Instant, + /// Sent SIGTERM for outliving its grace. + stopped: bool, +} + +#[cfg(unix)] +impl Retiring { + fn new(side: bool, since: std::time::Instant) -> Self { + Self { + side, + since, + stopped: false, + } + } +} + +/// How long a retired child may take to finish: one job's time limit, plus a +/// margin to write its answer. With no time limit, a fixed ceiling. +#[cfg(unix)] +fn retire_grace(max_scan_time: std::time::Duration) -> std::time::Duration { + const MARGIN: std::time::Duration = std::time::Duration::from_secs(5); + const NO_LIMIT: std::time::Duration = std::time::Duration::from_secs(600); + if max_scan_time.is_zero() { + NO_LIMIT + } else { + max_scan_time + MARGIN + } +} + /// Newest mtime of the watched source — the reload trigger for a sidecar that /// writes the volume without sending `RELOAD` (clamd's `SelfCheck`). For a /// **directory** this is the newest mtime **across the whole tree**: the loader @@ -700,7 +812,7 @@ const SUPERVISOR_TICK: std::time::Duration = std::time::Duration::from_secs(10); /// subtree), so the watch must too — an in-place overwrite deep in the tree bumps /// only its own directory's mtime, which a top-level-only scan would miss. For a /// single **file** (a prebuilt database) it is just that file's mtime — an atomic -/// swap replaces it with a newer-mtime inode, so the poll fires. `None` if the +/// swap replaces its inode, and so its mtime, so the poll fires. `None` if the /// path can't be stat'd. Symlinks are not followed (no cycles). /// /// Also the ICAP server's reload trigger, which is why it is not Unix-gated: @@ -720,28 +832,72 @@ pub(crate) fn datadir_mtime(dir: &std::path::Path) -> Option, + now: std::time::SystemTime, +) -> bool { + last != Some(now) +} + +/// Sleep for `d`, or less if a supervisor signal arrived since the last nap. #[cfg(unix)] fn nap(d: std::time::Duration) { - let ts = libc::timespec { - tv_sec: d.as_secs() as _, - tv_nsec: d.subsec_nanos() as _, - }; + use std::sync::atomic::Ordering; + let mut fds = [libc::pollfd { + // A negative fd (not armed) is ignored by `poll`: a plain sleep. + fd: WAKE_READ.load(Ordering::Relaxed), + events: libc::POLLIN, + revents: 0, + }]; + let ms = d.as_millis().min(libc::c_int::MAX as u128) as libc::c_int; + // SAFETY: `fds` is one valid `pollfd` for the duration of the call. unsafe { - libc::nanosleep(&ts, std::ptr::null_mut()); + libc::poll(fds.as_mut_ptr(), 1, ms); } + if fds[0].fd < 0 { + return; + } + let mut buf = [0u8; 16]; + // SAFETY: reads into a local buffer from a nonblocking descriptor this + // process owns; stops at EAGAIN. + while unsafe { libc::read(fds[0].fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) } > 0 {} + // Cleared after draining: a signal before this point is handled by the + // loop the caller is about to run, one after it writes a fresh byte. + WAKE_PENDING.store(false, Ordering::SeqCst); } /// The worker's per-job wall-clock alarm. Terminating immediately is the whole /// point: the scan blew its time budget (possibly stuck inside a dependency /// that never returns to a cooperative checkpoint), and there is no safe way to /// unwind in-process — so we `_exit` (async-signal-safe) and let the supervisor -/// respawn a replacement. +/// respawn a replacement. The client gets a verdict first, as in +/// [`on_sigabrt`]: a connection closed with no reply reads as clean to some +/// clients. #[cfg(unix)] extern "C" fn on_sigalrm(_sig: libc::c_int) { + let fd = CURRENT_CONN_FD.load(std::sync::atomic::Ordering::Relaxed); + if fd >= 0 { + unsafe { + libc::write( + fd, + TIMEOUT_REPLY.as_ptr() as *const libc::c_void, + TIMEOUT_REPLY.len(), + ); + } + } unsafe { libc::_exit(EXIT_TIMEOUT) } } +/// Reply [`on_sigalrm`] emits, framed like [`ABORT_REPLY`]. +#[cfg(unix)] +const TIMEOUT_REPLY: &[u8] = b": scan exceeded --max-scan-secs LIMITS-EXCEEDED ERROR\n\0"; + /// The fd of the connection this worker is currently serving, for the abort /// handler below. `-1` when idle. A raw atomic because a signal handler may read /// nothing that could lock. @@ -783,6 +939,37 @@ extern "C" fn on_sigabrt(_sig: libc::c_int) { unsafe { libc::_exit(EXIT_ABORTED) } } +/// Whether the worker is inside a job: between `arm` and `disarm`. +#[cfg(unix)] +static IN_JOB: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Reply [`on_sigterm`] emits, framed like [`ABORT_REPLY`]. +#[cfg(unix)] +const STOPPED_REPLY: &[u8] = b": scan stopped before it finished ERROR\n\0"; + +/// A worker told to stop (a shutdown, its supervisor gone, or a retired +/// generation past its grace) answers the job it is in, as [`on_sigalrm`] +/// does. Between jobs it just exits: a line written then would be read as the +/// answer to the client's next command. +#[cfg(unix)] +extern "C" fn on_sigterm(_sig: libc::c_int) { + let fd = CURRENT_CONN_FD.load(std::sync::atomic::Ordering::Relaxed); + if fd >= 0 && IN_JOB.load(std::sync::atomic::Ordering::Relaxed) { + unsafe { + libc::write( + fd, + STOPPED_REPLY.as_ptr() as *const libc::c_void, + STOPPED_REPLY.len(), + ); + } + } + unsafe { libc::_exit(EXIT_STOPPED) } +} + +/// Exit code for a worker stopped by [`on_sigterm`]. +#[cfg(unix)] +const EXIT_STOPPED: libc::c_int = 92; + /// Exit code for a worker that aborted mid-scan and managed to say so. #[cfg(unix)] const EXIT_ABORTED: libc::c_int = 91; @@ -829,16 +1016,6 @@ fn current_address_space() -> Option { (page > 0).then(|| pages.saturating_mul(page as u64)) } -/// How much address space one job may use so the whole pool fits in RAM. -/// -/// `None` when the host's memory cannot be read, in which case the configured -/// value stands unmodified. -/// -/// The signature database is shared copy-on-write by every worker, so it is -/// subtracted ONCE rather than per worker. A slice of RAM is held back for the -/// kernel and everything else on the box — running the machine to exactly zero -/// free just moves the kill from one process to another. -#[cfg(unix)] /// Bring the in-core extraction budget inside the memory a job is actually /// granted, so the deterministic cap fires before the kernel one. /// @@ -859,6 +1036,7 @@ fn current_address_space() -> Option { /// /// Clamping here restores the intended order. It only ever *lowers* a budget, /// so it cannot make a scan look at more than the operator asked for. +#[cfg(unix)] pub(crate) fn fit_limits_to_job_memory(opts: &mut ScanOptions, job_memory: u64) { /// Of the memory a job is granted, the share extraction buffers may claim. /// The rest covers the matcher's own working set — chiefly the lowercase @@ -884,35 +1062,82 @@ pub(crate) fn fit_limits_to_job_memory(opts: &mut ScanOptions, job_memory: u64) } /// This process's cgroup memory ceiling, if it has one. +#[cfg(unix)] +fn cgroup_memory_limit() -> Option { + let own = std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default(); + cgroup_limit_from(&own, |p| std::fs::read_to_string(p).ok()) +} + +/// The smallest memory limit on the cgroup `proc_self_cgroup` names and its +/// ancestors, reading files through `read`. /// /// v2 first (`memory.max`, the unified hierarchy every current runtime uses), -/// then the v1 path. Both spell "no limit" as a sentinel — literal `max` on v2, -/// a number near `u64::MAX` on v1 — which reads as *unlimited*, not as a cap of -/// that size. Absent or unreadable means no cgroup ceiling to apply. -fn cgroup_memory_limit() -> Option { - for path in [ - "/sys/fs/cgroup/memory.max", // cgroup v2 - "/sys/fs/cgroup/memory/memory.limit_in_bytes", // cgroup v1 - ] { - let Ok(raw) = std::fs::read_to_string(path) else { +/// then v1 (`memory.limit_in_bytes`). Both spell "no limit" as a sentinel +/// (literal `max` on v2, a number near `u64::MAX` on v1), which reads as +/// *unlimited*, not as a cap of that size. A path the mount does not show (a +/// v1 container sees its own cgroup at the mount root) ends up at the root, +/// which is then the process's own. +#[cfg(unix)] +fn cgroup_limit_from(proc_self_cgroup: &str, read: impl Fn(&str) -> Option) -> Option { + let mut v2 = None; + let mut v1 = None; + for line in proc_self_cgroup.lines() { + let mut f = line.splitn(3, ':'); + let (Some(_), Some(controllers), Some(path)) = (f.next(), f.next(), f.next()) else { continue; }; - let raw = raw.trim(); - if raw == "max" { - return None; - } - if let Ok(v) = raw.parse::() { - // v1 reports "unlimited" as a huge page-aligned number rather than a - // word; anything past the host's own RAM is not a real ceiling. - if v > 0 && v < (u64::MAX >> 10) { - return Some(v); - } - return None; + if controllers.is_empty() { + v2 = Some(path); + } else if controllers.split(',').any(|c| c == "memory") { + v1 = Some(path); } } - None + let lowest = |mount: &str, file: &str, own: &str| -> Option> { + let mut dir = own.trim_end_matches('/').to_string(); + let mut found = None; + let mut limit: Option = None; + loop { + if let Some(raw) = read(&format!("{mount}{dir}/{file}")) { + found = Some(()); + let raw = raw.trim(); + // v1 reports "unlimited" as a huge page-aligned number rather + // than a word; anything past the host's own RAM is no ceiling. + if let Ok(v) = raw.parse::() { + if v > 0 && v < (u64::MAX >> 10) { + limit = Some(limit.map_or(v, |l| l.min(v))); + } + } + } + match dir.rfind('/') { + Some(i) => dir.truncate(i), + None => break, + } + } + found.map(|()| limit) + }; + [ + v2.and_then(|p| lowest("/sys/fs/cgroup", "memory.max", p)), + v1.and_then(|p| lowest("/sys/fs/cgroup/memory", "memory.limit_in_bytes", p)), + // No `/proc/self/cgroup` to go by: the mount roots, as seen from here. + lowest("/sys/fs/cgroup", "memory.max", "/"), + lowest("/sys/fs/cgroup/memory", "memory.limit_in_bytes", "/"), + ] + .into_iter() + .flatten() + .next() + .flatten() } +/// How much address space one job may use so the whole pool fits in RAM. +/// +/// `None` when the host's memory cannot be read, in which case the configured +/// value stands unmodified. +/// +/// The signature database is shared copy-on-write by every worker, so it is +/// subtracted ONCE rather than per worker. A slice of RAM is held back for the +/// kernel and everything else on the box: running the machine to exactly zero +/// free just moves the kill from one process to another. +#[cfg(unix)] fn affordable_job_memory(workers: usize, shared_db_bytes: u64) -> Option { /// Leave this fraction of total RAM to the rest of the system. /// @@ -1033,10 +1258,105 @@ fn set_timer(d: std::time::Duration) { pub struct SideListener { /// Named in the supervisor's log when the child exits. pub name: &'static str, - /// Serves the listener it captured, in the forked child. Runs until the - /// process is stopped; returning at all means the listener is gone. + /// Serves the listener it captured, in the forked child. The descriptor is + /// the child's generation (see [`await_connection`]): once it is retired, + /// the child finishes what it is serving and exits 0. #[allow(clippy::type_complexity)] - pub serve: Box, Arc) + Send + Sync>, + pub serve: Box, Arc, std::os::fd::RawFd) + Send + Sync>, +} + +/// One generation of the pool: a pipe whose write end only the supervisor +/// holds. Its children wait on the read end alongside the listener, and +/// dropping the generation closes the write end, which they see as end of +/// file: an idle child leaves at once, a busy one after its current job. A +/// reload retires the old children this way rather than by signal, so no scan +/// in progress is cut short. +#[cfg(unix)] +struct Generation { + retire: std::os::fd::OwnedFd, + notice: std::os::fd::OwnedFd, +} + +#[cfg(unix)] +impl Generation { + fn new() -> io::Result { + use std::os::fd::FromRawFd; + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: `pipe` writes two descriptors into `fds`. + if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: both descriptors were just created and nothing else owns them. + let (retire, notice) = unsafe { + ( + std::os::fd::OwnedFd::from_raw_fd(fds[0]), + std::os::fd::OwnedFd::from_raw_fd(fds[1]), + ) + }; + for fd in fds { + // SAFETY: sets a flag on a descriptor this process owns. + unsafe { + libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC); + } + } + Ok(Self { retire, notice }) + } + + /// In a child just forked: drop its copy of the write end, so only the + /// supervisor's keeps the generation alive. Returns the end to wait on. + fn enter(&self) -> std::os::fd::RawFd { + use std::os::fd::AsRawFd; + // SAFETY: closes this process's inherited copy. The `OwnedFd` is the + // parent's value and is never dropped here: a child does not return. + unsafe { + libc::close(self.notice.as_raw_fd()); + } + self.retire.as_raw_fd() + } +} + +/// An accept error that only means there is nothing to accept right now. +#[cfg(unix)] +pub(crate) fn lost_accept_race(e: &io::Error) -> bool { + matches!( + e.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::Interrupted + ) +} + +/// Block until `listener` has a connection to accept (`true`) or the +/// generation `retire` belongs to has been retired (`false`). +#[cfg(unix)] +pub(crate) fn await_connection(listener: std::os::fd::RawFd, retire: std::os::fd::RawFd) -> bool { + let mut fds = [ + libc::pollfd { + fd: retire, + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: listener, + events: libc::POLLIN, + revents: 0, + }, + ]; + loop { + // SAFETY: `fds` is two valid `pollfd` for the duration of the call. + let n = unsafe { libc::poll(fds.as_mut_ptr(), 2, -1) }; + if n < 0 { + if io::Error::last_os_error().kind() == io::ErrorKind::Interrupted { + continue; + } + // Let the accept that follows report the failure. + return true; + } + if fds[0].revents != 0 { + return false; + } + if fds[1].revents != 0 { + return true; + } + } } /// Run the daemon as a prefork pool of `cfg.workers` worker processes. @@ -1065,11 +1385,14 @@ pub fn run_prefork( } let listener = bind_listener(&addr)?; + // Workers wait in `await_connection`, not in `accept`: several may wake for + // one connection, and the ones that lose the race must not block in accept. + listener.set_nonblocking()?; - // Warm every lazily-initialised structure (engine automaton, compiled YARA - // rules) BEFORE forking, so all workers share them read-only via COW rather - // than each compiling its own private copy on first scan. - let _ = scan_stream(&db, &b"MZ\x90\x00\x00\x00\x00\x00"[..]); + // Warm every lazily-initialised structure BEFORE forking, so all workers + // share them read-only via COW rather than each building its own copy on + // first scan. + exav_core::warm_up(&db); // Sized here, before the options are shared: the address space is measured // once the database is loaded and warmed, and both the per-job ceiling and @@ -1126,19 +1449,26 @@ pub fn run_prefork( // worker inherits it via copy-on-write. DAEMON_MAX_WORKERS.store(cfg.workers.max(1), Ordering::Relaxed); + arm_wakeup()?; install_handler(libc::SIGTERM, on_shutdown); install_handler(libc::SIGINT, on_shutdown); install_handler(libc::SIGHUP, on_sighup); install_handler(libc::SIGCHLD, on_sigchld); + let mut gen = Generation::new()?; let mut children = std::collections::HashSet::new(); for _ in 0..cfg.workers { - children.insert(spawn_worker(&listener, &db, &opts, &cfg)?); + children.insert(spawn_worker(&listener, &db, &opts, &cfg, &gen)?); } let mut side_pid = match side { - Some(s) => Some(spawn_side(s, &db, &opts)?), + Some(s) => Some(spawn_side(s, &db, &opts, &gen)?), None => None, }; + // Children of retired generations, still finishing their last job. They + // are reaped but not replaced, and stopped once their grace has passed. + let mut retiring: std::collections::HashMap = + std::collections::HashMap::new(); + let grace = retire_grace(cfg.max_scan_time); let mut last_mtime = datadir.as_deref().and_then(datadir_mtime); @@ -1157,13 +1487,23 @@ pub fn run_prefork( if pid <= 0 { break; // 0 = none exited yet; <0 = no children / error } + let side_name = side.map_or("worker", |s| s.name); + if let Some(r) = retiring.remove(&pid) { + log_child_exit(pid, status, if r.side { side_name } else { "worker" }, true); + continue; + } let is_side = side_pid == Some(pid); if is_side { side_pid = None; } else { children.remove(&pid); } - log_child_exit(pid, status, side.map_or("worker", |s| s.name), is_side); + log_child_exit( + pid, + status, + if is_side { side_name } else { "worker" }, + false, + ); if !SHUTDOWN.load(Ordering::Relaxed) { if is_side { // Unwrapping is sound: `is_side` is only ever true when a @@ -1172,9 +1512,10 @@ pub fn run_prefork( side.expect("a side listener exited, so there is one"), &db, &opts, + &gen, )?); } else { - children.insert(spawn_worker(&listener, &db, &opts, &cfg)?); + children.insert(spawn_worker(&listener, &db, &opts, &cfg, &gen)?); } } } @@ -1183,7 +1524,7 @@ pub fn run_prefork( // disk (a sidecar wrote it). Coalesce both into one reload per tick. let disk_changed = match datadir.as_deref().and_then(datadir_mtime) { Some(t) => { - let changed = last_mtime.map(|prev| t > prev).unwrap_or(true); + let changed = source_changed(last_mtime, t); if changed { last_mtime = Some(t); } @@ -1199,24 +1540,27 @@ pub fn run_prefork( new_db.signature_count() ); // Warm before forking so new workers share it COW. - let _ = scan_stream(&new_db, &b"MZ\x90\x00\x00\x00\x00\x00"[..]); + exav_core::warm_up(&new_db); db = Arc::new(new_db); - // Graceful re-fork: bring up a fresh generation from the new - // DB, then retire the old children (reaped on the next tick). - // The side listener goes with them, which is the whole of its - // signature swap — it comes back forked from the new database. - let mut old: Vec = children.drain().collect(); - old.extend(side_pid.take()); + // Replacing the generation drops the old one, which is the + // old children's notice to leave once idle. It happens + // before the fork so the new children never hold the old + // write end. The side listener goes with them, which is the + // whole of its signature swap: it comes back forked from + // the new database. + gen = Generation::new()?; + let since = std::time::Instant::now(); + retiring.extend( + children + .drain() + .map(|pid| (pid, Retiring::new(false, since))), + ); + retiring.extend(side_pid.take().map(|pid| (pid, Retiring::new(true, since)))); for _ in 0..cfg.workers { - children.insert(spawn_worker(&listener, &db, &opts, &cfg)?); + children.insert(spawn_worker(&listener, &db, &opts, &cfg, &gen)?); } if let Some(s) = side { - side_pid = Some(spawn_side(s, &db, &opts)?); - } - for pid in old { - unsafe { - libc::kill(pid, libc::SIGTERM); - } + side_pid = Some(spawn_side(s, &db, &opts, &gen)?); } } Err(e) => { @@ -1225,11 +1569,43 @@ pub fn run_prefork( } } - nap(SUPERVISOR_TICK); + // A retired child past its grace is stopped: an idle session or a + // client that never finishes its request would otherwise keep it, and + // the old signatures it serves, alive indefinitely. + let now = std::time::Instant::now(); + let mut tick = SUPERVISOR_TICK; + for (&pid, r) in retiring.iter_mut().filter(|(_, r)| !r.stopped) { + let due = r.since + grace; + if now >= due { + let what = if r.side { + side.map_or("worker", |s| s.name) + } else { + "worker" + }; + eprintln!( + "exav: {what} {pid} of a retired generation is still busy after {}s; stopping it", + grace.as_secs() + ); + // SAFETY: signals a child of this process that has not been + // reaped, so the pid cannot have been reused. + unsafe { + libc::kill(pid, libc::SIGTERM); + } + r.stopped = true; + } else { + tick = tick.min(due - now); + } + } + nap(tick); } // Graceful teardown: signal every child, then reap them. - let all: Vec = children.iter().copied().chain(side_pid).collect(); + let all: Vec = children + .iter() + .copied() + .chain(side_pid) + .chain(retiring.keys().copied()) + .collect(); for &pid in &all { unsafe { libc::kill(pid, libc::SIGTERM); @@ -1296,6 +1672,7 @@ fn spawn_side( side: &SideListener, db: &Arc, opts: &Arc, + gen: &Generation, ) -> io::Result { use std::io::Write as _; let _ = io::stderr().flush(); @@ -1306,6 +1683,7 @@ fn spawn_side( match pid { -1 => Err(io::Error::last_os_error()), 0 => { + let retire = gen.enter(); unsafe { libc::signal(libc::SIGTERM, libc::SIG_DFL); libc::signal(libc::SIGINT, libc::SIG_DFL); @@ -1316,7 +1694,7 @@ fn spawn_side( // disposition — a handler inherited from the supervisor would // otherwise decide what parent death means here. die_with_parent(supervisor); - (side.serve)(Arc::clone(db), Arc::clone(opts)); + (side.serve)(Arc::clone(db), Arc::clone(opts), retire); // The listener was the child's whole job, so there is nothing left // for it to do but let the supervisor respawn it. unsafe { libc::_exit(2) } @@ -1331,6 +1709,7 @@ fn spawn_worker( db: &Arc, opts: &Arc, cfg: &PoolConfig, + gen: &Generation, ) -> io::Result { // Flush so buffered parent output isn't duplicated into the child. use std::io::Write as _; @@ -1340,14 +1719,15 @@ fn spawn_worker( let pid = unsafe { libc::fork() }; match pid { -1 => Err(io::Error::last_os_error()), - 0 => worker_main(listener, db, opts, cfg, supervisor), // never returns + 0 => worker_main(listener, db, opts, cfg, supervisor, gen.enter()), // never returns n => Ok(n), } } /// A worker process: bound by the kernel limits, it accepts and serves -/// connections one at a time until it hits its job limit (then exits cleanly so -/// the supervisor recycles it). Never returns. +/// connections one at a time until it hits its job limit or its generation +/// `retire` is retired (then exits cleanly, and the supervisor replaces it in +/// the first case only). Never returns. #[cfg(unix)] fn worker_main( listener: &BoundListener, @@ -1355,6 +1735,7 @@ fn worker_main( opts: &ScanOptions, cfg: &PoolConfig, supervisor: libc::pid_t, + retire: std::os::fd::RawFd, ) -> ! { // Apply the kernel-enforced resource caps to *this* process. // @@ -1379,21 +1760,27 @@ fn worker_main( install_handler(libc::SIGALRM, on_sigalrm); // Report an allocation failure to the client instead of dying silently. install_handler(libc::SIGABRT, on_sigabrt); - // Restore default disposition for the signals the parent handles, so the - // supervisor's kill terminates us and inherited handlers don't fire here. + // Replace the handlers inherited from the supervisor, so its kill + // terminates us and its flag-setting handlers don't fire here. unsafe { - libc::signal(libc::SIGTERM, libc::SIG_DFL); libc::signal(libc::SIGINT, libc::SIG_DFL); libc::signal(libc::SIGHUP, libc::SIG_DFL); libc::signal(libc::SIGCHLD, libc::SIG_DFL); } - // Only now, with SIGTERM back to its default disposition: arranged before - // the reset, parent death would run the handler this worker inherited from - // the supervisor, which only sets a flag — and the worker would go on to - // block in `accept()` exactly as if nothing had been arranged at all. + install_handler(libc::SIGTERM, on_sigterm); + // Only now, with SIGTERM handled here: arranged before, parent death would + // run the handler this worker inherited from the supervisor, which only + // sets a flag, and the worker would go on to block in `accept()` exactly + // as if nothing had been arranged at all. die_with_parent(supervisor); - let arm = || set_timer(cfg.max_scan_time); - let disarm = || set_timer(std::time::Duration::ZERO); + let arm = || { + IN_JOB.store(true, std::sync::atomic::Ordering::Relaxed); + set_timer(cfg.max_scan_time) + }; + let disarm = || { + set_timer(std::time::Duration::ZERO); + IN_JOB.store(false, std::sync::atomic::Ordering::Relaxed); + }; // A `RELOAD` command reaches a worker, not the supervisor; forward it by // signalling the parent, which owns the pool and does the re-fork. let reload = || unsafe { @@ -1416,9 +1803,15 @@ fn worker_main( let mut jobs = 0u64; loop { + if !await_connection(listener.as_raw_fd(), retire) { + unsafe { libc::_exit(0) } + } + // The listener is non-blocking, and on some systems an accepted socket + // inherits that; a job's reads are written for a blocking one. let outcome: io::Result<()> = match listener { BoundListener::Unix(l) => match l.accept() { Ok((stream, _)) => { + let _ = stream.set_nonblocking(false); let _ = stream.set_read_timeout(Some(SOCKET_READ_TIMEOUT)); // Record the fd so `on_sigabrt` can answer on this connection // if the scan aborts, instead of closing without a word. @@ -1442,7 +1835,8 @@ fn worker_main( disarm(); // ensure the timer is off between connections r } - Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + // Another worker took the connection this one woke for. + Err(e) if lost_accept_race(&e) => continue, Err(e) => unsafe { eprintln!("exav: worker accept error: {e}"); libc::_exit(1); @@ -1450,6 +1844,7 @@ fn worker_main( }, BoundListener::Tcp(l) => match l.accept() { Ok((stream, _)) => { + let _ = stream.set_nonblocking(false); let _ = stream.set_read_timeout(Some(SOCKET_READ_TIMEOUT)); // As on the Unix path: without the fd, `on_sigabrt` has // nowhere to send its last word and an aborted scan closes @@ -1476,7 +1871,7 @@ fn worker_main( disarm(); r } - Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) if lost_accept_race(&e) => continue, Err(e) => unsafe { eprintln!("exav: worker accept error: {e}"); libc::_exit(1); @@ -1496,14 +1891,16 @@ fn worker_main( } /// Decode a reaped child's wait-status into a human-readable cause, so the -/// operator can see *why* it died (timeout / OOM / CPU / recycle). `side_name` -/// names the extra listener's child when `is_side`; everything else is a worker. +/// operator can see *why* it died (timeout / OOM / CPU / recycle). `what` names +/// the child; `retired` says it belonged to a generation a reload replaced. #[cfg(unix)] -fn log_child_exit(pid: libc::pid_t, status: libc::c_int, side_name: &str, is_side: bool) { +fn log_child_exit(pid: libc::pid_t, status: libc::c_int, what: &str, retired: bool) { let cause = if libc::WIFEXITED(status) { match libc::WEXITSTATUS(status) { EXIT_TIMEOUT => "scan wall-clock timeout".to_string(), EXIT_ABORTED => "aborted mid-scan (reported to the client)".to_string(), + EXIT_STOPPED => "stopped".to_string(), + 0 if retired => "retired after a reload".to_string(), 0 => "recycled (job limit / clean exit)".to_string(), code => format!("exit code {code}"), } @@ -1517,7 +1914,6 @@ fn log_child_exit(pid: libc::pid_t, status: libc::c_int, side_name: &str, is_sid } else { "unknown".to_string() }; - let what = if is_side { side_name } else { "worker" }; eprintln!("exav: {what} {pid} exited: {cause}"); } @@ -2219,10 +2615,10 @@ impl StreamPayload { } } - /// The payload's bytes, or `None` when it is larger than `cap`. Used to feed - /// a payload back for multi-volume rejoining, which is the one thing that - /// needs a second look at the same bytes. - fn bytes_capped(&self, cap: u64) -> io::Result>> { + /// The payload's bytes, or `None` when it is larger than `cap`. For the + /// second looks at the same bytes: multi-volume rejoining, and + /// `--all-matches`. + pub(crate) fn bytes_capped(&self, cap: u64) -> io::Result>> { if self.len() > cap { return Ok(None); } @@ -2237,59 +2633,129 @@ impl StreamPayload { } } +/// A stream read into a seekable payload, as far as it could be held. +pub(crate) struct Held { + pub(crate) payload: StreamPayload, + /// Why the payload stops before the stream's end, when it does: there was + /// nowhere left to put the rest. The reader is left at that point. + pub(crate) short: Option, +} + /// Buffer an (already de-chunked / pre-capped) reader into a [`StreamPayload`]: /// up to the configured threshold in RAM, then spill the rest to a temp file. -/// Shared by the daemon stream verbs and the CLI stdin path so both get the same -/// seekable, container-aware scan under the same budgets. -pub(crate) fn buffer_to_seekable( - reader: &mut R, -) -> Result { +/// Shared by the daemon stream verbs and the CLI's stdin so all get the same +/// seekable, container-aware scan under the same budgets. A stream larger than +/// the budgets allow is held as far as they allow. +pub(crate) fn buffer_to_seekable(reader: &mut R) -> io::Result { + use crate::spill::SpillError; let threshold = crate::spill::config().threshold; let mut buf = Vec::new(); reader.by_ref().take(threshold).read_to_end(&mut buf)?; - if (buf.len() as u64) < threshold { - return Ok(StreamPayload::Mem(buf)); + let mut block = vec![0u8; 64 * 1024]; + let got = match (buf.len() as u64) < threshold { + true => 0, + false => read_full(reader, &mut block)?, + }; + if got == 0 { + return Ok(Held { + payload: StreamPayload::Mem(buf), + short: None, + }); } - // More data remains — spill the RAM head, then stream the rest to disk. The + // More data remains: spill the RAM head, then stream the rest to disk. The // budgets live in `SpillFile`: `RLIMIT_AS` caps address space and does not // cap a file, so without them a client that never stops sending fills the - // temp filesystem — a denial of service against the host rather than the + // temp filesystem, a denial of service against the host rather than the // scan, and one that outlives the connection. - let mut tmp = crate::spill::SpillFile::create()?; - tmp.write_all(&buf)?; - let mut block = vec![0u8; 64 * 1024]; - loop { - let got = reader.read(&mut block)?; - if got == 0 { + let mut tmp = match crate::spill::SpillFile::create() { + Ok(t) => t, + Err(SpillError::Budget(why)) => { + return Ok(Held { + payload: StreamPayload::Mem(buf), + short: Some(why), + }) + } + Err(SpillError::Io(e)) => return Err(e), + }; + let mut short = None; + for chunk in [&buf[..], &block[..got]] { + match tmp.write_all(chunk) { + Ok(()) => {} + Err(SpillError::Budget(why)) => { + short = Some(why); + break; + } + Err(SpillError::Io(e)) => return Err(e), + } + } + while short.is_none() { + let n = reader.read(&mut block)?; + if n == 0 { break; } - tmp.write_all(&block[..got])?; + match tmp.write_all(&block[..n]) { + Ok(()) => {} + Err(SpillError::Budget(why)) => short = Some(why), + Err(SpillError::Io(e)) => return Err(e), + } } let len = tmp.len()?; - Ok(StreamPayload::Disk(tmp, len)) + Ok(Held { + payload: StreamPayload::Disk(tmp, len), + short, + }) +} + +/// Scan what was held of a stream of `size` bytes. A payload that stops short +/// for want of room is treated as `--max-input-bytes` treats an input past it: +/// the held bytes get the whole scan, and with no detection in them the limit +/// is reported. +pub(crate) fn scan_held( + db: &Scanner, + opts: &ScanOptions, + held: &Held, + size: u64, +) -> io::Result<(ScanReport, Option)> { + let Some(why) = &held.short else { + return scan_payload(db, opts, &held.payload, size); + }; + let kept = held.payload.len(); + let (report, loc) = scan_payload(db, opts, &held.payload, kept)?; + if report.verdict.category() == VerdictCategory::Infected { + return Ok((report, loc)); + } + let reason = format!("{why}; scanned the first {kept} bytes only"); + Ok(( + ScanReport::new(Verdict::LimitsExceeded { reason }, Vec::new()), + None, + )) } /// Scan a materialized payload via the seekable (container-aware) path, returning /// the report and the nested match location (`None` for a top-level hit). +/// +/// `size` is the input's full size. It is larger than the payload when the +/// input went past `--max-input-bytes` and only its start was kept. pub(crate) fn scan_payload( db: &Scanner, opts: &ScanOptions, payload: &StreamPayload, + size: u64, ) -> io::Result<(ScanReport, Option)> { match payload { StreamPayload::Mem(v) => { - scan_seekable_located(db, std::io::Cursor::new(&v[..]), v.len() as u64, opts) + scan_seekable_located(db, std::io::Cursor::new(&v[..]), size, opts) } - StreamPayload::Disk(tmp, len) => { + StreamPayload::Disk(tmp, _) => { // A fresh handle positioned at 0; the `TempFile` stays alive (and // thus the file) for as long as the payload does. let file = tmp.reopen()?; - scan_seekable_located(db, file, *len, opts) + scan_seekable_located(db, file, size, opts) } } } -/// [`scan_payload`], timed into the process counters. +/// [`scan_held`], timed into the process counters. /// /// The ICAP listener times its own scans (it names the object from the request /// target, which this cannot see), so this is the daemon's stream verbs only — @@ -2297,18 +2763,19 @@ pub(crate) fn scan_payload( fn scan_payload_timed( db: &Scanner, opts: &ScanOptions, - payload: &StreamPayload, + held: &Held, + size: u64, target: &str, ) -> io::Result<(ScanReport, Option)> { let timer = crate::metrics::ScanTimer::start(); - let result = scan_payload(db, opts, payload); + let result = scan_held(db, opts, held, size); let category = match &result { Ok((report, _)) => report.verdict.category().into(), // A scan that failed is a scan that did not finish, which is the // partial column wherever else exav counts it. Err(_) => crate::metrics::Category::Partial, }; - timer.finish(target, payload.len(), category); + timer.finish(target, held.payload.len(), category); result } @@ -2322,35 +2789,18 @@ fn instream( opts: &ScanOptions, reader: &mut BufReader, ) -> io::Result { - let max = opts.max_scan_size; - let mut stream = Instream::new(reader, max); - let payload = match buffer_to_seekable(&mut stream) { - Ok(p) => p, - // A budget with no room is a stream nobody could examine, which is a - // verdict the client is owed. Letting it out as an I/O error would drop - // the connection instead, and a client with no answer decides for - // itself — the one outcome an over-full temp filesystem must not buy. - Err(crate::spill::SpillError::Budget(reason)) => { - let _ = stream.drain(); - return Ok(format!("stream: {reason} UNSCANNABLE ERROR")); - } - Err(crate::spill::SpillError::Io(e)) => return Err(e), - }; - let over = stream.over_limit; - let truncated = stream.truncated(); - // The reply is already decided; a drain that stops at its cap only means the - // connection will not be reusable. + let mut stream = Instream::new(reader, opts.max_scan_size); + let held = buffer_to_seekable(&mut stream)?; + // Counts what was not held, so the size is the stream's. A drain that + // stops at its cap only means the connection will not be reusable. let _reached_terminator = stream.drain()?; - if over { - let max = max.unwrap_or(0); - return Ok(format!("stream: size exceeds {max} LIMITS-EXCEEDED ERROR")); - } // Answering a prefix would answer a question the client never finished // asking, and `OK` on the benign head of a file is a bypass anyone can drive. - if truncated { - return Ok("stream: stream ended before its terminator UNSCANNABLE ERROR".to_string()); + // Not a partial either: `--partial-as ok` would turn it back into that `OK`. + if stream.truncated() { + return Ok(format!("stream: {STREAM_CUT} ERROR")); } - let (report, _loc) = scan_payload_timed(db, opts, &payload, "stream")?; + let (report, _loc) = scan_payload_timed(db, opts, &held, stream.total(), "stream")?; Ok(verdict_line("stream", &report)) } @@ -2373,32 +2823,20 @@ fn exinstream( // per-top-level-file bound, which `--max-input-bytes` sets. Naming the other // flag here would send an operator to raise a setting that changes nothing. let mut stream = Instream::new(reader, opts.max_scan_size); - let payload = match buffer_to_seekable(&mut stream) { - Ok(p) => p, + let held = match buffer_to_seekable(&mut stream) { + Ok(h) => h, Err(e) => { let _ = stream.drain(); return Ok(json_error(&format!("stream read error: {e}"))); } }; - let over = stream.over_limit; - let truncated = stream.truncated(); - // The reply is already decided; a drain that stops at its cap only means the - // connection will not be reusable. + // As in INSTREAM: counts what was not held. let _reached_terminator = stream.drain()?; - if over { - return Ok(json_partial( - "LIMITS-EXCEEDED", - Some("stream exceeds max-filesize; not fully scanned"), - )); - } // As in INSTREAM: a prefix is not the file, so it gets no verdict. - if truncated { - return Ok(json_partial( - "TRUNCATED", - Some("stream ended before its terminator; not fully received"), - )); + if stream.truncated() { + return Ok(json_error(STREAM_CUT)); } - match scan_payload_timed(db, opts, &payload, "stream") { + match scan_payload_timed(db, opts, &held, stream.total(), "stream") { Ok((report, loc)) => Ok(verdict_json(&report, loc)), Err(e) => Ok(json_error(&format!("scan error: {e}"))), } @@ -2460,7 +2898,8 @@ fn exinstream_multi( ) -> io::Result { use serde_json::json; let mut names: Vec = Vec::new(); - let mut payloads: Vec = Vec::new(); + // `None` for a file the daemon could not hold. + let mut payloads: Vec> = Vec::new(); let mut entries: Vec = Vec::new(); // Set when the request stops before its terminator. Every file already read // still gets its verdict — those streams were complete — but the request as @@ -2496,44 +2935,37 @@ fn exinstream_multi( let name = String::from_utf8_lossy(&raw).into_owned(); let mut stream = Instream::new(reader, opts.max_scan_size); - let payload = match buffer_to_seekable(&mut stream) { - Ok(p) => p, + let held = match buffer_to_seekable(&mut stream) { + Ok(h) => h, Err(e) => { let _ = stream.drain(); return Ok(json_error(&format!("stream read error: {e}"))); } }; - let over = stream.over_limit; - let truncated = stream.truncated(); - stream.drain()?; - if truncated { - request_truncated = true; - } - let fields = if over { - json!({ - "status": "PARTIAL", - "category": "LIMITS-EXCEEDED", - "reason": "stream exceeds max-filesize; not fully scanned", - }) - } else if truncated { - json!({ - "status": "PARTIAL", - "category": "TRUNCATED", - "reason": "stream ended before its terminator; not fully received", - }) + // Past a drain that stopped at its cap, or at end of input, the next + // file's framing is not where the loop would look for it. + let cut = !stream.drain()? || stream.truncated(); + let fields = if stream.truncated() { + json!({"status": "ERROR", "reason": STREAM_CUT}) } else { - match scan_payload_timed(db, opts, &payload, &name) { + match scan_payload_timed(db, opts, &held, stream.total(), &name) { Ok((report, loc)) => verdict_fields(&report, loc), Err(e) => json!({"status": "ERROR", "reason": format!("scan error: {e}")}), } }; - match &payload { + match &held.payload { StreamPayload::Mem(v) => held_ram += v.len() as u64, StreamPayload::Disk(_, n) => held_spill += *n, } entries.push(named_entry(&name, fields, None)); names.push(name); - payloads.push(payload); + // Only a whole file can be a part of a rejoined set. + let whole = held.short.is_none() && held.payload.len() == stream.total(); + payloads.push(whole.then_some(held.payload)); + if cut { + request_truncated = true; + break; + } // Stop before accepting the file that would cross the line, so the // reply describes a request the daemon actually held rather than one it @@ -2557,6 +2989,8 @@ fn exinstream_multi( .position(|n| n == name) .ok_or_else(|| io::Error::other("no such stream"))?; payloads[i] + .as_ref() + .ok_or_else(|| io::Error::other("stream was not held"))? .bytes_capped(opts.deep_analysis_max)? .ok_or_else(|| io::Error::other("stream too large to rejoin")) }); @@ -2665,13 +3099,8 @@ fn json_error(message: &str) -> String { serde_json::json!({"v": 1, "status": "ERROR", "reason": message}).to_string() } -fn json_partial(tag: &str, message: Option<&str>) -> String { - let mut o = serde_json::json!({"v": 1, "status": "PARTIAL", "category": tag}); - if let Some(m) = message { - o["reason"] = serde_json::json!(m); - } - o.to_string() -} +/// The reason for a stream that ended before its terminator. +const STREAM_CUT: &str = "stream ended before its terminator"; /// A `Read` over a clamd INSTREAM chunk sequence: `` repeated, /// ended by a zero length. Presents the de-chunked payload as one stream. @@ -2767,9 +3196,17 @@ impl<'a, R: Read> Instream<'a, R> { } self.remaining -= n as u32; drained += n as u64; + self.total += n as u64; } Ok(true) } + + /// The stream's size: every byte received, including those past the + /// limit that were only counted. A lower bound when `drain` stopped at + /// its cap. + fn total(&self) -> u64 { + self.total + } } impl Read for Instream<'_, R> { @@ -2792,12 +3229,15 @@ impl Read for Instream<'_, R> { return Ok(0); } self.remaining -= n as u32; + let before = self.total; self.total += n as u64; if let Some(max) = self.max { if self.total > max { - // Stop feeding the scanner; the caller reports the limit. + // Hand over up to one byte past the limit, so the payload + // holds everything the scan looks at and shows it is over; + // the rest is only counted, by `drain`. self.over_limit = true; - return Ok(0); + return Ok((max + 1 - before) as usize); } } return Ok(n); @@ -2842,6 +3282,75 @@ mod tests { ); } + /// A signal that lands while the supervisor is awake (reaping, reloading) + /// still cuts its next nap short, so a crashed worker is replaced and a + /// `RELOAD` served at once rather than a tick later. SIGUSR2 stands in for + /// SIGCHLD so the children other tests spawn do not reach the handler. + #[test] + fn a_signal_before_the_nap_ends_it() { + arm_wakeup().unwrap(); + install_handler(libc::SIGUSR2, on_sigchld); + // SAFETY: raises a signal on this thread; its handler only wakes. + unsafe { + libc::raise(libc::SIGUSR2); + } + let start = std::time::Instant::now(); + nap(SUPERVISOR_TICK); + assert!( + start.elapsed() < std::time::Duration::from_secs(2), + "slept {:?} after the signal", + start.elapsed() + ); + } + + /// The ceiling is the one on this process's own cgroup and its ancestors, + /// not whatever sits at the mount root. + #[test] + fn the_cgroup_limit_is_read_where_the_process_lives() { + let files: std::collections::HashMap<&str, &str> = [ + ("/sys/fs/cgroup/memory.max", "max\n"), + ("/sys/fs/cgroup/system.slice/memory.max", "8589934592\n"), + ( + "/sys/fs/cgroup/system.slice/exav.service/memory.max", + "max\n", + ), + ( + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "9223372036854771712\n", + ), + ( + "/sys/fs/cgroup/memory/docker/abc/memory.limit_in_bytes", + "1073741824\n", + ), + ] + .into_iter() + .collect(); + let read = |p: &str| files.get(p).map(|s| s.to_string()); + + // A service under a slice with a limit: the slice's limit applies. + let v2 = "0::/system.slice/exav.service\n"; + assert_eq!(cgroup_limit_from(v2, read), Some(8 << 30)); + // A container with its own namespace sees itself at the root. + assert_eq!(cgroup_limit_from("0::/\n", read), None); + // v1, memory controller listed with others. + let v1 = "5:cpu,cpuacct:/docker/abc\n4:memory:/docker/abc\n"; + assert_eq!(cgroup_limit_from(v1, read), Some(1 << 30)); + // A v1 path this mount does not show falls back to the mount root. + let v1 = "4:memory:/elsewhere\n"; + assert_eq!(cgroup_limit_from(v1, read), None); + } + + /// A database swapped for one with an older mtime is still a change. + #[test] + fn an_older_mtime_is_still_a_change() { + use std::time::{Duration, SystemTime}; + let t = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + assert!(source_changed(None, t)); + assert!(!source_changed(Some(t), t)); + assert!(source_changed(Some(t), t + Duration::from_secs(1))); + assert!(source_changed(Some(t), t - Duration::from_secs(1))); + } + /// Clamping only ever lowers: a generous grant must leave the operator's /// configuration exactly as they set it. #[test] @@ -3316,7 +3825,9 @@ mod tests { payload.extend_from_slice(eicar()); let expected = payload.len() as u64; - let materialized = buffer_to_seekable(&mut &payload[..]).unwrap(); + let held = buffer_to_seekable(&mut &payload[..]).unwrap(); + assert!(held.short.is_none()); + let materialized = held.payload; assert!( matches!(materialized, StreamPayload::Disk(_, _)), "past the threshold the payload belongs on disk, not in RAM" @@ -3334,7 +3845,8 @@ mod tests { ); let db = Scanner::builtin(); - let (report, _) = scan_payload(&db, &ScanOptions::default(), &materialized).unwrap(); + let (report, _) = + scan_payload(&db, &ScanOptions::default(), &materialized, expected).unwrap(); assert!( matches!(report.verdict.category(), VerdictCategory::Infected), "the spilled bytes are what gets scanned" diff --git a/crates/exav/src/icap/chunked.rs b/crates/exav/src/icap/chunked.rs index 4faf5e7f..f409c665 100644 --- a/crates/exav/src/icap/chunked.rs +++ b/crates/exav/src/icap/chunked.rs @@ -145,6 +145,9 @@ pub(super) struct BodyOutcome { /// the client mid-send, and answering into a socket that still has unread /// bytes makes the kernel reset the connection — taking the verdict with it. pub(super) rejected: Option, + /// The body's size as the chunk headers declared it, kept or not. A lower + /// bound when `abandoned`. + pub(super) total: u64, } impl std::error::Error for BodyError {} @@ -236,6 +239,7 @@ pub(super) fn read_chunked_body( let mut over_limit = false; let mut drained: u64 = 0; let mut rejected: Option = None; + let mut total: u64 = 0; // One buffer reused for every discarded chunk: discarding exists so the // bytes are not held. let mut sink = [0u8; 16 * 1024]; @@ -262,8 +266,10 @@ pub(super) fn read_chunked_body( over_limit, abandoned: false, rejected, + total, }); } + total = total.saturating_add(size); // Split the chunk at the ceiling: what fits is kept so it can still be // scanned, the rest is read and dropped. Keeping the head matters @@ -293,6 +299,7 @@ pub(super) fn read_chunked_body( over_limit: true, abandoned: true, rejected, + total, }); } } @@ -324,6 +331,7 @@ pub(super) fn read_chunked_body( over_limit: true, abandoned: true, rejected, + total, }); } } diff --git a/crates/exav/src/icap/mod.rs b/crates/exav/src/icap/mod.rs index 249d6dad..422eb5a3 100644 --- a/crates/exav/src/icap/mod.rs +++ b/crates/exav/src/icap/mod.rs @@ -36,7 +36,8 @@ //! which is the one answer exav must never give for an object nobody examined. //! //! What keeps the two distinguishable is the threat name, not the header's -//! presence. A database hit is reported under its signature name; //! block under `Heuristics.Exav.` — the prefix ClamAV uses for a +//! presence. A database hit is reported under its signature name; a partial +//! block under `Heuristics.Exav.`, the prefix ClamAV uses for a //! policy block rather than a database entry, qualified by the scanner that //! synthesised it. An analyst reading an incident log can tell them apart, and //! [`X-Exav-Category`](IcapConfig) still carries the exact condition for a client @@ -232,7 +233,7 @@ pub fn serve_alone( let shared = Arc::new(ReloadableDb::from_arc(db)); let handle: Arc = Arc::clone(&shared) as Arc; std::thread::spawn(move || { - if let Err(e) = server.run(handle, opts) { + if let Err(e) = server.run(handle, opts, &|_| true) { eprintln!("exav: icap: listener stopped: {e}"); } // The listener is the whole job. Without it the process is a supervisor @@ -255,7 +256,7 @@ pub fn serve_alone( let Some(now) = crate::daemon::datadir_mtime(&dir) else { continue; }; - if last.map(|prev| now > prev).unwrap_or(true) { + if crate::daemon::source_changed(last, now) { last = Some(now); match reload() { Ok(new_db) => { @@ -282,12 +283,13 @@ pub fn serve_alone( /// clamd listener as well. A dedicated child keeps ICAP's threaded model while /// still sharing the supervisor's warmed database copy-on-write, and the /// supervisor re-forks it from the new database on every reload — so the -/// signature swap needs no in-process machinery at all. +/// signature swap needs no in-process machinery at all. The retired child +/// finishes the requests it is serving before it exits. #[cfg(unix)] pub fn forked_child(server: Server, metrics_interval: Duration) -> crate::daemon::SideListener { crate::daemon::SideListener { name: "icap", - serve: Box::new(move |db, opts| { + serve: Box::new(move |db, opts, retire| { announce(&server); // A child of its own, so its counters are its own: the clamd // workers' `STATS` cannot see them and this log line is where they @@ -295,8 +297,19 @@ pub fn forked_child(server: Server, metrics_interval: Duration) -> crate::daemon // thread does not survive one. crate::metrics::spawn_reporter(metrics_interval, "icap"); let handle: Arc = Arc::new(FixedDb::from_arc(db)); - if let Err(e) = server.run(handle, opts) { + // Shared with the other generation's child during a reload, so + // both wait for readiness instead of blocking in accept. + let wait = |l: &std::net::TcpListener| { + use std::os::fd::AsRawFd; + crate::daemon::await_connection(l.as_raw_fd(), retire) + }; + if let Err(e) = server.set_nonblocking() { eprintln!("exav: icap: listener stopped: {e}"); + std::process::exit(2); + } + match server.run(handle, opts, &wait) { + Ok(()) => std::process::exit(0), + Err(e) => eprintln!("exav: icap: listener stopped: {e}"), } std::process::exit(2); }), diff --git a/crates/exav/src/icap/server.rs b/crates/exav/src/icap/server.rs index 1c72f363..9a7488fa 100644 --- a/crates/exav/src/icap/server.rs +++ b/crates/exav/src/icap/server.rs @@ -7,10 +7,11 @@ //! reach for, and its purpose (killing a job that will not stop) is answered on //! this path by the in-engine budgets. -use std::io::{BufReader, Write}; +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; use std::net::{SocketAddr, TcpListener, TcpStream}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; use exav_core::{scan_seekable_located, ScanOptions, Scanner}; @@ -18,7 +19,7 @@ use super::chunked::{read_chunked_body, Body, BodyEnd, BodyError, BodyOutcome}; use super::config::IcapConfig; use super::response::{self, Decision}; use super::wire::{read_head, Entity, IcapMethod, RequestHead}; -use crate::daemon::StreamPayload; +use crate::daemon::Held; /// Supplies the signature database to scan the next request with. /// @@ -102,16 +103,34 @@ impl Server { &self.cfg } - /// Serve until the listener fails. + /// Make accepting non-blocking, for a listener more than one process waits on. + #[cfg(unix)] + pub(super) fn set_nonblocking(&self) -> std::io::Result<()> { + self.listener.set_nonblocking(true) + } + + /// Serve until `wait` says to stop, then finish the requests in progress + /// and return. `wait` blocks until there is a connection to accept (`true`) + /// or the server should stop (`false`). An error means the listener failed. pub(super) fn run( &self, db: Arc, opts: Arc, + wait: &dyn Fn(&TcpListener) -> bool, ) -> std::io::Result<()> { let live = Arc::new(AtomicUsize::new(0)); - for stream in self.listener.incoming() { - let stream = match stream { - Ok(s) => s, + let idle = Arc::new(Idle::default()); + while wait(&self.listener) { + let stream = match self.listener.accept() { + Ok((s, _)) => s, + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted + ) => + { + continue + } // One failed accept (a client that vanished between the SYN and // the accept, say) is not a reason to stop serving. Err(e) => { @@ -119,6 +138,8 @@ impl Server { continue; } }; + // An accepted socket can inherit the listener's non-blocking mode. + let _ = stream.set_nonblocking(false); if live.fetch_add(1, Ordering::Relaxed) >= self.cfg.max_connections { live.fetch_sub(1, Ordering::Relaxed); // Nothing is written back: at capacity, the cheapest safe act @@ -129,21 +150,66 @@ impl Server { let _ = stream.set_read_timeout(Some(self.cfg.idle_timeout)); let _ = stream.set_write_timeout(Some(self.cfg.idle_timeout)); let _ = stream.set_nodelay(true); - let (cfg, db, opts, live) = ( + let (cfg, db, opts, live, idle) = ( Arc::clone(&self.cfg), Arc::clone(&db), Arc::clone(&opts), Arc::clone(&live), + Arc::clone(&idle), ); std::thread::spawn(move || { let _guard = LiveGuard(live); - serve_connection(stream, &cfg, db.as_ref(), &opts); + serve_connection(stream, &cfg, db.as_ref(), &opts, &idle); }); } + idle.close_all(); + while live.load(Ordering::SeqCst) > 0 { + std::thread::sleep(std::time::Duration::from_millis(50)); + } Ok(()) } } +/// Connections waiting for their next request, so a server that is stopping +/// can close them rather than wait out the idle timeout. A connection in the +/// middle of a request is not in here and is left to finish it. +#[derive(Default)] +struct Idle { + closing: AtomicBool, + next_id: AtomicUsize, + waiting: Mutex>, +} + +impl Idle { + /// Block until the next request starts arriving on `reader`. `Ok(false)` + /// at end of stream, or when the server is stopping. + fn next_request(&self, id: usize, reader: &mut BufReader) -> std::io::Result { + if let Ok(s) = reader.get_ref().try_clone() { + self.lock().insert(id, s); + } + // After registering: either `close_all` sees this connection, or this + // sees the flag it set first. + let arrived = if self.closing.load(Ordering::SeqCst) { + Ok(false) + } else { + reader.fill_buf().map(|b| !b.is_empty()) + }; + self.lock().remove(&id); + arrived + } + + fn close_all(&self) { + self.closing.store(true, Ordering::SeqCst); + for s in self.lock().values() { + let _ = s.shutdown(std::net::Shutdown::Both); + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.waiting.lock().unwrap_or_else(|e| e.into_inner()) + } +} + /// Decrements the live-connection count however the handler thread ends. struct LiveGuard(Arc); @@ -159,7 +225,13 @@ enum Next { Close, } -fn serve_connection(stream: TcpStream, cfg: &IcapConfig, db: &dyn Signatures, opts: &ScanOptions) { +fn serve_connection( + stream: TcpStream, + cfg: &IcapConfig, + db: &dyn Signatures, + opts: &ScanOptions, + idle: &Idle, +) { let mut reader = BufReader::new(match stream.try_clone() { Ok(s) => s, Err(e) => { @@ -168,8 +240,15 @@ fn serve_connection(stream: TcpStream, cfg: &IcapConfig, db: &dyn Signatures, op } }); let mut writer = stream; + let id = idle.next_id.fetch_add(1, Ordering::Relaxed); for _ in 0..cfg.keepalive_requests { - match handle_request(&mut reader, &mut writer, cfg, db, opts) { + let next = idle + .next_request(id, &mut reader) + .and_then(|more| match more { + true => handle_request(&mut reader, &mut writer, cfg, db, opts), + false => Ok(Next::Close), + }); + match next { Ok(Next::KeepAlive) => {} Ok(Next::Close) => break, Err(e) => { @@ -321,12 +400,13 @@ fn modify( std::io::Read::read_exact(reader, &mut headers)?; let (body_entity, _) = enc.terminal(); - let mut body: Option = None; + let mut body: Option = None; let mut over_limit = false; let mut abandoned = false; - // Set when a spill budget refused the object, which decides the verdict - // without a scan: exav had nowhere to put the bytes, so it never saw them. + // Set when a spill budget refused the rest of the object: what was held + // is scanned, and the verdict is at best the limit. let mut rejected: Option = None; + let mut total = 0u64; if body_entity.has_body() { let mut buf = Body::new(); @@ -343,6 +423,7 @@ fn modify( }; over_limit = outcome.over_limit; abandoned = outcome.abandoned; + total = outcome.total; take_rejection(&mut rejected, &outcome); // Preview handling. A `0; ieof` terminator says the preview was the @@ -383,43 +464,26 @@ fn modify( }; over_limit = o.over_limit; abandoned = o.abandoned; + total = total.saturating_add(o.total); take_rejection(&mut rejected, &o); } - body = Some(buf.finish()?); + body = Some(Held { + payload: buf.finish()?, + short: rejected.clone(), + }); } - // The head of an over-limit object is still scanned. Refusing by size - // without looking would throw away a detection sitting in the part that did - // fit, and the engine's own size handling makes the same choice. - let mut decision = match &rejected { - // Nothing was buffered, so there is nothing to scan and no detection to - // lose by saying so. - Some(reason) => Decision::Partial("UNSCANNABLE", reason.clone()), - None => scan( - scanner, - opts, - body.as_ref(), - &headers, - enc.span(Entity::ReqHdr), - ), - }; - if over_limit && rejected.is_none() && !matches!(decision, Decision::Infected(_)) { - // The same ceiling, reported with the same words, as a stream this size - // arriving at the clamd listener: it is one scan setting, not a property - // of the port the object came in on. - decision = Decision::Partial( - "LIMITS-EXCEEDED", - format!("size exceeds {}", opts.max_scan_size.unwrap_or(0)), - ); - } - // Both partials above were decided here rather than by the engine, so - // neither has been through the policy `decide` applies. Run it on the - // finished decision, which is the only place every partial passes through: - // otherwise `--partial-as limits-exceeded=found` would turn an over-size - // object into a detection when the engine reported the limit and leave it a - // bare block when this listener did, for the same object and the same - // reason. Idempotent — a decision already remapped is no longer `Partial`. - decision = apply_found_policy(decision); + // The scan every other entry point runs: the part of an object over + // `--max-input-bytes`, or past what the spill budgets could hold, is + // scanned, and the limit is reported only with no detection in it. + let decision = scan( + scanner, + opts, + body.as_ref(), + total, + &headers, + enc.span(Entity::ReqHdr), + ); // A deployment can choose to take delivery of an object exav could not // fully examine. Whether that choice can be honoured for *this* object is a @@ -475,7 +539,7 @@ fn modify( .map(|(s, e)| &headers[s..e]) .unwrap_or(&[]); Ok(Some( - response::echo(hdr_entity, hdr, body).noting(&decision), + response::echo(hdr_entity, hdr, body.map(|h| h.payload)).noting(&decision), )) } @@ -572,14 +636,15 @@ fn apply_found_policy(decision: Decision) -> Decision { fn scan( scanner: &Scanner, opts: &ScanOptions, - body: Option<&StreamPayload>, + body: Option<&Held>, + size: u64, headers: &[u8], req_hdr: Option<(usize, usize)>, ) -> Decision { - let Some(payload) = body else { + let Some(held) = body else { return Decision::Clean; }; - if payload.len() == 0 { + if size == 0 { return Decision::Clean; } let owned = named_opts(opts, headers, req_hdr); @@ -591,11 +656,11 @@ fn scan( // something was slow. let timer = crate::metrics::ScanTimer::start(); let decision = decide(std::panic::catch_unwind(std::panic::AssertUnwindSafe( - || crate::daemon::scan_payload(scanner, named, payload), + || crate::daemon::scan_held(scanner, named, held, size), ))); timer.finish( named.filename.as_deref().unwrap_or(""), - payload.len(), + held.payload.len(), decision.category(), ); decision diff --git a/crates/exav/src/main.rs b/crates/exav/src/main.rs index f76b3d16..d11990e2 100644 --- a/crates/exav/src/main.rs +++ b/crates/exav/src/main.rs @@ -662,8 +662,9 @@ struct Cli { )] max_jobs_per_worker: Option, - /// Largest top-level input exav will scan. A larger file is reported - /// LIMITS-EXCEEDED (never a silent OK, unlike ClamAV). K/M/G/T suffixes; + /// Largest top-level input exav will scan. A larger one has its first bytes + /// scanned and is reported LIMITS-EXCEEDED unless they hold a detection + /// (never a silent OK, unlike ClamAV). K/M/G/T suffixes; /// `0` means no limit. exav default: no limit. `--clamav-compat` sets 100M. #[arg( long = "max-input-bytes", @@ -721,14 +722,15 @@ struct Cli { max_spill_bytes: Option, /// The most temp space every in-flight object may occupy **together**, - /// across the whole process. K/M/G/T suffixes; `0` means no limit. + /// within one process. K/M/G/T suffixes; `0` means no limit. /// [default: 8G] /// /// The one a per-object cap cannot stand in for: a hundred connections at /// 2G each is a 200G worst case, and filling the temp filesystem is a denial /// of service against the host that outlives the connection causing it. /// Size it against the free space on --spill-dir, not against the object - /// size you expect. + /// size you expect. Under the worker pool each worker and the ICAP child + /// counts separately, so the pool as a whole may use this much per process. #[arg( long = "max-total-spill-bytes", value_name = "SIZE|off", @@ -754,8 +756,13 @@ struct Cli { /// member/sub-container, an LZ window, a decrypted blob) may use. Every /// forced-materialization site obeys it. Not a cap on total /// memory: several buffers are live at once across nesting levels, and - /// `--max-extracted-bytes` is what bounds their sum. K/M/G/T suffixes. exav - /// default: 256M. + /// `--max-extracted-bytes` is what bounds their sum. K/M/G/T suffixes; + /// `off` or `0` means no limit. exav default: 256M. + /// + /// It is also the largest file the full signature engine can scan, and it + /// wins over `--max-extracted-bytes` when both are given. A larger file is + /// matched against literal signatures and whole-file hashes only, and + /// reported LIMITS-EXCEEDED unless one of those matches. #[arg( long = "max-object-bytes", env = "EXAV_MAX_OBJECT_BYTES", @@ -769,8 +776,8 @@ struct Cli { /// **CPU/time** bound, NOT a memory bound — streamed members are scanned /// without being held in RAM (that is capped by --max-object-bytes), so this /// can be set far higher to fully scan multi-gigabyte members, paying only in - /// scan time. Guards re-scanning/decompression-time bombs. K/M/G/T suffixes. - /// exav default: 10G. + /// scan time. Guards re-scanning/decompression-time bombs. K/M/G/T suffixes; + /// `off` or `0` means no limit. exav default: 10G. #[arg( long = "max-matcher-bytes", env = "EXAV_MAX_MATCHER_BYTES", @@ -779,6 +786,18 @@ struct Cli { )] max_scanned_bytes: Option, + /// Most x86 instructions the PE unpacking emulator may run across one + /// top-level file, summed over every packed executable it contains. A scan + /// that runs out is LIMITS-EXCEEDED. `off` or `0` means no limit. exav + /// default: 1000000000 (seconds to tens of seconds of CPU). + #[arg( + long = "max-pe-emulation-steps", + env = "EXAV_MAX_PE_EMULATION_STEPS", + value_name = "N|off", + value_parser = parse_count + )] + max_pe_emulation_steps: Option, + /// Maximum nesting depth for recursive unpacking — a zip inside a tar /// inside a disk image. exav default: 16. `--clamav-compat` sets 17. /// @@ -1829,7 +1848,7 @@ fn main() -> ExitCode { } for path in &cli.paths { match path.to_str() { - Some("-") => scan_stdin(&db, &cli, &mut totals), + Some("-") => scan_stdin(&db, &opts, &cli, &mut totals), #[cfg(feature = "http-scan")] Some(s) if s.starts_with("http://") || s.starts_with("https://") => { if cli.allow_http_scan { @@ -2174,7 +2193,8 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { // per-object cap and the core-side structural buffer (deep_analysis_max) // together, so one knob governs the largest single allocation on every // materialization path. - if let Some(b) = cli.max_buffer_bytes { + let no_limit_at_zero = |n: u64| if n == 0 { u64::MAX } else { n }; + if let Some(b) = cli.max_buffer_bytes.map(no_limit_at_zero) { opts.limits.max_buffer_bytes = b; opts.deep_analysis_max = b; } @@ -2182,9 +2202,12 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { // from memory — a streamed member is bounded by this, not by the buffer cap, // so raising it scans larger members (in RAM bounded by // --max-object-bytes) at the cost of scan time only. - if let Some(s) = cli.max_scanned_bytes { + if let Some(s) = cli.max_scanned_bytes.map(no_limit_at_zero) { opts.limits.max_scanned_bytes = s; } + if let Some(n) = cli.max_pe_emulation_steps.map(no_limit_at_zero) { + opts.limits.max_pe_emulation_steps = n; + } // --max-unpack-depth: nesting depth. exav default 16; compat 17. if let Some(r) = cli.max_recursion.or_else(|| compat.then_some(17)) { opts.limits.max_recursion = r; @@ -2620,8 +2643,8 @@ fn guard_not_empty(db: Scanner, source: &str, allow_no_db: bool) -> Result Result { - // `--clamav-compat` selects exact ClamAV naming: the `.UNOFFICIAL` suffix and - // `YARA.` prefix on signatures from an unofficial database. Provenance is + // `--clamav-compat` selects exact ClamAV naming: the `.UNOFFICIAL` suffix on + // signatures from an unofficial database. Provenance is // recorded per signature at load, always; the suffix itself is applied at // report time from `ScanOptions::unofficial_suffix`, so one loaded database // serves both compat and non-compat scans. @@ -2749,12 +2772,105 @@ fn scan_volume_sets( } } +/// Where one object's bytes are. A regular file is scanned in place; stdin +/// and a FIFO given as a path cannot seek, so they are held first (in RAM, +/// then a spill file), as the daemon holds a stream. Either way the object +/// gets the one scan. +enum Input<'a> { + File(&'a Path), + /// What was held, and the stream's size. + Held(daemon::Held, u64), +} + +impl<'a> Input<'a> { + fn open(path: &'a Path, opts: &ScanOptions) -> io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::FileTypeExt; + if std::fs::metadata(path)?.file_type().is_fifo() { + return Self::hold(std::fs::File::open(path)?, opts); + } + } + #[cfg(not(unix))] + let _ = opts; + Ok(Self::File(path)) + } + + /// Hold a stream up to one byte past `--max-input-bytes`, which is all the + /// scan looks at. What is left is read only to learn the stream's size, + /// up to the daemon's drain cap, so a stream that never ends cannot hold + /// the scan. + fn hold(mut r: impl io::Read, opts: &ScanOptions) -> io::Result { + let limit = opts.max_scan_size.map_or(u64::MAX, |m| m.saturating_add(1)); + let held = daemon::buffer_to_seekable(&mut io::Read::take(&mut r, limit))?; + let mut size = held.payload.len(); + if held.short.is_some() || size >= limit { + size += io::copy( + &mut io::Read::take(&mut r, daemon::MAX_DRAIN_BYTES), + &mut io::sink(), + )?; + } + Ok(Self::Held(held, size)) + } + + fn size(&self) -> u64 { + match self { + Self::File(p) => std::fs::metadata(p).map(|m| m.len()).unwrap_or(0), + Self::Held(_, n) => *n, + } + } + + fn scan(&self, db: &Scanner, opts: &ScanOptions) -> io::Result { + match self { + Self::File(p) => scan_path(db, p, opts), + Self::Held(held, n) => daemon::scan_held(db, opts, held, *n).map(|(r, _)| r), + } + } + + /// The object's bytes, or `None` when it is larger than `cap` or was not + /// held whole. + fn bytes_capped(&self, cap: u64) -> io::Result>> { + match self { + Self::File(p) => { + let mut data = Vec::new(); + io::Read::read_to_end( + &mut io::Read::take(std::fs::File::open(p)?, cap.saturating_add(1)), + &mut data, + )?; + Ok((data.len() as u64 <= cap).then_some(data)) + } + Self::Held(held, n) if held.short.is_none() && *n <= cap => { + held.payload.bytes_capped(cap) + } + Self::Held(..) => Ok(None), + } + } +} + fn scan_one(path: &Path, db: &Scanner, opts: &ScanOptions, cli: &Cli, totals: &mut Totals) { + match Input::open(path, opts) { + Ok(input) => scan_input(path, &input, db, opts, cli, totals), + Err(e) => { + totals.scanned += 1; + report_error(&path.display().to_string(), &e.to_string(), cli, totals); + } + } +} + +/// Scan one object, named `path` in the output. +fn scan_input( + path: &Path, + input: &Input, + db: &Scanner, + opts: &ScanOptions, + cli: &Cli, + totals: &mut Totals, +) { if cli.allmatch { - return scan_one_allmatch(path, db, opts, cli, totals); + return scan_one_allmatch(path, input, db, opts, cli, totals); } totals.scanned += 1; - let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + let size = input.size(); totals.data_scanned += size; // Isolate each file: a parser panic on a crafted input must not abort // the whole run, and must count as an error — never a clean result. @@ -2763,7 +2879,7 @@ fn scan_one(path: &Path, db: &Scanner, opts: &ScanOptions, cli: &Cli, totals: &m exav_core::profile::enable(); } let mut scanned = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| scan_path(db, path, opts))); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| input.scan(db, opts))); if cli.profile { let prof = exav_core::profile::take(); // `--profile` returns before `report_result`, so the partial policy has @@ -2865,49 +2981,16 @@ fn report_error(name: &str, message: &str, cli: &Cli, totals: &mut Totals) { } } -fn scan_stdin(db: &Scanner, cli: &Cli, totals: &mut Totals) { - use std::io::Read; - totals.scanned += 1; - let opts = build_scan_options(cli); - let scanned = std::panic::catch_unwind(std::panic::AssertUnwindSafe( - || -> io::Result { - // Buffer stdin to a SEEKABLE source (RAM small / temp file large) and - // run the full container-aware scan — so `cat archive.zip | exav -` - // detects malware INSIDE the archive, like clamscan. The old - // `scan_stream` was flat and missed it (and ignored `opts`). - let max = opts.max_scan_size; - let limit = max.map(|m| m.saturating_add(1)).unwrap_or(u64::MAX); - let stdin = io::stdin(); - let mut capped = stdin.lock().take(limit); - let payload = match daemon::buffer_to_seekable(&mut capped) { - Ok(p) => p, - // Nowhere to put it is not "nothing found in it". - Err(spill::SpillError::Budget(reason)) => { - return Ok(exav_core::ScanReport::new( - exav_core::Verdict::Unscannable { reason }, - Vec::new(), - )) - } - Err(spill::SpillError::Io(e)) => return Err(e), - }; - if let Some(m) = max { - if payload.len() > m { - return Ok(exav_core::ScanReport::new( - exav_core::Verdict::LimitsExceeded { - reason: format!("stdin exceeds max-input-bytes {m}"), - }, - Vec::new(), - )); - } - } - let (report, _loc) = daemon::scan_payload(db, &opts, &payload)?; - Ok(report) - }, - )); - match scanned { - Ok(Ok(report)) => report_result("stdin", report, cli, totals), - Ok(Err(e)) => report_error("stdin", &e.to_string(), cli, totals), - Err(_) => report_error("stdin", "internal error while scanning", cli, totals), +/// Scan stdin: held like any stream, then the same per-object path as a file, +/// so `--all-matches`, `--profile` and the totals apply to it too. +fn scan_stdin(db: &Scanner, opts: &ScanOptions, cli: &Cli, totals: &mut Totals) { + let name = Path::new("stdin"); + match Input::hold(io::stdin().lock(), opts) { + Ok(input) => scan_input(name, &input, db, opts, cli, totals), + Err(e) => { + totals.scanned += 1; + report_error("stdin", &e.to_string(), cli, totals); + } } } @@ -3874,22 +3957,22 @@ impl ReadWrite for std::net::TcpStream { } } -/// `--all-matches` scan of one file: report every matching signature. Works on a -/// buffered copy (bounded by deep-analysis-max); a larger file falls back to a -/// normal single-match scan so it is never silently skipped. +/// `--all-matches` scan of one object: report every matching signature. Works +/// on a buffered copy (bounded by deep-analysis-max); a larger object falls +/// back to a normal single-match scan so it is never silently skipped. fn scan_one_allmatch( path: &Path, + input: &Input, db: &Scanner, opts: &ScanOptions, cli: &Cli, totals: &mut Totals, ) { - use std::io::Read; totals.scanned += 1; // Counted here as well as on the single-match path: a summary reporting // "Data scanned: 0.00 MB" for a multi-megabyte archive reads exactly like a // scan that skipped its contents, and is expensive to tell apart from one. - totals.data_scanned += std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + totals.data_scanned += input.size(); let name = path.display().to_string(); // Two ceilings, one fallback. `deep_analysis_max` is how much this path can // buffer; `max_scan_size` is how much the operator said may be scanned at @@ -3902,22 +3985,19 @@ fn scan_one_allmatch( let cap = opts.max_scan_size.map_or(opts.deep_analysis_max, |max| { opts.deep_analysis_max.min(max) }); - let mut data = Vec::new(); - let read = std::fs::File::open(path).and_then(|f| { - f.take(cap.saturating_add(1)) - .read_to_end(&mut data) - .map(|_| ()) - }); - if let Err(e) = read { - report_error(&name, &e.to_string(), cli, totals); - return; - } - if data.len() as u64 > cap { + let data = match input.bytes_capped(cap) { + Ok(d) => d, + Err(e) => { + report_error(&name, &e.to_string(), cli, totals); + return; + } + }; + let Some(data) = data else { // Too big for all-match; fall back to a single-match scan, which scans // the budgeted prefix before reporting the limit, so a detection in the // part that did fit still wins. let scanned = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| scan_path(db, path, opts))); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| input.scan(db, opts))); match scanned { Ok(Ok(report)) => report_result(&name, report, cli, totals), // An I/O failure has a cause worth printing; only a panic is truly @@ -3927,7 +4007,7 @@ fn scan_one_allmatch( Err(_) => report_error(&name, "internal error while scanning", cli, totals), } return; - } + }; let found = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { exav_core::analyze_all_with_outcome(db, &data, opts) })); @@ -4303,6 +4383,15 @@ fn parse_limit_secs(s: &str) -> Result { .map_err(|_| format!("expected a number of seconds or `off`, got `{s}`")) } +/// A count, or `off`, which reads as `0`. +fn parse_count(s: &str) -> Result { + if s.eq_ignore_ascii_case("off") { + return Ok(0); + } + s.parse::() + .map_err(|_| format!("expected a number or `off`, got `{s}`")) +} + /// [`parse_limit_secs`] for a flag whose field is a `u32`. fn parse_limit_secs_u32(s: &str) -> Result { let n = parse_limit_secs(s)?; @@ -4785,6 +4874,8 @@ mod tests { assert_eq!(opts.limits.max_recursion, 16); assert_eq!(opts.limits.max_members, 100_000); + assert_eq!(opts.limits.max_pe_emulation_steps, 1_000_000_000); + // The --clamav-compat preset supplies ClamAV's documented defaults. let opts = build_scan_options(&Cli::parse_from(["exav", "--clamav-compat"])); assert_eq!(opts.max_scan_size, Some(100 * 1024 * 1024)); @@ -4793,6 +4884,33 @@ mod tests { assert_eq!(opts.limits.max_members, 10_000); } + /// `off` (and `0`) on a limit is "no limit", never a limit of zero. + #[test] + fn off_lifts_a_limit() { + let _env = env_guard(); + let opts = build_scan_options(&Cli::parse_from(["exav", "--max-pe-emulation-steps", "7"])); + assert_eq!(opts.limits.max_pe_emulation_steps, 7); + for off in ["off", "0"] { + let opts = build_scan_options(&Cli::parse_from([ + "exav", + "--max-object-bytes", + off, + "--max-matcher-bytes", + off, + "--max-pe-emulation-steps", + off, + ])); + assert_eq!(opts.limits.max_buffer_bytes, u64::MAX, "{off}: object"); + assert_eq!(opts.deep_analysis_max, u64::MAX, "{off}: object (deep)"); + assert_eq!(opts.limits.max_scanned_bytes, u64::MAX, "{off}: matcher"); + assert_eq!( + opts.limits.max_pe_emulation_steps, + u64::MAX, + "{off}: emulation" + ); + } + } + /// Pins the ClamAV-compatibility surface documented by the flag matrix /// (`www/src/content/docs/reference/clamav-flag-matrix.md`). /// diff --git a/crates/exav/src/policy.rs b/crates/exav/src/policy.rs index 15c6c31a..2524140c 100644 --- a/crates/exav/src/policy.rs +++ b/crates/exav/src/policy.rs @@ -38,11 +38,11 @@ pub(crate) fn current() -> PartialAs { /// Apply the policy to a finished report, in place. /// -/// The single point where `pass` and `alert` take effect, so a partial +/// The single point where `ok` and `found` take effect, so a partial /// object reaches an exit code, a `clamd` reply, a JSON record and a summary /// counter having already been through it. Doing it per surface would be four /// chances to forget one, and forgetting the CLI's would mean an object the -/// operator asked to pass still exiting 2. +/// operator asked to pass still exiting 3. /// /// `found` is a no-op for the conditions the engine names itself /// (`--partial-as password-protected=found` becomes `Heuristics.Encrypted.*` diff --git a/crates/exav/src/spill.rs b/crates/exav/src/spill.rs index 881b3d0e..bc461b72 100644 --- a/crates/exav/src/spill.rs +++ b/crates/exav/src/spill.rs @@ -15,7 +15,8 @@ //! * [`SpillConfig::threshold`] — how much stays in RAM before any file exists. //! * [`SpillConfig::max_object`] — the largest single object that may be spilled. //! * [`SpillConfig::max_total`] — how much every in-flight object *together* may -//! occupy, across the whole process. +//! occupy within one process. Under the prefork pool that is per worker and +//! per ICAP child, not across the pool. //! //! The third is the one a per-object limit cannot replace. A listener serving a //! hundred connections with a 2 GiB per-object cap has a 200 GiB worst case, and diff --git a/crates/exav/tests/daemon_client.rs b/crates/exav/tests/daemon_client.rs index 11d2fb04..5a2d4df5 100644 --- a/crates/exav/tests/daemon_client.rs +++ b/crates/exav/tests/daemon_client.rs @@ -461,7 +461,7 @@ fn an_unexaminable_stream_is_partial_over_the_wire_not_a_hard_error() { let (code, out) = d.client(&["--send-as", "contents"], &[&big]); assert!( - out.contains("UNSCANNABLE"), + out.contains("LIMITS-EXCEEDED"), "the category has to survive the trip: {out}" ); assert!( @@ -471,6 +471,179 @@ fn an_unexaminable_stream_is_partial_over_the_wire_not_a_hard_error() { assert_eq!(code, 3, "which is exit 3, not 2: {out}"); } +/// A job cut off by `--max-scan-secs` is answered, not closed in silence: a +/// connection that ends with no reply reads as a clean scan to some clients. +#[test] +fn a_timed_out_job_gets_an_answer() { + use std::io::{Read, Write}; + let d = Daemon::start("077", &["--max-scan-secs", "1"]); + let mut s = std::os::unix::net::UnixStream::connect(&d.sock).unwrap(); + s.write_all(b"zINSTREAM\0").unwrap(); + s.write_all(&4u32.to_be_bytes()).unwrap(); + s.write_all(b"abcd").unwrap(); + // Never finished: the job runs into its wall-clock limit. + s.set_read_timeout(Some(Duration::from_secs(30))).unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + let reply = String::from_utf8_lossy(&out); + assert!( + reply.contains("LIMITS-EXCEEDED ERROR"), + "no verdict for a job the timer stopped: {reply:?}" + ); +} + +/// `EXINSTREAM` answers a stream it could not hold as `INSTREAM` does: partial, +/// over a limit, not an error. `MULTI` answers it for that file alone. +#[test] +fn an_unexaminable_exinstream_is_partial_too() { + use std::io::{Read, Write}; + let d = Daemon::start( + "077", + &["--spill-dir", "off", "--spill-threshold-bytes", "1M"], + ); + let big = vec![b'A'; 4 << 20]; + let chunks = |body: &[u8]| { + let mut m = Vec::new(); + for c in body.chunks(1 << 20) { + m.extend_from_slice(&(c.len() as u32).to_be_bytes()); + m.extend_from_slice(c); + } + m.extend_from_slice(&0u32.to_be_bytes()); + m + }; + let ask = |msg: &[u8]| { + let mut s = std::os::unix::net::UnixStream::connect(&d.sock).unwrap(); + s.write_all(msg).unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + let text = String::from_utf8_lossy(&out) + .trim_end_matches('\0') + .to_string(); + serde_json::from_str::(&text) + .unwrap_or_else(|e| panic!("not JSON ({e}): {text}")) + }; + + let one = ask(&[&b"zEXINSTREAM\0"[..], &chunks(&big)].concat()); + assert_eq!(one["status"], "PARTIAL", "{one}"); + assert_eq!(one["category"], "LIMITS-EXCEEDED", "{one}"); + + let mut multi = b"zEXINSTREAM MULTI\0".to_vec(); + for (name, body) in [("big.bin", &big[..]), ("eicar.txt", eicar())] { + multi.extend_from_slice(&(name.len() as u32).to_be_bytes()); + multi.extend_from_slice(name.as_bytes()); + multi.extend_from_slice(&chunks(body)); + } + multi.extend_from_slice(&0u32.to_be_bytes()); + let many = ask(&multi); + assert_eq!(many["files"][0]["category"], "LIMITS-EXCEEDED", "{many}"); + assert_eq!(many["files"][1]["status"], "FOUND", "{many}"); +} + +/// A stream the daemon turns down before scanning (over `--max-input-bytes`, +/// or with nowhere to hold it) is a partial like any other, so `--partial-as` +/// decides it on `INSTREAM` and `EXINSTREAM` alike. +#[test] +fn a_stream_turned_down_follows_partial_as() { + use std::io::Write; + fn chunks(body: &[u8], terminate: bool) -> Vec { + let mut m = Vec::new(); + for c in body.chunks(1 << 20) { + m.extend_from_slice(&(c.len() as u32).to_be_bytes()); + m.extend_from_slice(c); + } + if terminate { + m.extend_from_slice(&0u32.to_be_bytes()); + } + m + } + fn ask(d: &Daemon, verb: &str, body: &[u8], terminate: bool) -> String { + let mut s = std::os::unix::net::UnixStream::connect(&d.sock).unwrap(); + s.write_all(format!("z{verb}\0").as_bytes()).unwrap(); + s.write_all(&chunks(body, terminate)).unwrap(); + s.shutdown(std::net::Shutdown::Write).unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + String::from_utf8_lossy(&out) + .trim_end_matches('\0') + .to_string() + } + fn multi(name: &str, body: &[u8]) -> Vec { + let mut m = (name.len() as u32).to_be_bytes().to_vec(); + m.extend_from_slice(name.as_bytes()); + m.extend_from_slice(&chunks(body, true)); + m.extend_from_slice(&0u32.to_be_bytes()); + m + } + fn ask_multi(d: &Daemon, body: &[u8]) -> String { + let mut s = std::os::unix::net::UnixStream::connect(&d.sock).unwrap(); + s.write_all(b"zEXINSTREAM MULTI\0").unwrap(); + s.write_all(&multi("f.bin", body)).unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + String::from_utf8_lossy(&out) + .trim_end_matches('\0') + .to_string() + } + let big = vec![b'A'; 3 << 20]; + let limits = [ + "--max-input-bytes", + "2M", + "--spill-dir", + "off", + "--spill-threshold-bytes", + "4M", + ]; + let spill = ["--spill-dir", "off", "--spill-threshold-bytes", "1M"]; + + // Default policy: the flag the limit comes from is the one exav has. + let d = Daemon::start("077", &limits); + let line = ask(&d, "INSTREAM", &big, true); + assert!( + line.contains("max-input-bytes") && line.ends_with("LIMITS-EXCEEDED ERROR"), + "{line}" + ); + let json = ask(&d, "EXINSTREAM", &big, true); + assert!(json.contains("max-input-bytes"), "{json}"); + assert!(!json.contains("max-filesize"), "{json}"); + drop(d); + + for (policy, instream, status) in [ + ("ok", "stream: OK", "\"status\":\"OK\""), + // The name a file over the limit gets, since it is the same scan. + ( + "found", + "stream: Heuristics.Limits.Exceeded.MaxFileSize FOUND", + "\"status\":\"FOUND\"", + ), + ] { + let d = Daemon::start("077", &[&limits[..], &["--partial-as", policy]].concat()); + assert_eq!(ask(&d, "INSTREAM", &big, true), instream, "{policy}"); + let json = ask(&d, "EXINSTREAM", &big, true); + assert!(json.contains(status), "{policy}: {json}"); + let json = ask_multi(&d, &big); + assert!(json.contains(status), "{policy}: {json}"); + } + + let d = Daemon::start("077", &[&spill[..], &["--partial-as", "ok"]].concat()); + assert_eq!(ask(&d, "INSTREAM", &big, true), "stream: OK"); + let json = ask(&d, "EXINSTREAM", &big, true); + assert!(json.contains("\"status\":\"OK\""), "{json}"); + let json = ask_multi(&d, &big); + assert!(json.contains("\"status\":\"OK\""), "{json}"); + + // A stream the client never finished is not an object exav could not + // examine but a request that never arrived, so no policy passes it: an + // `OK` there is a verdict for whoever hangs up after a benign head. + let line = ask(&d, "INSTREAM", b"head", false); + assert!( + line.ends_with(" ERROR") && !line.contains("UNSCANNABLE"), + "{line}" + ); + let json = ask(&d, "EXINSTREAM", b"head", false); + assert!(json.contains("\"status\":\"ERROR\""), "{json}"); + assert!(!json.contains("category"), "{json}"); +} + /// `--json` is a machine stream: the informational lines `-v` prints would be /// parse errors in it. #[test] diff --git a/crates/exav/tests/icap_server.rs b/crates/exav/tests/icap_server.rs index c1967aee..4598a4a7 100644 --- a/crates/exav/tests/icap_server.rs +++ b/crates/exav/tests/icap_server.rs @@ -774,10 +774,12 @@ fn an_object_past_the_size_limit_blocks_rather_than_passing() { assert_eq!(r.code, 200, "an unscanned object must not get a 204: {r:?}"); assert_eq!(r.header("X-Exav-Category"), Some("LIMITS-EXCEEDED")); - // Word for word what the clamd listener says about a stream this size. - // There is one size setting, and it does not answer differently depending - // on which port the object arrived at. - assert_eq!(r.header("X-Exav-Reason"), Some("size exceeds 1024")); + // Word for word what the clamd listener and a file path say about an + // object this size: it is one scan, whatever the object arrived through. + assert_eq!( + r.header("X-Exav-Reason"), + Some("file size 65536 exceeds max-input-bytes 1024; scanned first 1024 bytes only") + ); // Blocked in the c-icap vocabulary too, under a name that says which // condition blocked it rather than borrowing a database signature's. assert_eq!( @@ -1208,7 +1210,7 @@ fn an_object_with_nowhere_to_spill_gets_a_verdict_rather_than_a_dropped_connecti let r = c.recv(); assert_eq!(r.code, 200, "an object nobody buffered is not a 204: {r:?}"); - assert_eq!(r.header("X-Exav-Category"), Some("UNSCANNABLE")); + assert_eq!(r.header("X-Exav-Category"), Some("LIMITS-EXCEEDED")); assert!( r.header("X-Exav-Reason") .unwrap_or_default() @@ -1263,7 +1265,7 @@ fn no_spill_keeps_scanned_bytes_off_the_disk() { )); let r = c.recv(); assert_eq!(r.code, 200, "{r:?}"); - assert_eq!(r.header("X-Exav-Category"), Some("UNSCANNABLE")); + assert_eq!(r.header("X-Exav-Category"), Some("LIMITS-EXCEEDED")); assert!( r.header("X-Exav-Reason") .unwrap_or_default() diff --git a/crates/exav/tests/listeners.rs b/crates/exav/tests/listeners.rs index 869b8e17..7a1b516b 100644 --- a/crates/exav/tests/listeners.rs +++ b/crates/exav/tests/listeners.rs @@ -164,10 +164,6 @@ fn clamd_instream(port: u16, body: &[u8]) -> String { } /// The same, reporting a dropped connection instead of panicking on it. -/// -/// A reload re-forks the worker pool, and a connection accepted across that -/// moment is reset — a real transient a client retries, not a failure. A test -/// polling for a reloaded signature has to be able to tell the two apart. fn try_clamd_instream(port: u16, body: &[u8]) -> io::Result { let mut s = dial(port, "clamd"); s.write_all(b"zINSTREAM\0")?; @@ -449,8 +445,8 @@ fn a_rewritten_signature_directory_is_reloaded() { let deadline = Instant::now() + Duration::from_secs(60); loop { - // A connection accepted while the supervisor re-forks its pool is - // reset, so a dropped one here means "not yet", not "wrong answer". + // A worker of the old generation may still take this connection, so + // an answer without the new signature means "not yet". let after = try_clamd_instream(clamd, b"malwareD"); if after.as_deref().unwrap_or("").contains("Exav.Test.Delta") { break; @@ -463,6 +459,254 @@ fn a_rewritten_signature_directory_is_reloaded() { } } +/// A reload does not cut short a scan already in progress: the worker holding +/// it finishes the job on the database it started with, then retires. +#[test] +fn a_reload_lets_a_scan_in_progress_finish() { + let sigs = sig_dir(); + let clamd = free_port(); + let server = Server::start( + TempDir::new().unwrap(), + &[ + "--workers", + "2", + "--listen", + &format!("clamd://127.0.0.1:{clamd}"), + "--sig-dir", + sigs.path().to_str().unwrap(), + ], + ); + assert_eq!(clamd_ping(clamd), "PONG"); + + let mut s = dial(clamd, "clamd"); + s.write_all(b"zINSTREAM\0").unwrap(); + s.write_all(&(ALPHA.len() as u32).to_be_bytes()).unwrap(); + s.write_all(ALPHA).unwrap(); + + std::fs::write( + sigs.path().join("more.ndb"), + "Exav.Test.Delta:0:*:6d616c7761726544\n", + ) + .unwrap(); + wait_for_log(&server, "reloading signatures"); + // Past the moment the old generation is told to go. + std::thread::sleep(Duration::from_secs(2)); + + s.write_all(&0u32.to_be_bytes()).unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + let reply = String::from_utf8_lossy(&out); + assert!( + reply.contains("Exav.Test.Alpha FOUND"), + "the scan in progress was dropped by the reload: {reply:?}\n{}", + server.stderr() + ); +} + +/// The same for an ICAP request, served by a child of its own. +#[test] +fn a_reload_lets_an_icap_request_in_progress_finish() { + let sigs = sig_dir(); + let (clamd, icap) = (free_port(), free_port()); + let server = Server::start( + TempDir::new().unwrap(), + &[ + "--workers", + "2", + "--listen", + &format!("clamd://127.0.0.1:{clamd}"), + "--listen", + &format!("icap://127.0.0.1:{icap}"), + "--sig-dir", + sigs.path().to_str().unwrap(), + ], + ); + wait_for_log(&server, "serving ICAP on tcp:"); + + let mut s = dial(icap, "icap"); + let http = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", ALPHA.len()); + let head = format!( + "RESPMOD icap://127.0.0.1:{icap}/avscan ICAP/1.0\r\n\ + Host: 127.0.0.1\r\n\ + Connection: close\r\n\ + Encapsulated: res-hdr=0, res-body={}\r\n\r\n", + http.len() + ); + s.write_all(head.as_bytes()).unwrap(); + s.write_all(http.as_bytes()).unwrap(); + s.write_all(format!("{:x}\r\n", ALPHA.len()).as_bytes()) + .unwrap(); + s.write_all(ALPHA).unwrap(); + + std::fs::write( + sigs.path().join("more.ndb"), + "Exav.Test.Delta:0:*:6d616c7761726544\n", + ) + .unwrap(); + wait_for_log(&server, "reloading signatures"); + std::thread::sleep(Duration::from_secs(2)); + + s.write_all(b"\r\n0\r\n\r\n").unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + let answer = String::from_utf8_lossy(&out); + assert!( + answer.contains("Exav.Test.Alpha"), + "the ICAP request in progress was dropped by the reload: {answer:?}\n{}", + server.stderr() + ); +} + +/// Processes whose parent is `pid`. +#[cfg(target_os = "linux")] +fn children_of(pid: u32) -> usize { + std::fs::read_dir("/proc") + .unwrap() + .filter_map(|e| std::fs::read_to_string(e.ok()?.path().join("stat")).ok()) + .filter(|stat| { + // `pid (comm) state ppid ...`; comm may hold spaces, so split after it. + stat.rsplit_once(')') + .and_then(|(_, rest)| rest.split_whitespace().nth(1)?.parse::().ok()) + == Some(pid) + }) + .count() +} + +/// A reload replaces the pool, it does not add to it: the retired generation's +/// workers are not replaced as they exit. +#[cfg(target_os = "linux")] +#[test] +fn a_reload_keeps_the_pool_its_size() { + let sigs = sig_dir(); + let (clamd, icap) = (free_port(), free_port()); + let server = Server::start( + TempDir::new().unwrap(), + &[ + "--workers", + "2", + "--listen", + &format!("clamd://127.0.0.1:{clamd}"), + "--listen", + &format!("icap://127.0.0.1:{icap}"), + "--sig-dir", + sigs.path().to_str().unwrap(), + ], + ); + assert_eq!(clamd_ping(clamd), "PONG"); + wait_for_log(&server, "serving ICAP on tcp:"); + let pid = server.child.id(); + assert_eq!(children_of(pid), 3, "two workers and the ICAP child"); + // An idle keep-alive connection does not keep a retired ICAP child alive. + let _idle = dial(icap, "icap"); + for (i, name) in ["d.ndb", "e.ndb"].iter().enumerate() { + std::fs::write( + sigs.path().join(name), + format!("Exav.Test.X{i}:0:*:6d616c776172655{i}\n"), + ) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(60); + while server.stderr().matches("reloading signatures").count() <= i { + assert!(Instant::now() < deadline, "no reload:\n{}", server.stderr()); + std::thread::sleep(Duration::from_millis(50)); + } + std::thread::sleep(Duration::from_secs(2)); + assert_eq!( + children_of(pid), + 3, + "after reload {}:\n{}", + i + 1, + server.stderr() + ); + } +} + +/// A retired worker does not outlive its grace period: one held by an idle +/// session would otherwise keep serving it the old signatures until the client +/// hangs up. +#[cfg(target_os = "linux")] +#[test] +fn a_retired_worker_is_stopped_after_its_grace() { + let sigs = sig_dir(); + let clamd = free_port(); + let server = Server::start( + TempDir::new().unwrap(), + &[ + "--workers", + "2", + "--max-scan-secs", + "1", + "--listen", + &format!("clamd://127.0.0.1:{clamd}"), + "--sig-dir", + sigs.path().to_str().unwrap(), + ], + ); + assert_eq!(clamd_ping(clamd), "PONG"); + let pid = server.child.id(); + + let mut session = dial(clamd, "clamd"); + session.write_all(b"zIDSESSION\0zPING\0").unwrap(); + let mut reply = [0u8; 8]; + session.read_exact(&mut reply).unwrap(); + assert_eq!(&reply, b"1: PONG\0"); + + std::fs::write( + sigs.path().join("more.ndb"), + "Exav.Test.Delta:0:*:6d616c7761726544\n", + ) + .unwrap(); + wait_for_log(&server, "reloading signatures"); + let reloaded = Instant::now(); + // The grace is one job's time limit and a margin: well under the minute an + // idle session is otherwise allowed. + while children_of(pid) > 2 { + assert!( + reloaded.elapsed() < Duration::from_secs(30), + "the retired worker is still up:\n{}", + server.stderr() + ); + std::thread::sleep(Duration::from_millis(100)); + } + let mut rest = Vec::new(); + let _ = session.read_to_end(&mut rest); + assert!(rest.is_empty(), "{rest:?}"); +} + +/// A worker stopped in the middle of a job answers it, as one stopped by its +/// time limit does: a connection closed with no reply reads as clean to some +/// clients. +#[test] +fn a_worker_stopped_mid_job_answers() { + let sigs = sig_dir(); + let clamd = free_port(); + let server = Server::start( + TempDir::new().unwrap(), + &[ + "--workers", + "1", + "--listen", + &format!("clamd://127.0.0.1:{clamd}"), + "--sig-dir", + sigs.path().to_str().unwrap(), + ], + ); + assert_eq!(clamd_ping(clamd), "PONG"); + let mut s = dial(clamd, "clamd"); + s.write_all(b"zINSTREAM\0").unwrap(); + s.write_all(&4u32.to_be_bytes()).unwrap(); + s.write_all(b"ab").unwrap(); + std::thread::sleep(Duration::from_millis(500)); + let status = Command::new("kill") + .args(["-TERM", &server.child.id().to_string()]) + .status() + .unwrap(); + assert!(status.success()); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + let reply = String::from_utf8_lossy(&out); + assert!(reply.contains("ERROR"), "no answer: {reply:?}"); +} + /// An updater-only run with nothing to fetch from is refused. A container told /// to keep signatures current, with no source configured, would otherwise sit in /// a loop doing nothing while every health check it has says it is fine. diff --git a/crates/exav/tests/one_scan.rs b/crates/exav/tests/one_scan.rs new file mode 100644 index 00000000..04aa24e2 --- /dev/null +++ b/crates/exav/tests/one_scan.rs @@ -0,0 +1,201 @@ +//! One object, one answer, whichever way it reaches exav: a file path, stdin, +//! a FIFO, `INSTREAM` or `EXINSTREAM`. +//! +//! The signatures are wildcard ones, which only the full engine matches; the +//! constant-memory literal pass cannot carry them. A verdict that depends on +//! the entry point shows up as one of these inputs missing the match. + +#![cfg(unix)] + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +#[path = "../src/tmpfile.rs"] +mod tmpfile; +use tmpfile::TempDir; + +const SIGS: &str = "Exav.Test.Wild:0:*:57494c44??4f4e45\nExav.Test.Wild2:0:*:5345434f??44\n"; + +/// Matches `Exav.Test.Wild`. +const PAYLOAD: &[u8] = b"WILD-ONE"; +/// Matches `Exav.Test.Wild2`. +const PAYLOAD2: &[u8] = b"SECO-D"; + +fn sig_dir() -> TempDir { + let d = TempDir::new().unwrap(); + std::fs::write(d.path().join("t.ndb"), SIGS).unwrap(); + d +} + +fn exav(sigs: &Path) -> Command { + let mut c = Command::new(env!("CARGO_BIN_EXE_exav")); + c.arg("-d").arg(sigs); + c +} + +/// Exit code and stdout of one run. +fn run(mut c: Command, stdin: Option<&[u8]>) -> (i32, String) { + c.stdin(if stdin.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + let mut child = c.spawn().unwrap(); + if let Some(bytes) = stdin { + let mut pipe = child.stdin.take().unwrap(); + let bytes = bytes.to_vec(); + std::thread::spawn(move || { + let _ = pipe.write_all(&bytes); + }); + } + let out = child.wait_with_output().unwrap(); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + ) +} + +/// The verdict word of every way the CLI can be handed `blob`. +fn cli_verdicts(sigs: &Path, blob: &[u8], args: &[&str]) -> Vec<(&'static str, i32, String)> { + let dir = TempDir::new().unwrap(); + let file = dir.path().join("input"); + std::fs::write(&file, blob).unwrap(); + + let mut by_path = exav(sigs); + by_path.args(args).arg(&file); + let mut by_stdin = exav(sigs); + by_stdin.args(args).arg("-"); + + let fifo = dir.path().join("fifo"); + let made = Command::new("mkfifo").arg(&fifo).status().unwrap(); + assert!(made.success()); + let writer = { + let fifo = fifo.clone(); + let blob = blob.to_vec(); + std::thread::spawn(move || { + if let Ok(mut f) = std::fs::OpenOptions::new().write(true).open(&fifo) { + let _ = f.write_all(&blob); + } + }) + }; + let mut by_fifo = exav(sigs); + by_fifo.args(args).arg(&fifo); + + let mut out = Vec::new(); + for (how, cmd, input) in [ + ("path", by_path, None), + ("stdin", by_stdin, Some(blob)), + ("fifo", by_fifo, None), + ] { + let (code, stdout) = run(cmd, input); + out.push((how, code, stdout)); + } + let _ = writer.join(); + out +} + +/// A daemon on a socket of its own. +struct Daemon { + child: Child, + sock: PathBuf, + _dir: TempDir, +} + +impl Drop for Daemon { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +impl Daemon { + fn start(sigs: &Path, args: &[&str]) -> Self { + let dir = TempDir::new().unwrap(); + let sock = dir.path().join("d.sock"); + let child = exav(sigs) + .arg("--listen") + .arg(format!("clamd://{}", sock.display())) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(60); + while std::os::unix::net::UnixStream::connect(&sock).is_err() { + assert!(Instant::now() < deadline, "the daemon never answered"); + std::thread::sleep(Duration::from_millis(50)); + } + Self { + child, + sock, + _dir: dir, + } + } + + fn stream(&self, verb: &str, blob: &[u8]) -> String { + let mut s = std::os::unix::net::UnixStream::connect(&self.sock).unwrap(); + s.write_all(format!("z{verb}\0").as_bytes()).unwrap(); + for c in blob.chunks(64 * 1024) { + s.write_all(&(c.len() as u32).to_be_bytes()).unwrap(); + s.write_all(c).unwrap(); + } + s.write_all(&0u32.to_be_bytes()).unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + String::from_utf8_lossy(&out) + .trim_end_matches('\0') + .to_string() + } +} + +/// Over `--max-input-bytes` with the payload inside the limit: found, by the +/// full engine, on every entry point. With it past the limit: the limit, on +/// every entry point. +#[test] +fn every_entry_point_scans_the_start_of_an_oversize_input_the_same_way() { + let sigs = sig_dir(); + let limit = ["--max-input-bytes", "1K"]; + + let mut inside = PAYLOAD.to_vec(); + inside.extend(vec![b'.'; 64 * 1024]); + for (how, code, out) in cli_verdicts(sigs.path(), &inside, &limit) { + assert!(out.contains("Exav.Test.Wild FOUND"), "{how}: {out}"); + assert_eq!(code, 1, "{how}: {out}"); + } + + let mut past = vec![b'.'; 64 * 1024]; + past.extend_from_slice(PAYLOAD); + for (how, code, out) in cli_verdicts(sigs.path(), &past, &limit) { + assert!(out.contains("LIMITS-EXCEEDED"), "{how}: {out}"); + assert_eq!(code, 3, "{how}: {out}"); + } + + let d = Daemon::start(sigs.path(), &limit); + let line = d.stream("INSTREAM", &inside); + assert_eq!(line, "stream: Exav.Test.Wild FOUND"); + let json = d.stream("EXINSTREAM", &inside); + assert!(json.contains("Exav.Test.Wild"), "{json}"); + let line = d.stream("INSTREAM", &past); + assert!(line.contains("max-input-bytes"), "{line}"); + assert!(line.ends_with("LIMITS-EXCEEDED ERROR"), "{line}"); +} + +/// Options that only the file path used to honour reach stdin too. +#[test] +fn stdin_gets_the_options_a_file_gets() { + let sigs = sig_dir(); + let mut both = PAYLOAD.to_vec(); + both.extend_from_slice(b" and "); + both.extend_from_slice(PAYLOAD2); + for (how, code, out) in cli_verdicts(sigs.path(), &both, &["--all-matches"]) { + assert!( + out.contains("Exav.Test.Wild") && out.contains("Exav.Test.Wild2"), + "{how}: {out}" + ); + assert_eq!(code, 1, "{how}: {out}"); + } +} diff --git a/docs/BYTECODE.md b/docs/BYTECODE.md index 1545bdc2..58c42d8c 100644 --- a/docs/BYTECODE.md +++ b/docs/BYTECODE.md @@ -233,3 +233,36 @@ strictly-bounded sandbox is invariant throughout. Phases A–E (~6–8 weeks) reach ~78% coverage with real, safe detections; the `disasm_x86` long tail (Phase F) is the bulk of the remainder. + +## Host APIs: 34 of 107 + +ClamAV's host-API table has 107 entries, byte-identical in 1.4.3 and 1.5.3. exav +implements 34. + +The `maxapi 89` in debug output is not the table size: it is a per-program +declaration in each `.cbc` header, and 89 is the highest value any shipped +program declares. So indices 0–89 are the reachable surface: exav implements 34 +of those 90, and 56 are unimplemented. Indices 90–106 (JSON accessors, LZMA and +bzip2 stream contexts, `get_file_reliability`, `engine_scan_options_ex`) exist in +ClamAV, but no program in the shipped database references any of them. + +| Group | Missing | APIs | +|---|---|---| +| Buffer pipes | 9 | `buffer_pipe_new`, `_new_fromfile`, `_read_avail`, `_read_get`, `_read_stopped`, `_write_avail`, `_write_get`, `_write_stopped`, `_done` | +| Debug / trace | 10 | `debug_print_str`, `debug_print_uint`, `debug_print_str_start`, `debug_print_str_nonl`, `trace_directory`, `trace_scope`, `trace_source`, `trace_op`, `trace_value`, `trace_ptr` | +| Engine / environment | 9 | `bytecode_rt_error`, `engine_scan_options`, `engine_db_options`, `extract_set_container`, `input_switch`, `disable_bytecode_if`, `disable_jit_if`, `check_platform`, `running_on_jit` | +| Maps | 8 | `map_new`, `map_addkey`, `map_setvalue`, `map_remove`, `map_find`, `map_getvaluesize`, `map_getvalue`, `map_done` | +| Hashsets | 6 | `hashset_new`, `_add`, `_remove`, `_contains`, `_done`, `_empty` | +| PDF extras | 6 | `pdf_get_obj_num`, `pdf_set_flags`, `pdf_getobj`, `pdf_getobjflags`, `pdf_setobjflags`, `pdf_get_dumpedobjid` | +| Decompression | 3 | `inflate_init`, `inflate_process`, `inflate_done` | +| JS normalisation | 3 | `jsnorm_init`, `jsnorm_process`, `jsnorm_done` | +| Self-test | 2 | `test1`, `test2` | +| **Total** | **56** | of the 90 reachable indices | + +At the time of writing all 85 bytecode programs in the official database run to +completion, and across 400 real malware samples one stub was reached +(`get_environment`). A program declares a `maxapi` ceiling; it does not call +every API below it. Some of the 56 are unlikely to matter (the debug and trace +groups are compile-time instrumentation; `running_on_jit` and `disable_jit_if` +have an obvious answer on an engine with no JIT). The pipes, maps and `inflate` +groups would be real work if a future program used them. diff --git a/docs/COMPARISON_NOTES.md b/docs/COMPARISON_NOTES.md new file mode 100644 index 00000000..fe1d1c38 --- /dev/null +++ b/docs/COMPARISON_NOTES.md @@ -0,0 +1,106 @@ +# Comparison with ClamAV: measurements and corrections + +Moved out of the website's comparison page, which describes behaviour. This file +keeps the dated measurements and the record of claims that were corrected. + +## Measured against clamd: the 2026-07-28 run + +8,978 live-malware samples, both engines loading the same daily-only signature +set (339,917 signatures). Only the 5,436 files where both engines returned an +answer are counted; the rest hit a resource ceiling in the memory-capped clamd +container and measure the harness, not the engines. + +exav ran at full capability (its own extractors and limits). The harness defaults +to `--clamav-compat`, which trades reach for reproducibility; measuring exav's +added coverage in that mode understates it by roughly 3x, so these figures come +from a `COMPAT=0` pass over the same files. + +| | exav | clamd | +|---|---|---| +| detections | 897 | 796 | +| detected by this engine alone | **104** | 0 | + +104 exav-only detections: 11.6% of exav's detections, 1.91% of files scanned. (In +`--clamav-compat` the same corpus yields 32, or 3.9%; the difference is the +extraction reach compat switches off.) Both engines move, so re-measure before +quoting. + +Every one was verified. 101 of the 104 report a nested match location: the hit is +inside a member clamd did not unpack, and re-scanning exav's own extracted +members with clamd (see `exav-unpack/examples/dump_members.rs`) makes clamd flag +them under the same signature name while still calling the container clean. Of +the other three, two are `Target:0` RTF-exploit signatures legitimately matching +RTF files, and one is `Win.Trojan.Mimikatz` inside a PE that was base64-encoded +inside an RTF: none of the signature's seven subsignatures occurs in the raw +file, and all seven occur in the decoded image. None was a false positive. + +Across all 5,436 comparable files, the `FN` count (clamd detected, exav returned +`OK`) was zero, and the `CAREFUL_FN` count (clamd names the malware, exav +declines to call the file clean but cannot name it) was also zero once the two +samples that sat there (an MPRESS-packed dropper and a UPX image carrying a bare +`PackHeader`) were unpacked. + +A later code review found silent-miss paths the corpus had not exercised +(container hashes and raw signatures over `INSTREAM`/stdin/ICAP, an ignored +signature masking a later member, ZIP members hidden from the central directory, +files over the deep-analysis limit reported clean). They were fixed with tests; +this run predates those fixes. + +### Why the run makes no speed claim + +The differential harness is a compliance harness: it answers "do the two engines +agree", and is tuned for throughput so the question can be asked often. Its +timings exist to spot a wedged file or a much slower engine, and they are +confounded in at least four ways: + +- **Scan ordering.** The engines run in separate phases over the same corpus, + clamd first. Whatever runs second reads a partly warmed page cache, and a warm + read is worth roughly 10x a cold one. That favours exav. +- **Concurrency contention.** Under more than one job, a file's elapsed time is + mostly time spent competing with other jobs. +- **Unequal environments.** clamd ran memory-capped inside Docker; exav ran + natively on the host. +- **Different work per file.** The compat and full-capability passes do not + extract the same amount. + +An earlier revision of the website quoted a mean/median table from such a run +and concluded a 1.27x speed ratio. That was not supportable from the data it +cited, so it was withdrawn. A speed claim needs a dedicated benchmark (one job, +warm cache, repeated runs, equal environments). + +## Corrections + +### RAR multi-volume is not a gap + +An earlier revision of the comparison page listed RAR volume joining as a +capability gap. One verification pass had tested it on a host whose ClamAV build +ships no unrar library at all, so every RAR returned `OK`, which says nothing +about volume joining. A second pass on 1.5.3 reported the opposite. + +Settled with a control: a single-volume RAR containing EICAR is detected by +ClamAV 1.5.3 (so RAR support is live), while a three-volume set whose payload +lives only in the last volume returns `OK` when volume 1 is scanned with all +siblings present. ClamAV does not join RAR volumes. + +exav reports the split member and hands over the part that is in the scanned +volume while saying the rest is missing. Before that fix a stored split member +was emitted with no reason attached, so its prefix read as a complete member and +a multi-volume archive scanned clean. That was a silent truncation, closed for +both RAR3 and RAR5. + +### Signature-format coverage + +The count of signature lines exav could not load was 20 at one point during the +first release cycle, and reached zero. An earlier version of that list led with +"byte-compare subsignatures", which was wrong: the classifier guessed the cause +from punctuation and filed every alternation as a byte-compare, because both +contain a parenthesis. Byte-compare causes zero skips. A second entry, +"malformed database line", was wrong the same way: the line loads in clamscan, +and the real cause was the `:`-suffix on a subsignature reference. + +### Things once listed as gaps + +Each of these was listed as a shortfall at some point and turned out to be a +format ClamAV does not handle either: ACE, StuffIt, Inno Setup, WIM, ZIP method +10 (DCL Implode), the PDF image filters, KWAJ, NTFS, FAT, VHD, VHDX, QCOW2, VMDK, +UDF, and YARA modules. The website's comparison page keeps the current table. diff --git a/docs/PARTIAL.md b/docs/PARTIAL.md index 63b13f66..8dfff39b 100644 --- a/docs/PARTIAL.md +++ b/docs/PARTIAL.md @@ -35,7 +35,7 @@ One line grammar, everywhere: **`path: [reason ][CATEGORY ]STATUS`**, status las ``` path: OK path: Win.Trojan.Agent-1234 FOUND -path: file size 200000 exceeds max-scan-size 1024 LIMITS-EXCEEDED PARTIAL +path: file size 200000 exceeds max-input-bytes 1024 LIMITS-EXCEEDED PARTIAL path: Can't open file ERROR ``` @@ -127,7 +127,7 @@ has reported `OK` for something it folded, the fact is gone from the wire. ### CLI `--json` ```json -{"category":"LIMITS-EXCEEDED","file":"big.bin","reason":"file size 200000 exceeds max-scan-size 1024…","status":"PARTIAL"} +{"category":"LIMITS-EXCEEDED","file":"big.bin","reason":"file size 200000 exceeds max-input-bytes 1024…","status":"PARTIAL"} ``` `category` is present only under `PARTIAL` — the other statuses have nothing to @@ -136,7 +136,7 @@ sub-classify. Key order is not part of the contract; keys serialise sorted. ### clamd wire — `SCAN` / `CONTSCAN` / `MULTISCAN` / `INSTREAM` / `FILDES` / `SCANURL` ``` -path: file size 200000 exceeds max-scan-size 1024 LIMITS-EXCEEDED ERROR +path: file size 200000 exceeds max-input-bytes 1024 LIMITS-EXCEEDED ERROR ``` Same grammar as stdout; **only the status word differs**, and it has to. See §4. diff --git a/docs/STREAMING_FORMATS.md b/docs/STREAMING_FORMATS.md new file mode 100644 index 00000000..b5433e27 --- /dev/null +++ b/docs/STREAMING_FORMATS.md @@ -0,0 +1,34 @@ +# Streaming the last three container formats + +exav optimises for memory, not bandwidth. A streamable format is walked member by +member straight off the seekable source, so peak memory is one member's window +rather than the whole decoded object. Most containers already work that way (see +`is_streamable`), including tar, zip, 7z and cab. Three do not, for different +reasons, and only two of them are worth changing. + +**DMG: the tractable one, and the next concrete step.** `decompress_udif` +materialises the entire virtual disk into one `Vec` because the HFS+/APFS +readers need random access over it, so a 4 GB image costs 4 GB of resident +memory (bounded by the peak-buffer limit, which means large images are refused +rather than scanned). The reader is already the right shape: `DmgReader` is +position-based, mapping an offset to a BLKX run and decompressing that run. Two +pieces are missing. First a decompressed-run cache: `read` currently +re-decompresses the containing run on every call, which is fine for a linear +pass and unusable under the seek storms a filesystem crate generates. Then +`impl Seek`, after which the reader can be handed to the filesystem crates +directly instead of a materialised image. Roughly 150 lines, and it converts a +whole-image allocation into one cached run. + +**RAR: real work, deliberately not rushed.** `rar3_unpack.rs` and +`rar5_unpack.rs` are vendored decoders that build the complete output `Vec`. +Streaming them means turning both into resumable `Read` state machines that +retain only the already-capped LZ window: major surgery on fuzzed, +differential-tested code, where a subtle break costs correctness on a format +attackers actively use. It is a focused effort of its own, not a change to slip +in alongside others. + +**PDF: measured, and not worth doing.** `extract_pdf` already decompresses +stream objects one at a time into a budget-bounded buffer, and the file itself +has to be buffered regardless because the xref table lives at the end and the +recovery path scans the whole file. Peak is therefore already input plus one +bounded object; streaming would save a single bounded body. diff --git a/www/src/content/docs/concepts/architecture.md b/www/src/content/docs/concepts/architecture.md index 289c604e..2220fa10 100644 --- a/www/src/content/docs/concepts/architecture.md +++ b/www/src/content/docs/concepts/architecture.md @@ -1,41 +1,35 @@ --- title: Architecture -description: How exav is organized — the two input modes, the crate layout, and how each ClamAV signature type compiles to a distinct runtime structure. +description: How exav is organized, the crate layout, and how each ClamAV signature type compiles to a distinct runtime structure. --- -exav is a small set of focused crates around a constant-memory scanning core. -This is the high-level picture; deeper implementation detail lives in the -repository's `docs/`. +exav is a small set of focused crates around one scanning engine. This is the +high-level picture; implementation detail lives in the repository's `docs/`. -## Two input modes +## Input -exav scans through one of two paths, and the difference decides what analysis is -possible: +Every input is scanned through a seekable source (`Read + Seek`): a local file, +an HTTP range reader, or a buffered stream. Stdin, `INSTREAM` and ICAP bodies are +buffered first (in memory, then in a temporary file), because container formats +need to seek. `scan_seekable` types the input, walks containers through the +reader (fetching only the directory and the members it scans), and runs the +engine. See [Streaming & memory](/concepts/streaming-memory/) for what is held in +memory and what is streamed. -- **Stream** (`Read`: stdin, pipes) — `scan_stream` runs the constant-memory - pattern + hash core. Unlimited size, but structural unpacking needs seeking, so - a pure pipe does pattern+hash only. -- **Seekable** (`Read + Seek`: local files, or an HTTP range reader over HTTP - range GETs) — `scan_seekable` additionally drives archive extraction through - the reader, fetching only the directory and members it actually scans. - -See [Streaming & memory](/concepts/streaming-memory/) for the core. - -The code is split across a handful of crates — `exav` and the WASI build are -the front-ends, `exav-core` is the engine, `exav-unpack` does extraction (and -backs the standalone `exav-grep`). See +`exav` and the WASI build are the front ends, `exav-core` is the engine, and +`exav-unpack` does extraction (and backs the standalone `exav-grep`). See [How the crates compose](#how-the-crates-compose) below, and [Subprojects](/subprojects/) for what each one owns. ## Not one monolithic engine -Different ClamAV signature types compile to different runtime structures — there -is deliberately **not** one giant matcher: +Different ClamAV signature types compile to different runtime structures rather +than one matcher: | Source | Runtime structure | Matching | |---|---|---| | `.ndb` bodies + `.ldb` literal subsignatures | a shared **Aho-Corasick automaton** keyed on a literal anchor per body | an automaton hit fans out to every body sharing that anchor; each candidate is then verified (wildcards / gaps / nibbles / alternation / offset / nocase) | -| `.ldb` PCRE subsignatures | `regex` objects compiled lazily, gated by the trigger expression | linear-time, DoS-safe regex | +| `.ldb` PCRE subsignatures | regexes compiled lazily, gated by the trigger expression | linear-time `regex`; patterns with backreferences use a backtracking engine under a step bound | | `.hsb` / `.hdb` | a size-keyed hash table | whole-file digest lookup | | `.mdb` / `.msb` | a section-hash table | per-PE-section digest lookup | | `.cdb` | container-metadata matchers | matched on archive members (name/size/encryption/position) | @@ -43,9 +37,10 @@ is deliberately **not** one giant matcher: | `.cbc` | a [sandboxed bytecode interpreter](/concepts/bytecode-sandbox/) (no JIT) | trigger-gated programs run on extracted buffers | | `.yar` / `.yara` | a [native YARA engine](/guides/yara/) | near-full YARA, no runtime codegen | -So the engine is really: *one Aho-Corasick automaton (fed by ndb + ldb literal -subsigs) + several cheap hash tables + lazy regex + interpreters.* Almost all of -the memory cost is the automaton — see [the database internals](/guides/prebuilt-database/). +So the engine is one Aho-Corasick automaton (fed by `.ndb` and `.ldb` literal +subsignatures), several cheap hash tables, lazy regexes and interpreters. Almost +all of the memory cost is the automaton; see the +[prebuilt database](/guides/prebuilt-database/). ## How the crates compose @@ -177,15 +172,14 @@ A scan flows through it like this: The five verdicts are the whole output vocabulary. Anything that stops a scan -early produces one of the last three rather than `Clean` — see -[Verdicts & exit codes](/reference/verdicts/). +early produces one of the last three rather than `Clean` (see +[Verdicts & exit codes](/reference/verdicts/)). -The split is not cosmetic. Extraction touches hostile bytes first and hardest, -so it lives in its own `#![forbid(unsafe_code)]` crate with its own budget and -panic containment; a malformed archive cannot reach the engine, let alone the -host. It also means you can take one piece — a build system that needs to look -inside archives does not need a virus scanner, and a reverse engineer who wants -to unpack a packed executable needs neither. +Extraction touches hostile bytes first and hardest, so it lives in its own +`#![forbid(unsafe_code)]` crate with its own budget and panic containment. It +also means one piece can be used alone: a build system that needs to look inside +archives does not need a virus scanner, and a reverse engineer who wants to +unpack a packed executable needs neither. `exav-pe-emu` sits below the extractor for the same reason. Running a packer's stub is the one place where the scanner executes attacker-authored *control flow* @@ -199,13 +193,12 @@ instructions, API, and examples. The `exav-unpack` crate recursively extracts archives and structured documents (see [Supported formats](/reference/formats/)). Every path runs under -**decompression-bomb defenses**: output-byte, ratio, file-count, recursion-depth, -and cumulative scan-byte budgets. A bomb is `LIMITS-EXCEEDED`, never `OK`; a -member with an unsupported codec or encryption is `UNSCANNABLE` / -`PASSWORD-PROTECTED`, never silently dropped — the -[never-silent invariant](/concepts/design-principles/#never-a-silent-clean) reaching all the way down. +decompression-bomb defenses: output-byte, ratio, file-count, recursion-depth, +cumulative scan-byte and emulation-step budgets. A bomb is `LIMITS-EXCEEDED`; a +member with an unsupported codec or encryption is `UNSCANNABLE` or +`PASSWORD-PROTECTED`. ## Content-based typing -File type is decided by **magic bytes**, never by extension — an executable -renamed `.jpg` is still typed and scanned as an executable. +File type is decided by magic bytes, not by extension: an executable renamed +`.jpg` is still typed and scanned as an executable. diff --git a/www/src/content/docs/concepts/archive-extraction.md b/www/src/content/docs/concepts/archive-extraction.md index 22edd9e5..797da285 100644 --- a/www/src/content/docs/concepts/archive-extraction.md +++ b/www/src/content/docs/concepts/archive-extraction.md @@ -1,54 +1,40 @@ --- title: Archive extraction -description: How exav unpacks containers — the parity principle, bounded in-memory extraction, dual indexing, salvage, and why every member exav cannot read is reported rather than skipped. +description: How exav unpacks containers, from the parity principle and bounded in-memory extraction to dual indexing, salvage, and why a member exav cannot read is reported rather than skipped. --- -Most malware does not arrive as a bare executable. It arrives inside something: -a ZIP, an ISO, a 7z, a Word document, an installer. So a scanner's real job is -less "match bytes" than "reach the bytes", and the unpacker is where that fight -happens. +Most malware does not arrive as a bare executable. It arrives inside something: a +ZIP, an ISO, a 7z, a Word document, an installer. A scanner's real job is less +"match bytes" than "reach the bytes", and the unpacker is where that happens. -exav's extraction lives in `exav-unpack` — a standalone crate with **no -dependency on the scanning engine**, `#![forbid(unsafe_code)]`, and usable on its -own as a safe extraction library. +exav's extraction lives in `exav-unpack`, a standalone crate with no dependency +on the scanning engine, `#![forbid(unsafe_code)]`, and usable on its own as a safe +extraction library. ## The parity principle -Coverage is not a feature list, it is a boundary. **If the software on the -target's machine can open a container and exav cannot, that container is an -evasion vector by construction** — the payload lands intact and the scanner said -nothing useful about it. +If the software on the target's machine can open a container and exav cannot, +that container is an evasion vector: the payload lands intact and the scanner +said nothing useful about it. -The attacker picks the delivery format, so the target to match is the *union* of -what real extractors handle: 7-Zip, WinRAR, WinZip, The Unarchiver, Windows -Explorer, macOS Archive Utility, libarchive. Not the intersection, not the common -case, and emphatically not "what showed up in last month's corpus" — a format -appearing zero times in a sample says nothing about the file someone is sent -tomorrow. +The attacker picks the delivery format, so the target is the union of what real +extractors handle: 7-Zip, WinRAR, WinZip, The Unarchiver, Windows Explorer, macOS +Archive Utility, libarchive. Not the intersection, and not what showed up in last +month's corpus. -[Supported formats](/reference/formats/) tracks this as an audited matrix: -every container and codec those extractors open, with exav's status against each, -and [one authoritative gap list](/reference/formats/#the-complete-gap-list) of -what is still missing. +[Supported formats](/reference/formats/) tracks this as a matrix: every container +and codec those extractors open, exav's status against each, and +[the gap list](/reference/formats/#the-complete-gap-list) of what is still +missing. -## Two input modes - -What exav can do depends on whether it can seek: - -- **Stream** (stdin, a pipe) — the constant-memory pattern and hash core only. - Unlimited size, but structural unpacking needs to seek, so a pure pipe gets - pattern+hash matching over the raw bytes. -- **Seekable** (a file, or an HTTP range reader) — full recursive extraction, - fetching only the directory and the members actually scanned. - -This is why `cat archive.zip | exav -` and `exav archive.zip` are not equivalent, -and why the streaming path is honest about it rather than pretending. +Every input is extracted through a seekable source: a file, an HTTP range reader +(fetching only the directory and the members it scans), or a stream buffered +first (see [Streaming & memory](/concepts/streaming-memory/)). ## The shape of the unpacker -One sentence carries most of it: **a member is bytes plus a budget, a container -is something that yields members, and nesting is that same step applied to what -comes out.** +A member is bytes plus a budget, a container is something that yields members, +and nesting is the same step applied to what comes out. The two doors into extraction @@ -64,8 +50,8 @@ comes out.** - DOOR A — STREAMING - DOOR B — BUFFERED + DOOR A: STREAMING + DOOR B: BUFFERED @@ -86,10 +72,10 @@ comes out.** first 4 KiB holds ONE member at a time, the container is never fully resident - zip · tar · gzip · … + zip · tar · 7z · cab · iso · … needs the WHOLE container as one slice, bounded before it is allocated - 7z · rar · cab · iso · … + rar · ole · pdf · chm · … bytes + the SAME Budget one Budget for the whole tree, not per container each member re-enters @@ -105,54 +91,42 @@ comes out.** -The two doors are the thing to hold on to. `identify()` types the bytes, and -`streams_natively` decides which one opens. Door A hands back members one at a -time and never has the container resident; door B needs the whole container as a -single slice before it can start. Which door a format uses is a property of that -format's decoder, not of how you invoked exav. +`identify()` types the bytes, and `streams_natively` decides which door opens. +Door A hands back members one at a time and never has the container resident; +door B needs the whole container as one slice before it can start. Which door a +format uses is a property of its decoder, not of how exav was invoked. -Everything past the door is the same for both: a member is bytes carrying the -**same** `Budget` as its parent, and scanning it re-enters `identify()`. That is -why nesting needs no special case — and why `maxExtractedBytes` bounds an archive -tree rather than each container in it. +Past the door both are the same: a member is bytes carrying the same `Budget` as +its parent, and scanning it re-enters `identify()`. So nesting needs no special +case, and the extraction budget bounds an archive tree rather than each container +in it. ### Where the code departs from the picture -The map is clean; the territory has four bumps to know about before you read the -source. +**There are two dispatches.** `dispatch_stream` and `dispatch_extract` are +separate functions with separate format coverage. A change to one does not +appear in the other, and a format can be reachable through one door only. When a +hook or check does not fire, suspect this first. -**There are two dispatches, not one.** `dispatch_stream` and `dispatch_extract` -are separate functions with separate format coverage. A change made to one does -not appear in the other, and a format can be reachable through one door only. If -you are tracing why a hook or a check does not fire, this is the first thing to -suspect. - -**Door B is the default, not the exception.** Most formats land in it, and which -door a given one uses is answered by `streams_natively` rather than by any count -written down here. -`ArchiveInner` — the seekable `Archive` API — has native arms for `Zip`, -`Tar` and `Gzip`, a `Lazy` arm for the single-stream compressors (bzip2, xz, -zstd), and a `Buffered` arm that reads the reader into memory and hands it to -door B. That last arm is where 7z, rar, cab, iso and the rest sit. +**The `Archive` API has its own split.** `ArchiveInner` has native arms for +`Zip`, `Tar` and `Gzip`, a `Lazy` arm for the single-stream compressors (bzip2, +xz, zstd), and a `Buffered` arm that reads the source into memory and hands it to +door B, which is where every other format sits in that API. **An empty `Archive::list()` means "no directory", not "no members".** Only -formats that carry an index can be listed without extracting: a ZIP's central -directory, a tar's headers. A gzip has no index by construction — its single -member's name and size are not knowable without decompressing. Callers that -treat an empty list as an empty archive get that wrong, which is why the wasm -bindings walk the archive instead of passing the empty slice outward. +formats with an index can be listed without extracting: a ZIP's central +directory, a tar's headers. A gzip's single member has no name or size until it +is decompressed, which is why the wasm bindings walk the archive instead of +passing an empty list outward. **A ZIP has more members than its directory admits to.** Extraction also scans -the bytes the central directory does not claim, because a member with a local -header and no directory entry is hidden from every reader that walks the -directory alone — while the tool that opens the archive extracts it anyway. Both -doors do this; a listing reports those members at indices straight after the -directory's own. +the bytes the central directory does not claim (see below). Both doors do this; +a listing reports those members at indices straight after the directory's own. -## Bounded, in memory, budget-before-allocation +## Bounded, in memory, budget before allocation -Every extraction runs under a `Budget`. Five limits, all reserved *before* a -member is read rather than checked after: +Every extraction runs under one `Budget`, reserved before a member is read rather +than checked after: | Limit | Bounds | |---|---| @@ -160,14 +134,16 @@ member is read rather than checked after: | per-member size | one member's decompressed size | | compression ratio | output ÷ input, the decompression-bomb guard | | file count | members across the archive and everything nested | -| recursion depth | archives inside archives inside archives | +| recursion depth | archives inside archives | +| matcher bytes | bytes fed to the matcher across the tree, a CPU bound | +| emulation steps | instructions the PE unpacking emulator runs across the tree | -Reserving first is what makes the peak bounded rather than merely detected: a -"1 GB from 4 KB" member never allocates 1 GB and then gets rejected. +Reserving first keeps the peak bounded: a "1 GB from 4 KB" member never +allocates 1 GB and then gets rejected. -**Nothing is written to disk.** The extractor materializes members in memory -only, which removes zip-slip, path traversal and symlink attacks as a class — -there is no filesystem write for a `../../` path to escape into. +Nothing is written to disk. Members exist in memory only, so zip-slip, path +traversal and symlink attacks do not apply: there is no filesystem write for a +`../../` path to escape into. ## Containers have two indexes @@ -175,118 +151,103 @@ A ZIP lists its members twice: a local file header before each member's data, an a central directory at the end. Every normal reader trusts the central directory, because that is what the specification says is authoritative. -Which means you can hide a member: leave it out of the central directory and keep -its local header and data in the file. Plenty of extractors still write it out. +So a member can be hidden: leave it out of the central directory and keep its +local header and data in the file. Plenty of extractors still write it out. -exav does **dual indexing** — after the central-directory pass it scans the raw -bytes for local headers the directory didn't cover, and extracts those too. The -same path doubles as salvage when the central directory is corrupt, forged or +exav does dual indexing: after the central-directory pass it scans the raw bytes +for local headers the directory did not cover, and extracts those too. The same +path is the salvage route when the central directory is corrupt, forged or truncated. -The gate matters as much as the scan. `PK\x03\x04` is four bytes and occurs by -chance in ordinary binaries, so a credibility check (version, reserved flag bits, -a real compression method, a NUL-free path within the length limit) decides what -counts as a member. Across 3,000 live malware samples it produced no false hits. +`PK\x03\x04` is four bytes and occurs by chance in ordinary binaries, so a +credibility check (version, reserved flag bits, a real compression method, a +NUL-free path within the length limit) decides what counts as a member. -ISO images have the same shape and the trick is used more openly: a CD image -carries several volume descriptors, each with an independent directory tree over -the same sectors. A malicious ISO lists its payload in only the Joliet tree and -leaves the primary tree empty, so a reader that parses the other one sees an -empty disc. exav walks **every** tree. +ISO images have the same shape: a CD image carries several volume descriptors, +each with its own directory tree over the same sectors. A malicious ISO lists its +payload only in the Joliet tree and leaves the primary tree empty, so a reader +that parses the primary one sees an empty disc. exav walks every tree. ## Recovering what a naive reader would skip -Two cases where the easy behaviour is to give up, and giving up is wrong: - -**Deferred sizes.** A streaming ZIP writer sets a flag and puts the member's -sizes in a *data descriptor after the data*, so the local header alone doesn't -say where the member ends. exav recovers the extent — from the descriptor's own -`PK\x07\x08` marker when its declared size is self-consistent, otherwise by -running to the next header (only for deflate, which self-terminates; a stored -member would silently swallow the descriptor and the following headers into its -content). - -**Truncated streams.** A gzip cut short mid-DEFLATE is common in the wild, and -`zcat` happily recovers everything before the cut. Discarding that prefix would -be a real miss, because the payload is often right there in it. exav scans the -salvaged prefix, and the nesting works: a truncated `gzip(tar(...))` decompresses -the gzip, walks the intact tar entries, and scans each one. - -And if nothing matches, the verdict is **`OK`** — not "not fully scanned". exav -scans for malware, it is not a file-integrity validator. When a *sequential* -stream simply runs out of input, every byte that exists was scanned; the missing -tail is *absent*, not *hidden*. - -That distinction is the whole game, and it is why indexed formats are treated -differently: a damaged ZIP index can leave member data *present but never -enumerated*, which is content that exists and was not read. Hence dual indexing — -without it, "clean" on a damaged ZIP would be a claim about bytes nobody looked -at. - -## A member exav cannot read is reported, never dropped - -This is the rule the whole design bends around. When extraction cannot produce a -member's content — unsupported codec, encryption without a working password, a -size deferred beyond recovery, an extent running past EOF — exav emits a -**metadata-only member** carrying the name, size and reason, and the scan -resolves to `UNSCANNABLE` or `PASSWORD-PROTECTED`. - -It does not quietly omit it. Omitting one member is how a scanner reports a file -clean on the strength of a scan that never looked inside it, and a credible -header means those bytes *are* a member the target will extract. - -The metadata still earns its keep: `.cdb` container signatures match on member -name, size, encryption flag and position, so a fake `invoice.pdf.exe` inside an -archive is caught by name even when its content is unreadable. - -## Encryption is not a wall, and it is a signal - -exav decrypts what it can rather than stopping at "encrypted": ZIP (ZipCrypto and -WinZip AES), 7z AES-256 including encrypted headers, encrypted DMG, PDF, and the -Office family — legacy XLS RC4 and XOR obfuscation, plus OOXML standard and agile -AES. - -Two defaults do most of the work with no configuration. Office documents are -tried with **`VelvetSweatshop`**, Excel's hard-coded default password, which -opens zero-click for the victim while looking opaque to a scanner that stops at -"encrypted". ZIPs are tried against a small built-in list of the passwords -malware distribution actually uses (`infected`, `virus`, …). Beyond that, -`--passwords` and ClamAV `.pwdb` databases supply your own. - -An archive that stays encrypted is still not nothing: "this member is encrypted" -is itself a detectable property via `.cdb` signatures, and -`--partial-as password-protected=found` turns it into a detection. - -## Hostile input, everywhere +**Deferred sizes.** A streaming ZIP writer sets a flag and puts the member's sizes +in a data descriptor after the data, so the local header does not say where the +member ends. exav recovers the extent from the descriptor's `PK\x07\x08` marker +when its declared size is consistent, otherwise by running to the next header. +The latter is only done for deflate, which terminates itself; a stored member +would swallow the descriptor and the following headers into its content. + +**Truncated streams.** A gzip cut short mid-DEFLATE is common, and `zcat` recovers +everything before the cut, so exav scans the salvaged prefix too. Nesting works: a +truncated `gzip(tar(...))` decompresses the gzip, walks the intact tar entries, +and scans each one. + +If nothing matches, the verdict is `OK`. exav is not a file-integrity validator: +when a sequential stream runs out of input, every byte that exists was scanned, +and the missing tail is absent rather than hidden. Indexed formats are different: +a damaged ZIP index can leave member data present but never enumerated, which is +why dual indexing exists. + +## A member exav cannot read is reported + +When extraction cannot produce a member's content (an unsupported codec, +encryption without a working password, a size deferred beyond recovery, an extent +running past the end of the file), exav emits a metadata-only member carrying the +name, size and reason, and the scan resolves to `UNSCANNABLE` or +`PASSWORD-PROTECTED`. A credible header means those bytes are a member the target +will extract, so leaving it out would be a clean verdict on something nobody +looked at. + +The metadata is still matched: `.cdb` container signatures match on member name, +size, encryption flag and position, so a fake `invoice.pdf.exe` inside an archive +is caught by name even when its content is unreadable. + +## Encryption + +exav decrypts what it can: ZIP (ZipCrypto and WinZip AES), 7z AES-256 including +encrypted headers, encrypted DMG, PDF, and Office (legacy XLS RC4 and XOR +obfuscation, OOXML standard and agile AES). + +Two defaults need no configuration. Office documents are tried with +`VelvetSweatshop`, Excel's built-in default password, which opens without a +prompt for the victim but looks opaque to a scanner that stops at "encrypted". +ZIPs are tried against a short built-in list of passwords malware distribution +uses (`infected`, `virus`, …). `--passwords` and ClamAV `.pwdb` databases add +your own. + +An archive that stays encrypted is still a signal: "this member is encrypted" can +be matched by `.cdb` signatures, and `--partial-as password-protected=found` +turns it into a detection. An encrypted ZIP appended to a picture or document +(a polyglot that archive tools open by its trailing directory) is reported +`PASSWORD-PROTECTED` too, once its directory is found consistent. + +## Hostile input The extractor's entire input is attacker-controlled, so: - `#![forbid(unsafe_code)]`, compiler-enforced, across every decoder. - Release builds enable `overflow-checks`, so a wrapping size calculation traps - instead of silently bypassing a limit. -- Each file is scanned under `catch_unwind`; a parser panic on one crafted input - is an error for that file, never an aborted batch. -- The decoders are tested on **`wasm32`**, where `usize` is 32-bit. Integer and - capacity overflows on parsed offsets are invisible on a 64-bit host and abort - there — and it is the same build the [WASM sandbox](/guides/wasm-sandbox/) - ships. -- Checksums are **not** enforced by default. A wrong CRC is a corrupt archive, - not a reason to stop scanning, and treating it as fatal would hand attackers a - one-byte way to make a member unscannable. + instead of bypassing a limit. +- Each file is scanned under `catch_unwind`; a parser panic on one input is an + error for that file, not an aborted batch. +- The decoders are tested on `wasm32`, where `usize` is 32-bit and overflows on + parsed offsets that a 64-bit host would hide show up. It is also the build the + [WASM sandbox](/guides/wasm-sandbox/) ships. +- A read error from the source is reported, never taken for the end of the + container. +- Checksums are not enforced by default. A wrong CRC is a corrupt archive, not a + reason to stop scanning; making it fatal would give an attacker a one-byte way + to make a member unscannable. ## Validating a decoder -Every decoder here is checked against a **reference implementation's output**, -never against an encoder written next to it. A round-trip through a matching -encoder proves only that the two agree with each other, and passes happily when -both misread the format the same way. - -That is not hypothetical. While writing the `.Z` (LZW `compress`) decoder a -seven-case round-trip went green — and real `compress` output decoded to garbage -after the first few hundred bytes. The round-trip had validated a shared -misreading of when the code width increases. +Every decoder is checked against a reference implementation's output, never only +against an encoder written next to it: a round trip through a matching encoder +proves that the two agree, and passes when both misread the format the same way. +The `.Z` (LZW `compress`) decoder once passed such a round trip while decoding +real `compress` output to garbage after a few hundred bytes. -So each decoder gets an external oracle, built or downloaded locally: +So each decoder gets an external oracle: | Format | Oracle | |---|---| @@ -297,22 +258,22 @@ So each decoder gets an external oracle, built or downloaded locally: | NTFS | `ntfscat` | | LZ4 | `lz4 -dc` | | ARC | `arc` 5.21q and `nomarch` | -| Office OOXML/XLS crypto | the `ppmd-rust` and RustCrypto implementations | +| Office OOXML/XLS crypto | msoffcrypto-tool reference vectors | +| ZIP PPMd | the `ppmd-rust` encoder | | ARJ | a real archive, mutated to produce each failure case | -Where the format provides one, the integrity check does the same job from inside -the data: RAR CRC-32, WIM SHA-1, ARC CRC-16, UPX Adler-32, 7z AES CRC. A decoder -that is subtly wrong produces *plausible bytes* rather than an error, so a -checksum is often the only signal that anything went wrong. +Where the format carries one, the integrity check does the same job from inside +the data: RAR CRC-32, WIM SHA-1, ARC CRC-16, UPX Adler-32, 7z AES CRC. A subtly +wrong decoder produces plausible bytes rather than an error, so a checksum is +often the only sign. -Anything added here should come with an oracle. Being unable to name one is -itself the reason several formats in the -[gap list](/reference/formats/#the-complete-gap-list) stay closed. +A new decoder should come with an oracle; not having one is why several formats +in the [gap list](/reference/formats/#the-complete-gap-list) stay closed. ## See also -- [Supported formats](/reference/formats/) — the current container and codec list -- [Interesting quirks](/concepts/quirks/) — the war stories behind several of the - behaviours above -- [Design principles](/concepts/design-principles/#never-a-silent-clean) — the - invariant this all serves +- [Supported formats](/reference/formats/): the container and codec list. +- [Interesting quirks](/concepts/quirks/): the stories behind several of the + behaviours above. +- [Design principles](/concepts/design-principles/#never-a-silent-clean): the rule + this all serves. diff --git a/www/src/content/docs/concepts/bytecode-sandbox.md b/www/src/content/docs/concepts/bytecode-sandbox.md index 64d64f19..94c6b51a 100644 --- a/www/src/content/docs/concepts/bytecode-sandbox.md +++ b/www/src/content/docs/concepts/bytecode-sandbox.md @@ -1,93 +1,81 @@ --- title: Bytecode sandbox -description: What ClamAV .cbc bytecode signatures are, and why exav's memory-safe, non-JIT interpreter removes the code-execution class that has affected ClamAV. +description: What ClamAV .cbc bytecode signatures are, and why exav's memory-safe interpreter with no JIT removes the code-execution class that has affected ClamAV. --- -ClamAV's most expressive signature type is **bytecode**: a small program (written -in C, compiled to a custom VM bytecode, shipped as a `.cbc` file in -`bytecode.cvd`) that runs against a candidate file and decides whether it's -malicious. It expresses detection logic that pattern and logical signatures -can't — most valuably, a handful of **unpackers** that deobfuscate packed PEs so -the other millions of signatures can match the payload. +ClamAV's most expressive signature type is bytecode: a small program (written in +C, compiled to a custom VM bytecode, shipped as a `.cbc` file in `bytecode.cvd`) +that runs against a candidate file and decides whether it is malicious. It +expresses logic that pattern and logical signatures cannot, most usefully a +handful of unpackers that deobfuscate packed PEs so the other signatures can +match the payload. -exav implements a reader and a **memory-safe, sandboxed interpreter** for this -format. The headline is not extra coverage (the live DB is only 85 programs) — -it's that exav runs them **without the code-execution class** that has -affected ClamAV's bytecode subsystem. +exav implements a reader and a memory-safe, sandboxed interpreter for this +format. The live database holds few programs, so the point is less extra +coverage than running them without the code-execution class that has affected +ClamAV's bytecode subsystem. ## The format, in brief -A `.cbc` file is line-oriented: a `ClamBC` header, a **trigger line** (a `.ldb` -logical signature — the program runs *only* when this matches a file), and -records for types, API declarations, globals, function headers, basic-block -instruction streams, and strings. The instruction set is an LLVM-IR-like SSA -form. Programs cannot make syscalls; they reach the outside world only through a -fixed **host API** — a table of **107 functions**, identical in ClamAV 1.4.3 and -1.5.3: `read`/`seek`/`file_find` to inspect the file, PE/PDF/JSON accessors, -hashing, and `setvirusname` to report a hit. - -The trigger side is free for exav because it already has a full `.ldb` engine. - -## Why exav's interpreter has a smaller attack surface - -ClamAV's bytecode subsystem has a **documented code-execution history**, because -it executes DB-supplied programs in memory-unsafe C and historically via an LLVM -JIT: - -- **[CVE-2020-37167](https://www.cve.org/CVERecord?id=CVE-2020-37167)** — weak - validation in the bytecode interpreter's function-name processing (CWE-94 code - injection) lets a crafted `.cbc` corrupt the VM's memory and chain to - arbitrary code execution in the scanner process, with a +A `.cbc` file is line-oriented: a `ClamBC` header, a trigger line (a `.ldb` +logical signature; the program runs only when it matches), and records for types, +API declarations, globals, function headers, basic-block instruction streams +and strings. The instruction set is an SSA form close to LLVM IR. Programs cannot +make syscalls; they reach the outside world only through a fixed host API of 107 +functions (the same in ClamAV 1.4.3 and 1.5.3): `read`/`seek`/`file_find` to +inspect the file, PE/PDF/JSON accessors, hashing, and `setvirusname` to report a +hit. The trigger side comes for free with exav's `.ldb` engine. + +## Why the attack surface is smaller + +ClamAV's bytecode subsystem has a documented code-execution history, because it +runs database-supplied programs in memory-unsafe C and, historically, through an +LLVM JIT: + +- **[CVE-2020-37167](https://www.cve.org/CVERecord?id=CVE-2020-37167):** weak + validation in the interpreter's function-name processing (CWE-94) lets a + crafted `.cbc` corrupt the VM's memory and chain to arbitrary code execution + in the scanner process, with a [public ROP exploit](https://www.exploit-db.com/exploits/47687) (exploit-db - 47687, the `bytecode_vm` sandbox escape). **CVSS 8.4**; ClamAV before - 0.103.0. Also tracked by - [Ubuntu](https://ubuntu.com/security/CVE-2020-37167). -- The optional **LLVM JIT** generated and ran native code from bytecode — a large + 47687, the `bytecode_vm` sandbox escape). CVSS 8.4; ClamAV before 0.103.0. + Also tracked by [Ubuntu](https://ubuntu.com/security/CVE-2020-37167). +- The optional LLVM JIT generated and ran native code from bytecode, a large attack surface Cisco has been moving away from. exav removes these failure modes by construction: | Risk in ClamAV's C VM | exav | |---|---| -| Memory corruption in the VM (the CVE-2020-37167 class) | **Pure safe Rust, no `unsafe`** — every memory access is a bounds-checked slice; an out-of-range index panics into isolation, it cannot corrupt memory | -| Native code generation from bytecode (JIT spray, W^X issues) | **No JIT, ever** — interpret only | -| A program escaping the sandbox (syscalls, host memory) | The program sees only bounded `Vec`s and a fixed read-only file API — no syscalls, no host pointers | -| Runaway program (CPU/memory exhaustion) | **Instruction budget**, scratch-memory cap, and call-depth limit | -| A parser/VM bug taking down the scan | **`catch_unwind` per program** (and per file) — one bad program is skipped, the scan continues | -| Malformed/hostile `.cbc` | Fallible parser (no panics); 100% of the live DB parses cleanly; a program not fully understood is **never executed** | +| Memory corruption in the VM (the CVE-2020-37167 class) | Safe Rust with no `unsafe`: every access is a bounds-checked slice, and an out-of-range index panics into isolation instead of corrupting memory | +| Native code generation from bytecode | No JIT: interpretation only | +| A program escaping the sandbox (syscalls, host memory) | The program sees only bounded buffers and a fixed read-only file API: no syscalls, no host pointers | +| A runaway program | An instruction budget, a scratch-memory cap and a call-depth limit | +| A VM bug taking down the scan | `catch_unwind` per program and per file: a bad program is skipped and the scan continues | +| A malformed or hostile `.cbc` | A fallible parser with no panics; a program not fully understood is never executed | -The net: the worst a hostile or buggy `.cbc` can do to exav is *be skipped or -time out*. In ClamAV the worst case has been *arbitrary code execution*. For anyone -who disables ClamAV bytecode for safety, exav offers the capability **without -that trade-off**. +The worst a hostile or buggy `.cbc` can do to exav is be skipped or time out. For +anyone who disables ClamAV bytecode for safety, exav offers the capability +without that trade-off. ## Trigger-gated execution, and the API subset -Execution is **live in the normal scan path**, trigger-gated per program: a -program runs only when its logical-signature trigger matches. The gating is -per-program, not a global off switch. - -exav implements **34 of the 107 host APIs**. The other 73 are present as -**fail-safe stubs** rather than as a reason to skip the program: a call to one -returns a conservative value and is recorded, so the program still runs to the -end. Every stub that actually fabricates a result is surfaced in the run's -outcome (and warned under `EXAV_BC_WARN`), because a verdict that rests partly -on fiction must never look like one that does not. +Execution is live in the normal scan path and gated per program: a program runs +only when its logical-signature trigger matches. -What that buys, measured on a live `bytecode.cvd`: exav parses **100% of the 85 -programs** and **all 85 execute to completion** with no unsupported opcode. -Across 400 real malware samples, exactly **one** stub was ever reached -(`get_environment`). A `.cbc` header declares a `maxapi` *ceiling*, not a call -list — which is why a third of the table has been enough for all of the shipped -programs so far. The -[full per-group breakdown of the 56 unimplemented reachable APIs](/project/comparison-with-clamav/#bytecode-host-apis--34-of-107) -is in the ClamAV gap list. +exav implements part of the host API. The rest are fail-safe stubs rather than a +reason to skip a program: a call returns a conservative value and is recorded, so +the program still runs to the end. A stub that fabricates a result is recorded in +the run's outcome (and warned about under `EXAV_BC_WARN`), so a verdict that +rests partly on a stub is distinguishable from one that does not. On a live +`bytecode.cvd`, every program parses and runs to completion with no unsupported +opcode, and in practice programs rarely reach a stub: a `.cbc` header declares a +`maxapi` ceiling, not a call list. The per-API breakdown is in the repository's +[`docs/BYTECODE.md`](https://github.com/sylvinus/exav/blob/main/docs/BYTECODE.md). ## Real-world weight -Numerically the 85 programs are 0.002% of ~3.7M signatures, but uneven in value: -the ~6 **unpackers** matter most (missing one silently weakens detection -across many packed samples), the polymorphic-family checks each catch a whole -family, and a long tail of ~36 are legacy single-CVE checks. The detection delta -is modest — which is why exav's win here is the memory-safe, non-JIT sandbox, -more than the raw extra coverage. +Bytecode programs are a tiny fraction of the signatures, and uneven in value: the +few unpackers matter most (missing one weakens detection across many packed +samples), the polymorphic-family checks each cover a whole family, and a long +tail are legacy single-CVE checks. The detection difference is modest, which is +why the memory-safe sandbox is the main gain. diff --git a/www/src/content/docs/concepts/design-principles.md b/www/src/content/docs/concepts/design-principles.md index de3d7bbb..9516c3db 100644 --- a/www/src/content/docs/concepts/design-principles.md +++ b/www/src/content/docs/concepts/design-principles.md @@ -1,145 +1,121 @@ --- title: Design principles -description: The principles exav is built around — never a silent clean, memory safety, no runtime codegen, constant-memory streaming, minimal dependencies, drop-in compatibility, bounded work, and clean-room licensing. +description: The principles exav is built around, from never a silent clean and memory safety to no runtime code generation, bounded work, drop-in compatibility and clean-room licensing. --- -exav is built around a handful of principles. They reinforce each other: several -are about *safety* (of the host, and of the verdict), several are about *fit* -(dropping into an existing setup and staying lean), and one ties the rest -together — every bound the engine hits is surfaced, never hidden. +exav is built around a handful of principles. Some protect the host, some protect +the verdict, and some make exav practical to adopt. One ties them together: every +bound the engine hits is reported. ## Never a silent clean -This is the invariant the whole verdict model is built around: +> Never report a file clean unless it was fully scanned. -> **Never report a file clean unless it was actually, *fully* scanned.** +A clean verdict is a safety claim. If exav did not finish looking, `OK` would be +a false claim that an attacker can arrange on purpose, by padding a payload past +a size limit or wrapping it in something the scanner gives up on. So anything +that makes the scanner stop early, for any reason, is reported instead of being +folded into `OK`. -A scanner's clean verdict is a safety claim. If exav did not finish looking, -claiming `OK` is a lie an adversary will engineer. Anything that makes the -scanner *stop looking early* — for **any** reason, no exceptions — must surface, -never be swallowed into `OK`. - -### Why this is a security property, not a nicety - -ClamAV has a long-standing large-file limitation: files over ~2 GB are read but -scanned as **zero bytes** and still reported **`OK` / clean**. That's a silent -clean verdict on data that was never inspected — exactly the kind of gap an -attacker pads a payload to exploit. exav closes it, and generalizes the fix to -*every* reason a scan might stop early. - -The invariant holds in every mode, including `--clamav-compat`. Compat changes -the *names* exav reports and the rough scope of what it opens so a differential -run compares like with like — it does not switch unpackers off. A miss is a miss -whichever mode produced it. +ClamAV, for example, reads a file over about 2 GB, scans none of it, and reports +`OK`. exav closes that gap and every other way a scan can stop early. The rule +holds in every mode, including `--clamav-compat`, which changes the names exav +reports and what it opens, but not whether an incomplete scan is reported. Three rules follow: -1. **A detection always beats a limit.** A signature match is reported `FOUND`, - never downgraded because some *other* part of the input tripped a budget. -2. **Never refuse by size without scanning.** A file over `--max-extracted-bytes` - isn't skipped wholesale — the flat pattern/hash core still runs over the - in-budget bytes, so a payload in the scanned prefix is still caught. Otherwise - `cat malware huge.pad > evil` would be a one-line bypass. -3. **Not-fully-scanned is never clean — whatever the cause.** This is exhaustive: - it covers external resource limits **and exav's own internal work bounds**. If - the engine caps its own effort (a per-buffer verification-step budget that - bounds pathological wildcard backtracking, a matcher time/step budget), hitting - that cap without finishing surfaces as a non-clean verdict. - -Anything not fully scanned resolves to `LIMITS-EXCEEDED`, `UNSCANNABLE`, or -`PASSWORD-PROTECTED` — distinct, non-clean, and reported rather than hidden. See -[Verdicts & exit codes](/reference/verdicts/) for what each one means and how it -maps to a status tag and process exit code. - -### Stricter than ClamAV, on purpose - -This invariant holds even where it makes exav stricter than ClamAV. `clamscan` -silently returns `OK` after bounding its own work (its `alert-exceeds-max` / -`alert-encrypted` heuristics are off by default); exav does not. That is the one -deliberate behavioral difference a migrating user should know — see -[Migrating from ClamAV](/guides/migrating-from-clamav/). `--clamav-compat` -matches ClamAV's *limit values* for differential testing, but exav still surfaces -the outcome rather than hiding it. - -### The invariant is about the verdict, not about the policy - -Surfacing a bound and *acting* on it are different jobs. The engine always -reports `LIMITS-EXCEEDED`, `UNSCANNABLE` or `PASSWORD-PROTECTED` rather than -`OK`; what a deployment does with that verdict is the deployment's call, and some -of them will reasonably choose to deliver the object anyway rather than reject a -user's upload. - -Where a deployment makes that choice — [`--partial-as ok`](/guides/icap/) -is the one that does — it is opt-in, named per verdict, announced at startup, -logged per object, and still reported in the response. What the invariant rules -out is not an operator accepting a risk; it is a scanner *hiding* one. A pass -nobody configured, or one nobody can see in a log, is the failure this principle -exists to prevent. - -### The daemon refuses empty coverage too - -The same principle extends to configuration. With **no real signature database** -loaded (absent, empty, or a signature-less DB), exav **refuses to run** rather -than answer scans against near-zero coverage — a reachable scanner that silently -passes real malware as clean is the exact bypass exav exists to prevent. The tiny -built-in EICAR-only baseline is opt-in via `EXAV_ALLOW_NO_DB` (testing/CI only). +1. **A detection beats a limit.** A signature match is reported `FOUND`, even if + another part of the input hit a budget. +2. **Refusing by size still scans.** An input over `--max-input-bytes` is not + skipped: its first `--max-input-bytes` bytes get the same scan as a smaller + input, so a payload there is still caught, and the input is reported + `LIMITS-EXCEEDED`. Otherwise `cat malware huge.pad > evil` would be a bypass. + A file, stdin and every daemon verb go through that one scan. +3. **Not fully scanned is never clean, whatever the cause.** That covers external + limits and exav's own work bounds: a verification-step budget that stops a + pathological wildcard search, an emulator that runs out of steps. + +A scan that did not complete resolves to `LIMITS-EXCEEDED`, `UNSCANNABLE` or +`PASSWORD-PROTECTED`. See [Verdicts & exit codes](/reference/verdicts/) for what +each means and how it is reported. + +### Stricter than ClamAV + +`clamscan` returns `OK` after bounding its own work (its `alert-exceeds-max` and +`alert-encrypted` heuristics are off by default); exav does not. It is the one +behavioral difference a migrating user has to plan for (see +[Migrating from ClamAV](/guides/migrating-from-clamav/)). `--clamav-compat` +matches ClamAV's limit values for differential testing, and still reports the +outcome. + +### The verdict is not the policy + +Reporting a bound and acting on it are different jobs. The engine always reports +`LIMITS-EXCEEDED`, `UNSCANNABLE` or `PASSWORD-PROTECTED`; what a deployment does +with that is its own decision, and some will reasonably deliver the object anyway +rather than reject an upload. + +Where a deployment chooses that, with +[`--partial-as ok`](/reference/cli/#what-an-unscannable-object-becomes), the +choice is named per verdict, announced at startup, logged per object, and still +reported in the response. The rule does not forbid accepting a risk; it forbids +hiding one. + +### Refusing empty coverage + +With no real signature database loaded (absent, empty, or with no signatures), +exav refuses to run rather than answer scans against near-zero coverage. The +built-in EICAR-only baseline is opt-in with `--allow-no-db`, for testing. ## Memory safety -exav is written in Rust with minimal `unsafe`. The scanner parses hostile input -by design, so the core scanning and extraction crates are -`#![forbid(unsafe_code)]`; the residual `unsafe` lives only in audited, -widely-used dependency primitives (compression/crypto SIMD, OS syscalls), not in -attacker-driven parsing logic — and driving that surface down is an explicit, -ongoing goal. +exav is written in Rust. The scanning and extraction crates are +`#![forbid(unsafe_code)]`; the remaining `unsafe` lives in widely used +dependency primitives (compression and crypto SIMD, OS syscalls) and in the +daemon's calls into libc, not in the code that parses hostile input. Reducing it +further is an ongoing goal (see the [roadmap](/project/roadmap/)). -## No runtime codegen (W^X / no JIT) +## No runtime code generation -Nothing is compiled to native code at scan time. The bytecode programs and YARA -rules run on interpreters/evaluators, so there is no writable-executable memory -while scanning — closing an entire class of exploitation primitive. +Nothing is compiled to native code at scan time. Bytecode programs and YARA rules +run on interpreters, so there is no writable and executable memory while +scanning, which rules out a whole class of exploitation primitive. -## Clean-room & permissive licensing +## Clean-room, permissive licensing -exav is MIT-licensed and derives nothing from GPL sources. Every part is -implemented from public specifications and permissively-licensed -(MIT/BSD/Apache/public-domain) code with attribution. See +exav is MIT-licensed and derives nothing from GPL sources. Everything is +implemented from public specifications and permissively licensed +(MIT/BSD/Apache/public-domain) code, with attribution. See [Contributing](/project/contributing/) and [License](/project/license/). -## Constant-memory streaming +## Bounded work and memory -The per-scan working set stays flat regardless of file size. A multi-gigabyte -file and a one-kilobyte file use the same forward-pass machinery and the same -tiny (~2 MiB) working set — so exav scans inputs larger than RAM. See -[Streaming & memory](/concepts/streaming-memory/). +Every decode and match is bounded by size or by a step count, so hostile input +cannot turn a scan into an unbounded computation or allocation; the prefork +daemon adds a per-job wall-clock and CPU limit on top. Archives are walked member +by member, and every bound is a flag. Reaching one is reported, never a silent +truncation. See [Streaming & memory](/concepts/streaming-memory/) and +[Limits](/reference/limits/). ## Minimal dependencies -exav favors small leaf crates and a lean, pure-Rust dependency tree — for -control, auditability, `unsafe` surface, and binary/WASM size. The native +exav prefers small crates and a pure-Rust dependency tree, for auditability, +`unsafe` surface, and binary and WASM size. The native [YARA engine](/guides/yara/), for instance, is a compact tree-walking evaluator -that needs no WASM runtime and no JIT/codegen backend. Every crate in the -default build is listed on the [Dependencies](/reference/dependencies/) page. +with no WASM runtime and no code generation. Every crate in the default build is +listed on the [Dependencies](/reference/dependencies/) page. ## Drop-in compatibility exav loads the signature databases you already have, speaks the `clamd` wire -protocol, and matches an established scanner's CLI flags, output, and exit -codes — so it fits into an existing deployment without rewriting tooling. See +protocol, and prints `clamscan`'s output format and exit codes, so it fits into +an existing deployment without rewriting tooling. Its flags are its own. See [Comparison with ClamAV](/project/comparison-with-clamav/) and [Migrating from ClamAV](/guides/migrating-from-clamav/). -## Bounded work - -Every decode and match is size-, time-, and step-bounded, so hostile input can't -turn a scan into an unbounded computation or allocation. Hitting a bound is never -swallowed: it surfaces as a non-clean verdict, never a silent truncation. - ## How they fit together -*Memory safety* and *no runtime codegen* protect the **host** running exav. -*Never a silent clean* and *bounded work* protect the **verdict** — a bound is -always visible. *Constant-memory streaming*, *minimal dependencies* and *drop-in -compatibility* make exav practical to run at scale and to adopt in place. And -*clean-room licensing* keeps the whole thing permissively licensed and -independently built. +Memory safety and no runtime code generation protect the host running exav. +Never a silent clean and bounded work protect the verdict. Minimal dependencies +and drop-in compatibility make exav practical to run and adopt. Clean-room +licensing keeps it permissively licensed and independently built. diff --git a/www/src/content/docs/concepts/differential-testing.md b/www/src/content/docs/concepts/differential-testing.md index a2f1cf98..e88e45c1 100644 --- a/www/src/content/docs/concepts/differential-testing.md +++ b/www/src/content/docs/concepts/differential-testing.md @@ -1,104 +1,84 @@ --- title: Differential testing -description: How exav validates its drop-in correctness — scanning the same files with the same database as clamscan, and comparing verdicts. +description: How exav validates its compatibility by scanning the same files with the same database as clamscan and comparing verdicts. --- -exav's correctness as a drop-in is validated by **differential testing**: scan -the **same files** with the **same signature database** using both engines and -compare verdicts. Disagreements are exav bugs — either a false negative (clamscan -detects, exav misses) or a genuine false positive. +exav's compatibility is validated by differential testing: scan the same files +with the same signature database using both engines and compare verdicts. A +disagreement is either an exav bug (clamscan detects, exav misses) or something +to explain. -It is a **compliance** harness, not a benchmark. It records timings, which are -useful for spotting a wedged file or an engine that is dramatically slower, but -they are measured under concurrency against a cold page cache and are not -performance numbers. +It is a compliance harness, not a benchmark. It records timings, which help spot +a wedged file or a much slower engine, but they are taken under concurrency +against a cold page cache and are not performance numbers. -The harness lives in the repository's `scripts/` (tracked and reproducible); this -page is the concept, not the operator's runbook. +The harness lives in the repository's `scripts/`; this page is the concept, not +the runbook. ## The setup -- **Corpus:** a live-malware corpus, static-scan only, never executed - (gitignored). -- **Both engines, same DB:** both must load the *same* signature set (`daily.cvd` - only, or the full `main+daily`) or the comparison is meaningless. -- **Both driven by the same client:** one scanner program speaks to both daemons, - so a difference in results cannot come from a difference in how they were asked. -- **Every scan is all-match.** A verdict is the *set* of signatures that matched, - not whichever one an engine reached first. Comparing first-matches is the - single largest source of fake disagreement: both engines find the malware, each - names a different signature, and the diff calls it a conflict. -- **One engine resident at a time, whole corpus per phase.** ClamAV's verdicts are - a function of the pinned database, so its pass is run once and cached; iterating - on exav re-pays only its own pass. Running both daemons together on a small host - meant they competed for RAM, workers died, and a stuck job turned into rows - recorded as errors — differences that were artefacts of the harness rather than - of the engines. +- **Corpus:** live malware, scanned statically and never executed (not in the + repository). +- **Same database:** both engines load the same signature set (`daily.cvd` only, + or the full `main` + `daily`), or the comparison means nothing. +- **Same client:** one program drives both daemons, so a difference cannot come + from how they were asked. +- **All-match scans:** a verdict is the set of signatures that matched, not + whichever one an engine reached first. Comparing first matches is the largest + source of false disagreement: both engines find the malware and name different + signatures. +- **One engine at a time, whole corpus per phase:** ClamAV's verdicts depend only + on the pinned database, so its pass is run once and cached. Running both daemons + together on a small host made them compete for RAM and produced errors that + came from the harness, not the engines. ## Interpreting results -"They disagreed" is almost never the useful answer, so results are bucketed: +Results are bucketed, because "they disagreed" is rarely useful on its own: | Bucket | Meaning | |---|---| -| `AGREE` / `clean` | identical signature sets, or both clean — the headline metric | -| `PARTIAL` | sets overlap: same malware, one engine also named more | -| `NAMEDIFF` | sets are disjoint — a real disagreement about what this is | -| `FN` | clamscan detected, exav did not — **a real gap** | -| `EXAV_ONLY` | exav detected, clamscan clean — verify before assuming an FP | -| `CAREFUL` | exav flagged not-fully-scanned; clamscan said OK | -| `CAREFUL_FN` | clamscan detected; exav flagged not-fully-scanned | +| `AGREE` / `clean` | identical signature sets, or both clean: the headline metric | +| `PARTIAL` | the sets overlap: same malware, one engine also named more | +| `NAMEDIFF` | the sets are disjoint: a real disagreement about what this is | +| `FN` | clamscan detected, exav did not: a real gap | +| `EXAV_ONLY` | exav detected, clamscan said clean: verify before assuming a false positive | +| `CAREFUL` | exav reported not fully scanned; clamscan said OK | +| `CAREFUL_FN` | clamscan detected; exav reported not fully scanned | | `ERROR` | a scan failed, so there is nothing to compare | -**`EXAV_ONLY` is deliberately not called "FP".** On an 8,978-file run every one -was checked and none was an exav error: nearly all were hits inside members -clamd never unpacked, confirmed by re-scanning exav's own extracted members -*with clamd*, which then flags them under the same name. Naming the bucket "false -positive" invites fixing detections that are right. +`EXAV_ONLY` is not called "false positive" because it has mostly been hits inside +members clamd never unpacked, confirmed by feeding exav's extracted members back +to clamd, which then flags them under the same name. `CAREFUL` is likewise a +capability difference: clamscan returns `OK` for content it never opened. `FN` is +the bucket that means a bug. -`CAREFUL` is likewise a capability difference rather than a fault — clamscan -returns OK for content it never opened, and exav refuses to (see below). - -**FN** is the bucket that means a bug. Real gaps it has surfaced: imphash -ordinal encoding, section-hash on truncated PEs, and embedded-PE scanning (a PE -appended inside another file). - -Absolute hit rate is DB-bound, not a corpus property: with `daily` alone (most -coverage lives in `main.cvd`) both engines detect only a small fraction of a -random sample. The **agreement** is what matters. +The absolute hit rate depends on the database, not the corpus: with `daily` alone +(most coverage is in `main.cvd`) both engines detect a small fraction of a random +sample. Agreement is what matters. ## `--clamav-compat`: matching boundaries, not quirks -By default exav runs at **full capability**. For an apples-to-apples differential -run, `--clamav-compat` dials exav back to a stock ClamAV build's documented -defaults — the limit values (`--max-input-bytes 100M`, `--max-extracted-bytes 400M`, -`--max-unpack-depth 17`, `--max-members 10000`, `--decode none`), the extractor set, and -cosmetic naming (the `.UNOFFICIAL` suffix on unofficial-database signatures). Each -limit is also an individual flag, and an explicit flag always wins over the -preset; the extractor set and the naming have no flags of their own, being wanted -only for a differential run. +By default exav runs at full capability. For a like-for-like run, +`--clamav-compat` sets a stock ClamAV build's documented defaults: the limit +values (`--max-input-bytes 100M`, `--max-extracted-bytes 400M`, +`--max-unpack-depth 17`, `--max-members 10000`, `--decode none`), the extractor +set, `.UNOFFICIAL` naming for unofficial-database signatures, and +`--partial-as ok`, so a file ClamAV would call clean is answered `OK` here too. +Each limit is also its own flag, and an explicit flag wins over the preset. + +It matches ClamAV's documented boundaries, not its implementation quirks: it does +not reproduce each parser's internal caps and bail-outs (an RTF parser that stops +on an oversized control word and so never reaches an embedded payload, for +example). Reproducing those would mean copying weaknesses malware relies on, so +those divergences remain, and they are exav being more thorough. -It deliberately stops there. It matches ClamAV's *documented, well-defined -boundaries* — **not** its implementation quirks. It does not reproduce ClamAV's -internal parser caps and bail-outs (e.g. an RTF parser that aborts on oversized -control words and so never reaches an embedded payload). Matching ClamAV *there* -would mean re-implementing each format parser's weaknesses one by one — -deliberately degrading exav to mirror a limitation malware actively exploits. -Those residual divergences are exav being **more thorough**, not bugs, and they -stay even under `--clamav-compat`. In a differential run they're explained by -"ClamAV's parser bailed," not a missing knob. +A file exav could not fully examine is still logged under `--partial-as ok`, so +the harness can bucket it as `CAREFUL` rather than as agreement. :::caution -`--clamav-compat` is a diff-testing tool, not a production mode. It *reduces* -exav's detection capability so results reproduce clamscan's — it can miss malware -exav would otherwise catch (e.g. a UPX-packed Mirai ELF whose config exav -decompresses). Run exav at full capability in production. +`--clamav-compat` is for differential testing, not production. It reduces +detection so results reproduce clamscan's, and can miss malware exav would +otherwise catch (a UPX-packed Mirai ELF whose configuration exav decompresses, +for one). Run exav at full capability in production. ::: - -## What never gets reproduced - -The [never-silent-clean invariant](/concepts/design-principles/#never-a-silent-clean) is *not* undone by -`--clamav-compat`. Where stock ClamAV returns clean after bounding its own work, -exav still reports `LIMITS-EXCEEDED` / `UNSCANNABLE` / `PASSWORD-PROTECTED` and -exits 2. A differential harness buckets these as an expected, explained -divergence — exav surfacing an outcome ClamAV hides. diff --git a/www/src/content/docs/concepts/pe-emulation.md b/www/src/content/docs/concepts/pe-emulation.md index 3225a6e0..529f8deb 100644 --- a/www/src/content/docs/concepts/pe-emulation.md +++ b/www/src/content/docs/concepts/pe-emulation.md @@ -8,30 +8,28 @@ with a compressed or encrypted blob plus a small **stub** that rebuilds it in memory at start-up. On disk there is nothing for a signature to match: the original code is there, but not in a form anything can read. -The usual answer is a decoder per packer. exav has those for the common cases — -UPX, the aPLib family, MPRESS — and they are the fast path, because a decoder -that knows the format costs one decompression. But writing one per packer is a -race the scanner cannot win: the packer's author picks the format and is free to -change it, and the packers that matter vary it per build specifically to break -tools that assume a layout. +The usual answer is a decoder per packer. exav has those for the common cases +(UPX, the aPLib family, MPRESS), and they are the fast path. But a decoder per +packer is a race the scanner cannot win: the packer's author picks the format, +and the packers that matter vary it per build to break tools that assume a +layout. So exav also does the general thing. Whatever a packer compresses with, its stub -**must** reconstruct the original image in memory and jump to it — that is what -running the program means. exav runs the stub and takes the picture at the jump. +must rebuild the original image in memory and jump to it. exav runs the stub and +takes the picture at the jump. ## What runs A bounded x86-32 interpreter, in-process, with: -* **A sparse address space.** Pages exist once they are touched, so a hostile - `SizeOfImage` of 4 GB is not a memory bomb, and the page count is capped by - the same per-member ceiling as any other buffer the scanner materialises. -* **Instruction decoding by [`exav-x86`](/subprojects/exav-x86/)** — decode-only, - no assembler and no code generation. exav supplies the semantics. Getting - instruction *boundaries* right is where a decoder goes quietly wrong, so it - lives in a crate whose whole job is to be checked instruction by instruction - against an independent decoder — currently agreeing on all 94,245,255 decode - sites in the sample corpus. +* **A sparse address space.** Pages exist once touched, so a hostile + `SizeOfImage` of 4 GB is not a memory bomb, and the page count is capped like + any other buffer the scanner holds. +* **Instruction decoding by [`exav-x86`](/subprojects/exav-x86/):** decode-only, + with no assembler and no code generation; exav supplies the semantics. + Instruction boundaries are where a decoder goes quietly wrong, so the decoder + lives in a crate that is checked instruction by instruction against an + independent decoder over the sample corpus. * **Enough Windows to be believed.** A TEB and PEB, the loader's three module lists, and synthetic `kernel32`/`ntdll`/`user32` images with real export directories. Both ways a stub resolves imports work: calling @@ -39,125 +37,108 @@ A bounded x86-32 interpreter, in-process, with: by hand. The loader's own import binding is emulated too, because a packed file's stub calls `LoadLibraryA` through an IAT slot the loader filled in. * **Structured exception handling.** A fault is delivered to the stub's own SEH - chain and execution resumes from the `CONTEXT` the handler leaves behind — - throwing an exception at yourself and continuing from the handler is standard - anti-emulation, and an emulator that stops at the fault stops exactly where - the packer wanted it to. -* **The trap flag.** A stub that sets `TF` through `popfd` expects a - single-step exception after the next instruction and drives its whole - decryption loop from the handler. Ignore the flag and it spins forever; TELock - does exactly this. -* **Its own file, read-only.** Installers and self-extractors keep the payload - as an *overlay* past the last section and go back to disk for it with - `CreateFile`/`ReadFile` on their own path. That path — and only that path — - resolves, served from the bytes already in memory. + chain, and execution resumes from the `CONTEXT` the handler leaves. Raising an + exception at yourself and continuing from the handler is standard + anti-emulation; an emulator that stops at the fault stops where the packer + wanted. +* **The trap flag.** A stub that sets `TF` through `popfd` expects a single-step + exception after the next instruction and drives its decryption loop from the + handler (TELock does this). Ignoring the flag leaves it spinning. +* **Its own file, read-only.** Installers and self-extractors keep the payload as + an overlay past the last section and read it back with `CreateFile`/`ReadFile` + on their own path. That path, and only that one, resolves, served from the + bytes already in memory. * **The loader's quirks.** `PointerToRawData` is rounded down to 512 bytes, as - Windows does, because packers point a section a little past its real start and - rely on the rounding. Stack pages commit on demand, as the guard page does. + Windows does, because packers rely on it. Stack pages commit on demand, as the + guard page does. -Nothing escapes. There are no syscalls, no file or network access, and no host -memory beyond the page table. Malware being unpacked here is *interpreted*, -never executed: an emulated `CreateFile` returns a handle that does nothing. +Nothing escapes: no syscalls, no file or network access, no host memory beyond +the page table. The malware is interpreted, never executed; an emulated +`CreateFile` returns a handle that does nothing. ## When to take the picture The tell is packer-independent: -> execution arrives at an address **inside the image**, on a page **the stub -> itself wrote** and has **never executed** — or in a **different section** from -> the one the stub started in — and enough of that section has been rewritten to -> be a rebuilt program rather than a patched jump. - -Each clause earns its place. "Different section" alone fires on a stub calling a -helper. "Written page" alone fires on its scratch data. Without the bulk test, -an *obfuscated* program — not packed at all — that writes a jump table into -another section and transfers through it looks identical to an unpack. And -without the never-executed clause, the packers that unfold the original program -*over the same section their stub lives in* (MEW, RLPack) can never be seen at -all, because no section-level rule applies to them. - -If the stub wiped its own `MZ` header on the way out — an anti-dump move — the -headers are taken from the file, which still has them and which parsed. A -reconstructed image is not thrown away over four bytes the packer erased. - -What comes out is written as a **memory-layout PE**: sections at their virtual -addresses, `PointerToRawData == VirtualAddress`, entry point set to where the -stub transferred control. That is a valid PE the scanner re-parses like any -other file, so resources, overlays and nested content inside the recovered image -are reached too. +> execution arrives at an address inside the image, on a page the stub itself +> wrote and has never executed, or in a different section from the one the stub +> started in, and enough of that section has been rewritten to be a rebuilt +> program rather than a patched jump. + +Each clause is needed. "Different section" alone fires on a stub calling a +helper; "written page" alone fires on scratch data. Without the bulk test, an +obfuscated program that writes a jump table into another section looks like an +unpack. Without the never-executed clause, packers that unfold the program over +the section their stub lives in (MEW, RLPack) are never seen. + +If the stub wiped its own `MZ` header on the way out (an anti-dump move), the +headers are taken from the file, which still has them. + +The result is written as a memory-layout PE (sections at their virtual addresses, +`PointerToRawData == VirtualAddress`, entry point where the stub transferred +control), which the scanner parses like any other file, so resources, overlays +and nested content in the recovered image are reached too. ## When the stub wins -Stubs defend themselves, and some will not run to completion. The run stops and -**says why**: an instruction outside the implemented set (named), an export with -no implementation (named), a fault nothing handled, a budget. Two things follow -from that: +Some stubs will not run to completion. The run stops and says why: an +instruction outside the implemented set, an export with no implementation (both +named), a fault nothing handled, or a budget. * **Nothing is fabricated.** A dump is emitted only when it reads back as a - valid PE. A wrong guess fails that check and is discarded rather than handed - to the matcher as data. -* **The file is not called clean.** A packed executable exav could not unfold is - reported `UNSCANNABLE`, which is the - [never-silent-clean invariant](/concepts/design-principles/#never-a-silent-clean) - doing its job — the packed bytes were scanned, the image the stub would have - unpacked was not, and the report says so. - -A run that stopped part-way having rebuilt a substantial part of the image -reports that separately, as a *partial* reconstruction. Half a decompressed -image is worth scanning; it is not the same claim as "this is the original -program", and it is not named as though it were. + valid PE; a wrong guess is discarded rather than handed to the matcher. +* **An identified packer is not called clean.** If exav named the packer and + still could not unfold it, the file is `UNSCANNABLE`: the packed bytes were + scanned, the image the stub would unpack was not. A file sent to the emulator + only because of its shape, with no packer identified, is scanned as it is; + flagging every unusual executable would be noise. `--detect packed` turns an + identified packer exav could not unpack into a `Heuristics.Packed.*` + detection. +* **Running out of budget is a limit.** The emulator has an instruction budget + per stub, and `--max-pe-emulation-steps` (1,000,000,000 by default) bounds the + total across every packed executable in one file. A scan that reaches it is + `LIMITS-EXCEEDED`. + +A run that stopped part-way with a substantial part of the image rebuilt is +reported as a partial reconstruction: worth scanning, but not named as the +original program. ## What it will never unpack -Virtualizing protectors — VMProtect, Themida/WinLicense, Enigma. These translate -the protected functions into a private bytecode when the file is *built*, so -there is no moment at runtime when the original instructions exist in memory. -No emulator recovers them, however good, because there is nothing to recover. -exav does not spend the budget trying; it reports them, in wording that says -what is actually true about them. +Virtualizing protectors (VMProtect, Themida/WinLicense, Enigma) translate the +protected functions into a private bytecode when the file is built, so the +original instructions never exist in memory, and no emulator can recover them. +exav does not spend the budget trying; it reports them as what they are. ## Measured coverage -Against 276 samples produced by 23 real packers, 12 samples each (ordinary -Windows utilities, packed — fetched by `scripts/fetch-packed-pe.py`): +Against 276 samples from 23 real packers, 12 each (ordinary Windows utilities, +packed; fetched by `scripts/fetch-packed-pe.py` and checked by the corpus test in +`exav-unpack`): | | Packers | |---|---| -| **Unpacked** — *every* sample yields an image at least as large as the packed file | ASPack, BeRoEXEPacker, EXpressor, FSG, MEW, MPRESS, Molebox, NSPack, Neolite, PECompact, Packman, RLPack, UPX, WinUpack, Yoda-Crypter | +| **Unpacked**: every sample yields an image at least as large as the packed file | ASPack, BeRoEXEPacker, EXpressor, FSG, MEW, MPRESS, Molebox, NSPack, Neolite, PECompact, Packman, RLPack, UPX, WinUpack, Yoda-Crypter | | **Unpacked, but not on every sample** | Exe32pack (11/12), PEtite (11/12), JDPack (10/12), Eronana Packer (6/12) | -| **Payload recovered, image incomplete** | Amber (a .NET packer: the native stub hands the assembly to the runtime, and it is that assembly which comes back — on 11 of 12 samples) | -| **Nothing recovered** — the stub defended itself successfully | Alienyze, TELock, Yoda-Protector | - -That is **218 of 276 samples** (79%) giving back a complete image, and **15 of 23 -packers** doing so on every sample they were given. Where a stub wins, it costs -exav a verdict rather than a detection: the file is reported `UNSCANNABLE`, -never clean. - -No sample came back with a partial image. Recovery is all-or-nothing per sample: -the 218 that recovered anything all recovered an image at least as large as the -input. A packer at 10/12 is one where two particular samples failed outright, -not one that half-unpacks. - -Those 15 include four that ClamAV ships a hand-written unpacker for (ASPack, MEW, -NSPack, Yoda's Cryptor); PEtite, a fifth, unpacks on 11 of 12. Most of the rest -have no decoder anywhere, which is the point of running the stub instead of -writing one. - -Note what the measure is: **the size of what came back**, not whether the run -reached the original entry point. Only 152 of 276 do the latter, and it is the -weaker signal — several packers hand control to the unpacked program in a way -the heuristic cannot distinguish from an ordinary jump, and the run then ends a -little later at `ExitProcess` with the whole image rebuilt. An entry-point rule -that fires sooner fires more often and produces *smaller* dumps; stopping late -and dumping everything beats stopping at the first plausible-looking jump. +| **Payload recovered, image incomplete** | Amber (a .NET packer: the native stub hands the assembly to the runtime, and that assembly comes back, on 11 of 12 samples) | +| **Nothing recovered**: the stub defended itself | Alienyze, TELock, Yoda-Protector | + +Recovery is all-or-nothing per sample: every sample that recovered anything +recovered an image at least as large as the input. Four of the fully unpacked +packers are ones ClamAV ships a hand-written unpacker for (ASPack, MEW, NSPack, +Yoda's Cryptor); most of the rest have no decoder anywhere. + +The measure is the size of what came back, not whether the run reached the +original entry point. Several packers hand control to the unpacked program in a +way the heuristic cannot tell from an ordinary jump, and the run then ends a +little later at `ExitProcess` with the whole image rebuilt; stopping late and +dumping everything beats stopping at the first plausible jump. ## Cost -The emulator runs only on files whose *shape* says a packer built them — an -entry point outside a read-only first section, a section reserving far more -memory than it occupies on disk, an import table with a handful of entries, a -high-entropy section under a name no linker emits. On a corpus of 654 Windows -executables that gate routed 9%. Every run is bounded by an instruction budget, -a 64 MiB address space, a dump cap, and a progress check that ends a stub -spinning in an anti-emulation delay loop rather than paying for it to finish. -Median 0.4 s per emulated file, 5 s at the 90th percentile. +The emulator runs only on files whose shape says a packer built them: an entry +point outside a read-only first section, a section reserving far more memory +than it occupies on disk, an import table with a handful of entries, a +high-entropy section under a name no linker emits. Every run is bounded by the +instruction budgets above, a 64 MiB address space, a dump cap, and a progress +check that ends a stub spinning in an anti-emulation delay loop. diff --git a/www/src/content/docs/concepts/quirks.md b/www/src/content/docs/concepts/quirks.md index b0177e48..f1986b22 100644 --- a/www/src/content/docs/concepts/quirks.md +++ b/www/src/content/docs/concepts/quirks.md @@ -1,869 +1,459 @@ --- title: Interesting quirks -description: Things about malware scanning that surprise competent developers — hard-coded Excel passwords, signature databases that ship executable programs, archives with two indexes, and why a wrong checksum must never stop a scan. +description: Things about malware scanning that surprise competent developers, from hard-coded Excel passwords and signature databases that ship programs to archives with two indexes and why a wrong checksum must never stop a scan. --- Malware scanning is full of behaviour that looks like a bug until you learn the -adversary reason for it. This page collects the ones that surprised us while -building exav, with the code that implements each. +adversarial reason for it. This page collects the ones that surprised us while +building exav. -Everything here is either exav's own design decision, a **public** specification -(Microsoft/RFC/APPNOTE), or documented/observable ClamAV behaviour. exav derives -nothing from ClamAV's GPL source; where exav matches a ClamAV behaviour it does -so from the public spec, and that is noted. +Everything here is either exav's own design, a public specification (Microsoft, +RFC, APPNOTE), or documented or observable ClamAV behaviour. exav derives nothing +from ClamAV's GPL source. ## `VelvetSweatshop`: the password that isn't a secret -Build a malware corpus, scan the spreadsheets, and a striking number report as -**encrypted / password-protected** — yet they open in Excel with no prompt at -all. The trick is a single hard-coded string: **`VelvetSweatshop`**. - -`VelvetSweatshop` is the *default* password Excel uses when a workbook is -encrypted with default settings. Excel tries it automatically on open, so a -document "encrypted" with it opens **silently**. That is exactly what a malware -author wants: - -- To a scanner that stops at "encrypted", the payload is **opaque** — it can't - see the macros or the embedded objects, so it reports clean (or, in exav's - case, `PASSWORD-PROTECTED`). -- To the victim, the file opens **normally** and the macros run. - -Encrypt-to-hide while staying zero-click on the target. It applies to legacy -`.xls` (BIFF, RC4 / RC4-CryptoAPI) and to OOXML `.xlsx`/`.docx` (AES). The -mechanism is fully public: **[MS-OFFCRYPTO]** documents the key derivation and -ciphers, **[MS-XLS] §2.4.117** documents the legacy `FilePass` record that marks -a workbook stream encrypted. - -The three schemes, from [MS-OFFCRYPTO]: - -- *Legacy XLS (BIFF8)* — the `Workbook`/`Book` stream begins `BOF` then - `FilePass`; encryption type 1 is RC4 (basic) or RC4-CryptoAPI. The key comes - from the UTF-16LE password plus a salt (MD5 for basic RC4, SHA-1 for - CryptoAPI), a verifier block confirms it, then the stream is RC4-decrypted in - 1024-byte blocks, re-keyed per block, with a handful of records left plaintext. -- *OOXML standard encryption* — an OLE2/CFB wrapper holds `EncryptionInfo` + - `EncryptedPackage`; the key is `SHA-1(salt ‖ UTF-16LE(password))` spun - **50000** times, an AES-128-**ECB** verifier confirms it, and the package - decrypts back into the real `.xlsx` ZIP. -- *OOXML agile encryption* (the modern default) — AES-256-CBC with a per-blob - KDF. - -**What exav does:** decrypts all three families in -`crates/exav-unpack/src/formats/ole_crypto.rs` — legacy XLS RC4-basic and -RC4-CryptoAPI, the older XOR obfuscation scheme (§2.3.7), and both OOXML schemes, -standard (AES-ECB) and agile (AES-CBC). `ole.rs` routes an `EncryptionInfo` + -`EncryptedPackage` compound file through the decryptor and hands the recovered -`.zip` back to the ZIP path, so the document's parts get scanned normally. -`VelvetSweatshop` and the empty password are tried by default (plus any -`--passwords`), so the common case needs no configuration at all. - -Implemented clean-room from [MS-OFFCRYPTO] §2.3.4/§2.3.5/§2.3.6/§2.3.7 and -[MS-XLS] §2.4.117, verified byte-exact against an independent oracle on real -samples and against real-Office key-derivation vectors for the agile scheme. On a -live corpus, ~240 XLS that were opaque `PASSWORD-PROTECTED` now decrypt — 180 -surfacing `Heuristics.OLE2.ContainsMacros`. Files using a *non-default* password -stay `PASSWORD-PROTECTED`, correctly: we don't know it. - -> The string has been the Excel default since the late 1990s. It looks like an -> inside joke that shipped, and two decades later it is still active malware -> infrastructure. +Scan a malware corpus's spreadsheets and a striking number report as encrypted, +yet they open in Excel with no prompt. The trick is one hard-coded string: +`VelvetSweatshop`, the password Excel uses when a workbook is encrypted with +default settings. Excel tries it automatically on open, so such a document opens +silently. For a malware author that is ideal: + +- to a scanner that stops at "encrypted", the payload is opaque: it cannot see + the macros or embedded objects; +- to the victim, the file opens normally and the macros run. + +It applies to legacy `.xls` (BIFF, RC4 and RC4 CryptoAPI) and to OOXML +`.xlsx`/`.docx` (AES). The mechanism is public: [MS-OFFCRYPTO] documents the key +derivation and ciphers, and [MS-XLS] §2.4.117 the `FilePass` record that marks a +workbook stream encrypted. The three schemes: + +- *Legacy XLS (BIFF8):* the workbook stream begins `BOF` then `FilePass`; the key + comes from the UTF-16LE password plus a salt (MD5 for basic RC4, SHA-1 for + CryptoAPI), a verifier confirms it, and the stream is RC4-decrypted in + 1024-byte blocks, re-keyed per block, with a few records left in plaintext. +- *OOXML standard encryption:* an OLE2 wrapper holds `EncryptionInfo` and + `EncryptedPackage`; the key is `SHA-1(salt ‖ UTF-16LE(password))` spun 50000 + times, an AES-128-ECB verifier confirms it, and the package decrypts back into + the real `.xlsx` ZIP. +- *OOXML agile encryption* (the modern default): AES-256-CBC with a per-blob KDF. + +exav decrypts all three, plus the older XOR obfuscation, and hands the recovered +package back to the ZIP path so the document's parts are scanned normally. +`VelvetSweatshop` and the empty password are tried by default, so the common +case needs no configuration. A file with a non-default password stays +`PASSWORD-PROTECTED`. + +> The string has been the Excel default since the late 1990s, and it is still +> active malware infrastructure. ## The signature database ships executable programs Most people assume a signature is a pattern. ClamAV's most expressive signature -type is a **program**: a `.cbc` file in `bytecode.cvd` is C compiled to a custom -LLVM-IR-like VM bytecode, shipped inside the signature feed, which runs against a -candidate file and decides whether it is malicious. - -So a scanner that wants to be compatible does not get to write a matcher. It has -to **embed an interpreter** — with everything that implies: an instruction -budget, a scratch-memory cap, a call-depth limit, a bounds-checked pointer model, -and a panic boundary per program. +type is a program: a `.cbc` file in `bytecode.cvd` is C compiled to a custom VM +bytecode, which runs against a candidate file and decides whether it is +malicious. A compatible scanner therefore has to embed an interpreter, with an +instruction budget, a scratch-memory cap, a call-depth limit, a bounds-checked +pointer model and a panic boundary per program. The format is line-oriented: a `ClamBC` header with a nibble-encoded integer -scheme (a byte `0x6N` carries nibble `N`), a **trigger line** that is itself a -`.ldb` logical signature (the program runs *only* when that matches), then -records for types, API declarations, globals, function headers, basic-block -instruction streams, and strings. Programs cannot make syscalls; they reach the -world through a fixed API of ~96 functions — `read`/`seek`/`file_find`, PE/PDF/ -JSON accessors, hashing, `disasm_x86`, and `setvirusname` to report a hit. - -The live database (v339, Sep 2025) holds **85** programs. 71 of the 85 have a -single function. The most valuable are the ~6 **unpackers** — some of ClamAV's -unpacking is implemented *as bytecode*, so a missing one silently weakens -detection across every packed sample rather than costing one signature. - -The second surprise: **the signature database is untrusted input.** It is a -file, fetched over the network, parsed by your process, and in this case -*executed* by it. ClamAV's bytecode subsystem has a documented code-execution -history ([CVE-2020-37167](https://www.cve.org/CVERecord?id=CVE-2020-37167), -CVSS 8.4 — the `bytecode_vm` sandbox escape, exploit-db 47687; the optional LLVM -JIT that generated native code from database content). exav treats the container -the same way it treats a scanned file — `crates/exav-core/src/cvd.rs` bounds -CVD/CLD extraction with explicit caps because "the database is -attacker-controllable in the threat model". - -exav's interpreter (`crates/exav-core/src/bytecode/`) is pure safe Rust, no JIT -ever, with `catch_unwind` per program. One detail worth stealing: the budget is -**step count, not wall-clock** (`bytecode/exec.rs`) — a wall-clock deadline -would make detection depend on machine load, letting an attacker (or a busy -server) push a program past the deadline to evade detection. +scheme (a byte `0x6N` carries nibble `N`), a trigger line that is itself a `.ldb` +logical signature (the program runs only when it matches), then records for +types, API declarations, globals, functions, basic blocks and strings. Programs +reach the outside world only through a fixed API of 107 functions +(`read`/`seek`/`file_find`, PE/PDF/JSON accessors, hashing, `disasm_x86`, and +`setvirusname` to report a hit). The most valuable programs are unpackers: part +of ClamAV's unpacking is implemented as bytecode, so a missing one weakens +detection across every sample of that packer. + +The second surprise: the signature database is code your process runs. ClamAV's +bytecode subsystem has a code-execution history +([CVE-2020-37167](https://www.cve.org/CVERecord?id=CVE-2020-37167), CVSS 8.4, the +`bytecode_vm` sandbox escape; and the optional LLVM JIT that generated native +code from database content). exav's interpreter is safe Rust with no JIT and a +panic boundary per program, and its budget is a step count, not wall-clock time: +a deadline would make detection depend on machine load, letting an attacker (or +a busy server) push a program past it. See [Bytecode sandbox](/concepts/bytecode-sandbox/). ## The file extension is a lie, and so is the magic number -Nobody who has worked on this trusts an extension. File type comes from content -only — `crates/exav-core/src/filetype.rs` is explicit that it "never trust[s] the -file extension". Signatures are *scoped by type* (`Target:` in a signature, and -`CL_TYPE_*` container constraints), so getting the type wrong doesn't just skip a -parser, it silently deselects a whole class of signatures. - -The part people don't expect is that **magic bytes aren't enough either**. - -- **`CA FE BA BE` is two formats.** It is the Mach-O universal ("fat") binary - magic *and* the Java `.class` magic. exav resolves it structurally, not by - guessing: `formats/machofat.rs` claims a fat binary only when `nfat_arch` is in - `1..=64`, the whole arch table is present, and every slice lies within the - file; `.class` detection additionally requires a plausible major version (≥ 45, - i.e. JDK 1.1+), which a fat header's architecture count never is. -- **Short magics collide with ordinary binary data.** bzip2 is `BZh`, CAB is - `MSCF`, gzip is two bytes. A false hit inside a PE overlay or an ISO member - gets routed to that decoder, fails deep in parsing, and would report the whole - object `UNSCANNABLE`. So `Db::identify` (`crates/exav-core/src/lib.rs`) - confirms each weak magic against a stronger structural check first — the CAB - spec's `reserved1` field must be zero, gzip's compression method must be - deflate with valid flag bits — and a failed confirmation is scanned as raw - bytes rather than poisoning the verdict. -- **HTML has no magic at all.** `filetype.rs` detects it heuristically, and - deliberately conservatively, because ClamAV applies `Target:3` signatures only - to HTML — so over-typing makes an HTML-exploit signature fire on obfuscated - JavaScript that merely contains `Uint32Array(0x..)`. -- **Text detection has to be generous about high bytes.** `looks_textual` - accepts `0x80..=0xff` freely (it gates on NUL and a >5% density of - non-whitespace control bytes) — otherwise non-English text malware types as - binary and loses its `Target:7` ASCII-text coverage entirely. -- **The same container is several types.** A `.docx`, `.xlsx`, `.apk` and `.jar` - are all ZIPs. exav reads a bounded 1 MiB prefix and classifies by member name — - `[Content_Types].xml` plus `word/document.xml` / `xl/workbook.xml` / - `ppt/presentation.xml` — because many signatures are scoped to `CL_TYPE_OOXML_*` - and would otherwise either miss real Office documents or fire on plain ZIPs. - -Type detection then re-runs on every extracted child, with one deliberate -override: textual content pulled out of an OLE2 document is forced to OLE type, -so `Target:2` macro signatures apply and generic `Target:7` text signatures do -*not* — without that, a generic text macro signature false-positives on the -standard `Name="Project"` PROJECT stream of benign macro documents. +File type comes from content only. Signatures are scoped by type (`Target:` and +`CL_TYPE_*` container constraints), so getting the type wrong does not just skip +a parser, it deselects a whole class of signatures. And magic bytes are not +enough either: + +- **`CA FE BA BE` is two formats:** the Mach-O universal ("fat") binary magic and + the Java `.class` magic. exav claims a fat binary only when the architecture + count is between 1 and 64, the whole table is present and every slice lies + within the file; `.class` detection also requires a plausible major version + (45 or more), which a fat header's architecture count never is. +- **Short magics collide with ordinary data.** bzip2 is `BZh`, CAB is `MSCF`, + gzip is two bytes. A false hit inside a PE overlay would be routed to that + decoder, fail, and report the whole object `UNSCANNABLE`, so each weak magic is + confirmed against a structural check (CAB's `reserved1` must be zero, gzip's + method must be deflate with valid flag bits), and a failed confirmation is + scanned as raw bytes. +- **HTML has no magic at all.** It is detected conservatively, because + `Target:3` signatures apply only to HTML, and over-typing makes an HTML-exploit + signature fire on obfuscated JavaScript that merely contains + `Uint32Array(0x..)`. +- **Text detection has to be generous about high bytes.** Bytes `0x80..=0xff` + count as text (the gate is on NUL and a density of control bytes); otherwise + non-English text types as binary and loses its `Target:7` coverage. +- **The same container is several types.** `.docx`, `.xlsx`, `.apk` and `.jar` + are all ZIPs. exav reads a bounded prefix and classifies by member name + (`[Content_Types].xml` plus `word/document.xml`, `xl/workbook.xml` or + `ppt/presentation.xml`), because many signatures are scoped to + `CL_TYPE_OOXML_*`. + +Typing re-runs on every extracted child, with one override: text pulled out of an +OLE2 document is typed as OLE, so `Target:2` macro signatures apply and generic +`Target:7` text signatures do not. Without that, a generic text signature fires on +the standard `Name="Project"` stream of benign macro documents. ## `cat malware huge.pad > evil` is a one-line bypass -Every scanner has a maximum file size. The naive implementation — "file is over -the limit, skip it, report OK" — hands the adversary a bypass they can execute -with one shell command: append padding until you cross the threshold. - -The strongest published example is ClamAV's long-standing large-file behaviour: -a file over roughly 2 GB is **read but scanned as zero bytes, and still reported -`OK`**. Not "skipped", not "error" — clean. +Every scanner has a maximum file size. The naive implementation ("over the limit, +skip it, report OK") hands the attacker a bypass: append padding until the file +crosses the threshold. ClamAV's large-file behaviour is the known example: a file +over about 2 GB is read, scanned as zero bytes, and reported `OK`. -exav's rule, stated in the `exav-core` crate docs as anti-evasion invariant 2, is -**never refuse by size without scanning**. `scan_path` and `scan_seekable` -(`crates/exav-core/src/lib.rs`) run the flat pattern+hash core over the budgeted -prefix *first*; only if that finds nothing does the file get a limit verdict: +exav's rule is that refusing by size still scans. An input over +`--max-input-bytes` has its first `--max-input-bytes` bytes scanned first, as +any smaller input would be, and only then gets a limit verdict: ```text -file size 5368709120 exceeds max-scan-size 104857600; scanned first 104857600 bytes only +file size 5368709120 exceeds max-input-bytes 104857600; scanned first 104857600 bytes only ``` -Invariant 1 is the companion: **a detection always beats a limit.** A hit is -never downgraded to `LIMITS-EXCEEDED` or `CLEAN` because some *other* part of the -input tripped a budget. - -And invariant 3 is the one the rest hangs on: not-fully-scanned is never -`CLEAN`. See [Design principles](/concepts/design-principles/#never-a-silent-clean) -and [Verdicts & exit codes](/reference/verdicts/). - -> exav's streaming core makes this mostly moot for the size case anyway: it -> matches in a single forward pass with a flat ~2 MiB working set, so a 6 GiB file -> on a 4.8 GiB machine is scanned end to end. See -> [Streaming & memory](/concepts/streaming-memory/). +A detection always beats a limit, and an incomplete scan is never `OK`. See +[Design principles](/concepts/design-principles/#never-a-silent-clean). ## Containers have two indexes, and malware uses the one you don't read -A ZIP file lists its members twice: once as a **local file header** immediately -before each member's data, and once in the **central directory** at the end. Every -normal reader — including the `zip` crate, including your language's stdlib — -trusts the central directory, because that's what APPNOTE says is authoritative. - -That means you can hide a member: leave it out of the central directory while -leaving its local header and data in the file. Many extractors will still write it -out. A scanner that only enumerates the central directory never sees it. - -exav does **dual indexing** in `formats/zip.rs`: after the normal central-directory -pass, `scan_orphan_locals` builds a set of every known `header_start()`, scans the -raw bytes for `PK\x03\x04` local headers not in that set, and extracts those too. -The same path doubles as salvage — if the central directory is unparseable -(corrupt, forged, truncated), exav doesn't give up, it falls through to the raw -local-header scan. - -The subtle part is what to do with an orphan you *can't* read. Skipping it is the -tempting default and it is wrong: the header is credible, so the bytes are a -member the target will extract, and dropping it lets the containing file be -called clean on a scan that never looked inside. So every failure is reported -instead — encrypted, unsupported codec, size deferred to a trailing data -descriptor, or a declared extent running past EOF each yield a metadata-only -`Entry::unsupported` and an `UNSCANNABLE` verdict. - -The counterweight is that `PK\x03\x04` is four bytes and turns up by chance in -ordinary binaries, so reporting every hit would make clean files unscannable. -`plausible_local_header` gates on the fields a real writer must fill in -consistently — version ≤ 6.3, no reserved flag bits, a method APPNOTE actually -defines, and a name that is non-empty, within the path limit and free of NULs. -Across 3,000 live malware samples that gate produced no spurious hits at all. - -There is deliberately **no** separate cap on orphan count: each one is charged to -the archive-wide `max_members` budget, so exceeding it raises `LIMITS-EXCEEDED` -rather than quietly stopping. An arbitrary cap was in fact the bug — a real JAR -turned up whose End Of Central Directory record was simply gone, leaving 414 -members reachable only as local headers, and a few hundred members is completely -ordinary for a JAR. - -One more distinction in the code: an unparseable central directory is classified -`corrupt` → `UNSCANNABLE`, never `LIMITS-EXCEEDED`, because a resource bound and -a forged index are different claims. - -**ISO images have the same shape, and the trick is used more brazenly.** A CD -image carries several volume descriptors, each with its own independent directory -tree over the same sectors — the primary ISO9660 tree and the Joliet -supplementary tree (long Unicode names). A malicious ISO lists its payload in -**only one** of them, typically Joliet, leaving the primary tree empty, so a -reader that parses the other sees an empty disc. `formats/iso.rs::vd_roots` -enumerates every volume descriptor from sector 16 until the terminator and walks -the root of *each* primary and Joliet tree, Joliet first so the real long names -win when a file appears in both. The regression test names the shape it was -written for: an `Invoice.pdf.lnk` delivered inside an ISO. - -**gzip has this problem too, in miniature.** RFC 1952 §2.2 allows a `.gz` to be -several concatenated members, and `gzip`/`zcat` decompress all of them. Rust's -`flate2::GzDecoder` stops after the first. `formats/gzip.rs` uses -`MultiGzDecoder` because of an observed false negative: a two-member gzip whose -first member is a 1 KiB header and whose second holds the malware. - -Related, and the reason "type by content" isn't sufficient on its own: the same -carving logic runs for **appended** archives. `pe::embedded_archive_offsets` -enumerates ZIP/gzip/bzip2/xz/7z/RAR/CAB magics at offset > 0 — SFX stubs, PE -overlays, droppers that staple a ZIP onto a carrier. The dedicated SFX carver -(`formats/sfx.rs`) only claims one when the file *starts* with `MZ`/`ELF` **and** -an archive magic appears at least `MIN_SFX_OFFSET = 64` bytes in. - -A subtle rule falls out of that: when a carve guess fails to decode, exav must -**silently drop it**, not mark the carrier `UNSCANNABLE`. A false `1f8b08` byte -run inside a PE is not an encrypted archive; the carrier was already fully -pattern-scanned, and poisoning it would be a false positive against a scanner -that (correctly) calls such files clean. +A ZIP lists its members twice: a local file header before each member's data, +and the central directory at the end. Every normal reader trusts the central +directory, because APPNOTE says it is authoritative. So a member can be hidden: +leave it out of the central directory and keep its local header and data. Many +extractors still write it out. + +exav does dual indexing: after the central-directory pass it scans the raw bytes +for local headers the directory did not claim and extracts those too, and the +same path is the salvage route when the directory is corrupt or missing. An +orphan it cannot read (encrypted, an unsupported codec, a size deferred to a +data descriptor, an extent past the end of the file) is reported rather than +skipped: its header is credible, so it is a member the target will extract. + +`PK\x03\x04` is four bytes and appears by chance in ordinary binaries, so a +local header only counts if the fields a real writer fills in are consistent +(version at most 6.3, no reserved flag bits, a defined method, a non-empty name +within the path limit and free of NULs). Orphans are charged to the archive-wide +member budget rather than to a separate cap: a real JAR whose end-of-directory +record was gone had over 400 members reachable only as local headers. + +**ISO images have the same shape**, and the trick is used more openly. A CD image +carries several volume descriptors, each with its own directory tree over the +same sectors: the primary ISO9660 tree and the Joliet tree with long Unicode +names. A malicious ISO lists its payload in only one of them, leaving the other +empty, so a reader that parses the other sees an empty disc. exav walks every +tree, Joliet first so the long names win. + +**gzip has the problem in miniature.** RFC 1952 allows a `.gz` to be several +concatenated members, and `gzip`/`zcat` decompress all of them, while Rust's +`flate2::GzDecoder` stops after the first. A two-member gzip whose first member +is a 1 KiB decoy and whose second holds the malware was a real false negative. + +**Appended archives** are carved too: ZIP, gzip, bzip2, xz, 7z, RAR and CAB +magics after offset 0 (self-extractors, PE overlays, droppers that staple an +archive onto a carrier). A carve that fails to decode is usually a chance byte +run and is dropped rather than making the carrier `UNSCANNABLE`. The exception is +an encrypted ZIP appended to a picture or a document whose central directory +checks out: archive tools open such a polyglot through its trailing directory, so +it is reported `PASSWORD-PROTECTED`. ## "Encrypted" is itself a detection -Once you accept that malware encrypts things to blind scanners rather than to -protect them, encryption stops being an obstacle and becomes a feature you can -match on. - -**ClamAV's `.cdb` container-metadata format makes it a field.** The line is +Once malware encrypts to blind scanners rather than to protect anything, +encryption becomes something to match on. ClamAV's `.cdb` container-metadata +format makes it a field: ```text VirusName:ContainerType:ContainerSize:FileNameREGEX:FileSizeInContainer:FileSizeReal:IsEncrypted:FilePos:Res1:Res2[:MinFL[:MaxFL]] ``` -`IsEncrypted` is three-state (`1`, `0`, `*`), so a signature can say *"a ZIP -containing an encrypted member whose name matches `(?i)invoice.*\.exe`"* and fire -without ever decrypting anything. exav implements this in -`crates/exav-core/src/container.rs`. A real signature from `daily.cvd` shows how -much a metadata-only rule can do: +`IsEncrypted` is three-state (`1`, `0`, `*`), so a signature can say "a ZIP +containing an encrypted member whose name matches `(?i)invoice.*\.exe`" and fire +without decrypting anything. A real signature from `daily.cvd`: ```text Archive.Filetype.DualExtJS-6168221-2:CL_TYPE_ZIP:*:^[^/\\]+\.(doc|xls|ppt|pdf|png|gif|jpeg)\.js$:*:*:*:1:*: ``` -That is a double-extension detector — it catches -`PurchaseOrder_006231_Shanghuigou_20260605.pdf.js` by **name and position alone**, -with no content signature at all. (The `FilePos` field is 1-based. exav numbered -from 0 at first and missed every one of these — the test in `container.rs` -records the regression.) - -Two more places encryption shows up as signal: +That is a double-extension detector: it catches +`PurchaseOrder_006231_Shanghuigou_20260605.pdf.js` by name and position alone +(`FilePos` is 1-based). -- `--partial-as password-protected=found` turns a password-protected member - into an actual detection, - `Heuristics.Encrypted.Zip` / `.RAR` / `.7Zip` / `.PDF` / `.Doc` - (`encrypted_heuristic_name` in `crates/exav-core/src/lib.rs`). -- exav ships a **default password list**. `formats/zip.rs`: +Two more places encryption is a signal: - ```rust - const DEFAULT_ZIP_PASSWORDS: &[&str] = &["infected", "virus", "malware", "password", "123456"]; - ``` - - These are the malware-distribution conventions — "infected" is the standard - password for sharing samples — so a password-protected dropper cracks with zero - configuration, exactly as `VelvetSweatshop` does for Office. A `.pwdb` database - file and `--passwords` extend the pool. +- `--partial-as password-protected=found` turns a password-protected member into + a detection, `Heuristics.Encrypted.Zip` / `.RAR` / `.7Zip` / `.PDF` / `.Doc`. +- exav tries a default password list on ZIPs: `infected`, `virus`, `malware`, + `password`, `123456`. "infected" is the standard password for sharing samples, + so a password-protected dropper opens with no configuration, as + `VelvetSweatshop` does for Office. `.pwdb` databases and `--passwords` extend + the pool. ## A wrong checksum must never stop the scan -This one reads like a bug in every code review. exav extracts archive members -**without verifying their CRCs**, and `Budget::verify_checksums` is off by -default (`crates/exav-unpack/src/lib.rs`). - -The reason is in the comment: a malware scanner scans decompressed content -regardless of integrity metadata, because *"a wrong CRC must never stop a -member's bytes from being scanned — that would let an attacker downgrade a -detection by flipping a checksum byte."* One byte, anywhere in a trailer, and a -strict extractor stops looking. (This matches ClamAV, which ignores CRCs when -scanning. Turning verification on requires both the `checksums` Cargo feature and -an explicit opt-in, and is for extract-for-real use where a bad CRC is a genuine -"corrupt file" signal.) - -The same instinct generalises. A CAB whose `CFHEADER.cbCabinet` total-size field -is overwritten with `0xFFFFFFFF` defeats a strict parser; exav clamps it and -extracts the member anyway. A gzip with a deliberately corrupted CRC-32 trailer -still yields its payload. A truncated deflate stream is salvaged up to the cut, -because the malware is usually in the recovered prefix — half-downloaded -droppers and deliberately mangled tails are common in the wild, and `zcat` -recovers them too. - -Where the leniency has to *stop* is structure, not integrity. `formats/ole.rs` -falls back to a flat directory walk when a strict CFB reader rejects a compound -file (a broken red-black-tree sibling ordering, e.g. `_VBA_PROJECT` sorted before -`dir`, which many real Office documents and much malware carry) — but a member -that is *present and unreadable* still surfaces as not-fully-scanned rather than -clean. - -> The nuance for truncation: exav scans for malware, it is not a file-integrity -> validator. When a *sequential* stream runs out of input, every byte that exists -> has been scanned — the missing tail is absent, not hidden — so a clean result is -> a real clean. The not-fully-scanned verdicts are reserved for content that is -> **present but unscanned**: an encrypted member, an unsupported codec, or a -> member skipped to stay under a limit. +This reads like a bug in every code review: exav extracts archive members without +verifying their CRCs, and verification is off by default. A wrong CRC must never +stop a member's bytes from being scanned, or an attacker could downgrade a +detection by flipping one checksum byte. (ClamAV also ignores CRCs when scanning. +Verification needs the `checksums` Cargo feature and an explicit opt-in, for +extraction where a bad CRC is a real "corrupt file" signal.) + +The same instinct generalises. A CAB whose total-size field is overwritten with +`0xFFFFFFFF` defeats a strict parser; exav clamps it and extracts the member. A +gzip with a corrupted CRC-32 trailer still yields its payload. A truncated +deflate stream is salvaged up to the cut, because the malware is usually in the +recovered prefix, and `zcat` recovers it too. + +Leniency stops at structure. An OLE file a strict reader rejects (a broken +red-black-tree ordering, common in real Office documents and malware) is walked +as a flat directory instead, but a member that is present and unreadable is +still reported, not treated as clean. + +> When a sequential stream runs out of input, every byte that exists has been +> scanned: the missing tail is absent, not hidden, so a clean result is a real +> clean. The incomplete verdicts are for content that is present but unscanned: +> an encrypted member, an unsupported codec, a member over a limit. ## Identity that survives the bytes changing -Two files can share nothing byte-for-byte and still be provably the same malware. -Three mechanisms in the signature formats do this, each surprising in its own way. - -**imphash** (`crates/exav-core/src/pe.rs`) is the MD5 of the PE's *import table* -— comma-joined lowercase `dll_without_extension.function`, in import-table order -(the Mandiant definition). Recompile, repack, change every string, pad the -sections: as long as the binary links the same API functions in the same order, -the imphash is identical. exav loads these from `.imp` signatures -(`PEImportTableHash:PEImportTableSize:MalwareName`) where the "size" field is not -a byte count at all — it is the **number of imports**. - -The trap is ordinals. Imports by ordinal contribute `dll.ord`, not a function -name. If your PE parser renders an ordinal import as the string `ORDINAL 42` and -you hash that, or you drop ordinal imports entirely, you get a different hash and -miss every `.imp` signature. This was a real differential-testing finding. - -**Section hashes** (`.mdb` MD5, `.msb` SHA) hash one PE section instead of the -file, so an unchanged code section is still recognised after the resources, -overlay, or certificate table change. Two quirks: - -- The field order is **transposed** relative to whole-file hashes. `.hdb` is - `HASH:SIZE:NAME`; `.mdb` is `SIZE:HASH:NAME`. `crates/exav-core/src/hashes.rs` - keeps two separate parsers for exactly this reason. (The digest algorithm is - inferred from hex length: 32 → MD5, 40 → SHA-1, 64 → SHA-256.) -- The size key and the hashed bytes can legitimately disagree. `pe::section_slices` - keys on the section's **declared** `SizeOfRawData` but digests only the bytes - that actually exist on disk — overlay-trimmed and truncated PEs declare a raw - size that overruns EOF, and dropping those sections loses real detections. - -**TLSH** (`crates/exav-core/src/fuzzy.rs`) goes further: a locality-sensitive -whole-file hash matched by **distance**, not equality, so near-variants of a known -sample are caught. The signature carries its own threshold -(`tlsh:HASH:Name[:MaxDistance]`, default 100), which means the lookup is a linear -scan with a distance computation rather than a hash-table probe — a completely -different cost shape from every other signature type. It also silently declines on -inputs under ~50 bytes or too uniform to digest, so small droppers simply have no -fuzzy identity. +Two files can share nothing byte for byte and still be provably the same malware. + +**imphash** is the MD5 of the PE's import table: comma-joined lowercase +`dll_without_extension.function` in import order (the Mandiant definition). +Recompile, repack, change every string: as long as the binary links the same +functions in the same order, the imphash is identical. In `.imp` signatures +(`PEImportTableHash:PEImportTableSize:MalwareName`) the "size" is not a byte +count but the number of imports. The trap is ordinals: an import by ordinal +contributes `dll.ord`, and a parser that renders it as `ORDINAL 42`, or drops +it, gets a different hash and misses every `.imp` signature. + +**Section hashes** (`.mdb` MD5, `.msb` SHA) hash one PE section rather than the +file, so an unchanged code section is recognised after the resources, overlay +or certificate table change. Two quirks: the field order is transposed relative +to whole-file hashes (`.hdb` is `HASH:SIZE:NAME`, `.mdb` is `SIZE:HASH:NAME`), +and the size key and the hashed bytes can disagree: the key is the declared +`SizeOfRawData`, while only the bytes that exist are hashed, because truncated +PEs declare sections that run past the end of the file. + +**TLSH** is a locality-sensitive whole-file hash matched by distance rather than +equality, so near-variants of a known sample are caught. The signature carries +its own threshold (`tlsh:HASH:Name[:MaxDistance]`, default 100), so lookup is a +linear scan with a distance computation rather than a hash-table probe. It +declines inputs under about 50 bytes or too uniform to digest, so small droppers +have no fuzzy identity. ## Scanning icons, because malware dresses up -A trojan that wants to be double-clicked wears a familiar icon — Chrome, Adobe -Reader, a Word document, a folder. So AV engines hash icons **perceptually**, and -a signature can require *"these bytes AND this executable is wearing that icon."* - -exav implements ClamAV's `.idb` format in `crates/exav-core/src/icon.rs`. The -surprising parts: - -- The "hash" is not a digest. It is a 124-nibble hex blob that unpacks into a - struct: for each of six derived fields — colour, grayscale, bright, dark, edge, - non-edge — the average value and (x, y) location of the three most extreme - non-overlapping k×k windows, plus RGB sums and a colour-pixel count. The edge - field is a CIE-Lab colour-distance map, Sobel-filtered, normalised, bordered and - Gaussian-blurred. -- Matching is a **confidence score against a threshold**, not equality: ≥ 70 for - black-and-white icons, and 72/68/64 for 16/24/32-pixel colour icons. Only those - three icon sizes exist in the format; any other side length is dropped. -- The icon is never a detection by itself. It is an extra AND-clause: a logical - signature's `IconGroup1:`/`IconGroup2:` TDB fields add "…and the PE's icon - perceptually matches an `.idb` entry in these groups". Without an icon context - the constraint can't be satisfied, so the signature does not fire. -- Getting the icon out means walking PE resources: find `RT_GROUP_ICON` (type 14), - read the icon count, walk 14-byte `GRPICONDIRENTRY` records, resolve each - `icon_id` under `RT_ICON` (type 3), then decode the DIB. - -exav also supports the separate `fuzzy_img#<16-hex>` logical subsignature, which -is a **different algorithm entirely** — the 64-bit DCT perceptual hash from -Python's `imagehash` (`phash()`, median variant), matched by Hamming distance -(`crates/exav-core/src/fuzzy_img.rs`). Reproducing it byte-exactly means pinning -things that normally don't matter: BT.601 grayscale coefficients with round-half- -away-from-zero, a Lanczos3 resize to exactly 32×32 with no aspect preservation, -a ×2 scale after each 1-D DCT pass, the top-left 8×8 block including DC, a strict -`>` median threshold, MSB-first packing. Any one of those wrong and you match -nothing. - -## Signature matching is a regex engine, and a step budget is a trap - -`.ndb` bodies are hex with wildcards, and the wildcard set is richer than most -people expect (`crates/exav-core/src/engine/parse.rs::parse_elems`): - -| Construct | Meaning | -|---|---| -| `??` | any byte | -| `a?` / `?a` | nibble wildcard (high or low nibble fixed) | -| `*` | unbounded gap | -| `{n}` `{n-m}` `{-m}` `{n-}` | exact / range / at-most / at-least gap | -| `[n-m]` | same matching semantics as `{n-m}` | -| `(aa\|bb)` / `!(aa\|bb)` | alternation / negated alternation | -| `(B)` `(L)` `(W)` | **not** alternations — word/line/word-marker boundaries | - -That last row is the trap: `(B)`, `(L)` and `(W)` look exactly like one-option -alternations and are actually character-class boundary markers. A negated -alternation additionally requires all branches to be the same length, or the -whole signature is rejected. - -Two consequences follow. - -**A pattern with gaps is a regex engine, and the obvious way to run one is -exponential.** Verifying `A*B*C` by recursive descent means trying every gap -length at every gap; on repetitive content (a run of one byte, obfuscated -JavaScript) the branches multiply and a single signature can eat the CPU. The -usual answer is a step budget — but for a scanner a budget is a trap, because -"gave up looking" and "looked and found nothing" are the same return value, and -reporting the first as clean is exactly the silent-clean bug. - -So exav doesn't backtrack at all. In `crates/exav-core/src/engine/mod.rs` a token -program is run as a Thompson-style simulation over a set of reachable position -*intervals* (`gap_split_match` forward, `gap_split_match_backward` for a pattern -whose anchor sits past a gap). An unbounded gap becomes one interval expansion -instead of a per-length loop, and the literal after a gap is located once across -the whole window rather than once per surviving branch. The cost goes polynomial -and the answer is identical — including *which* start position is reported, which -the backward simulator reconstructs by replaying the old walk's preference order -against the reachable sets rather than by searching. - -A recursive walk is kept behind `EXAV_SPLIT_MATCH=0`, purely so a scan can be -run both ways and the verdicts diffed. It keeps the tight -`VERIFY_BUDGET` / `SCAN_VERIFY_BUDGET` caps; the simulator draws from a far -larger `SIM_BUDGET`, kept separate so the safe path's headroom can never relax -the dangerous path's cap. - -And the part that matters either way: when a pool runs out, the matcher has -*stopped* looking, not *finished* looking. Returning "no match" at that point -would report the file clean on the strength of a search that never completed — -the silent-clean bug, arrived at through the back door. So exhausting a pool -instead sets a `SCAN_TRUNCATED` flag, and the top-level scan reports -`LIMITS-EXCEEDED` rather than `CLEAN`. - -**The prefilter's anchor choice is counter-intuitive.** Everything is fed through -one shared Aho-Corasick automaton keyed on a literal "anchor" extracted from each -pattern, and the obvious heuristic — pick the longest literal run — is wrong. A -long run of a constant byte (zero padding, `0xFF` fill, BSS) matches repetitive -content millions of times, so it is a terrible prefilter *because* it is long. -`anchor_score` down-ranks by distinct-symbol count: a constant run scores 1, a -two-symbol repeat scores at most 3, and only a varied run scores its length. For -some patterns the best anchor sits *past* a variable gap, so exav anchors there -and verifies backwards across the gap — which cuts automaton hits by orders of -magnitude. - -A related size fact: in a real `daily.cvd` (~356k -signatures), `.ldb` logical signatures are **282,947** of them and `.ndb` bodies -are **312**. All the memory is the automaton. See -[Signatures](/guides/signatures/) and the -[architecture overview](/concepts/architecture/). - -## Six bounds, and the worst bomb isn't a zip bomb - -A decompression bomb defence that is only a maximum output size doesn't work: -set it low and you break legitimate large archives, set it high and a few -kilobytes of input still buys the attacker gigabytes of work. So there is no one -bound — there are six, and they measure different things -(`crates/exav-unpack/src/lib.rs`): - -| Bound | Default | What it stops | -|---|---|---| -| `max_compression_ratio` | 1000 | one stream that expands absurdly | -| `max_extracted_bytes` | 1 GiB | cumulative decompressed bytes across the whole recursion | -| `max_buffer_bytes` | 256 MiB | the largest **single** forced-materialisation — one buffer, of the several live at once | -| `max_members` | 100000 | member-count blowup | -| `max_recursion` | 16 | nesting depth | -| `max_scanned_bytes` | 10 GiB | cumulative bytes fed to the *matcher* | - -Every counter is crate-private and mutable only through `reserve`/`commit`/ -`charge_scan`/`count_entry`, so a caller can't bypass the defence by writing -them. Note that the ratio check is explicitly the *weak* one: the input size it -divides by is an attacker-declared header field, so `ratio_guard` is only a fast -reject and the absolute caps are the real bound. - -**The last row is the surprising one, and it isn't about decompression at all.** -Malware gets appended into carriers, so a scanner carves embedded PE images and -re-scans each one. A crafted disk image full of embedded executables makes that -**quadratic**: every carved suffix is itself a buffer containing all the later -embedded PEs. A 58 MB VHD was enough to run for hours. Two fixes, both in -`deep_analyze` (`crates/exav-core/src/lib.rs`): recursion into a carved image -passes `carve = false`, because the parent already enumerated every embedded -offset and a carved suffix is a subset; and every carve is charged against -`max_scanned_bytes`, which is deterministic and therefore trips identically on every -machine — unlike a wall-clock deadline, which makes verdicts depend on load. - -The subtlest bug in this whole area is a one-liner. The obvious way to cap a -member is `reader.take(cap)` — and at the cap, `take` returns `Ok(0)`, which is -**byte-for-byte indistinguishable from EOF**. The scanner reads its 256 MiB, -sees a clean end-of-stream, finds nothing, and reports a 50 GB bomb as fully -scanned and clean. exav's `BudgetReader` (`crates/exav-unpack/src/stream.rs`) -reads **one extra byte** at the cap purely to tell "ended" from "truncated": - -```rust -// At the cap. Probe one byte: if the member has more, it is a bomb. -``` - -If that probe returns data, the read fails with a `BUDGET_OVERFLOW` sentinel that -the extraction walk converts into `LIMITS-EXCEEDED` — so an over-budget member is -reported, never silently cut short and treated as fully scanned. - -A third class never decompresses anything: a header field that *lies about a -size* to drive a huge `Vec::with_capacity`. No data ever arrives; the allocation -alone is the DoS (and a capacity-overflow abort is **not** catchable by -`catch_unwind`). -exav caps attacker-driven pre-allocation at `PREALLOC_CAP = 16 MiB` and lets -buffers grow on demand under the budget instead. The 7z header parser goes -further and bounds a declared sub-stream count by `r.remaining()` — the bytes -physically left in the header — since every declared stream must cost header bytes -to describe. +A trojan that wants to be double-clicked wears a familiar icon (Chrome, Adobe +Reader, a Word document, a folder), so engines hash icons perceptually, and a +signature can require "these bytes and this icon". exav implements ClamAV's +`.idb` format: + +- The "hash" is not a digest. It is a 124-nibble blob that unpacks into, for each + of six derived fields (colour, grayscale, bright, dark, edge, non-edge), the + average value and position of the three most extreme non-overlapping windows, + plus RGB sums and a colour-pixel count. The edge field is a CIE-Lab + colour-distance map, Sobel-filtered, normalised, bordered and Gaussian-blurred. +- Matching is a confidence score against a threshold: 70 or more for black and + white icons, 72/68/64 for 16/24/32-pixel colour icons. Only those three sizes + exist in the format. +- An icon is never a detection by itself. It is an extra condition: a logical + signature's `IconGroup1:`/`IconGroup2:` fields add "and the PE's icon matches an + `.idb` entry in these groups". +- Getting the icon out means walking PE resources: `RT_GROUP_ICON` (type 14), its + 14-byte entries, each `icon_id` under `RT_ICON` (type 3), then the DIB. + +The separate `fuzzy_img#<16-hex>` subsignature is a different algorithm: the +64-bit DCT perceptual hash from Python's `imagehash` (`phash()`, median variant), +matched by Hamming distance. Reproducing it exactly means pinning details that +normally do not matter: BT.601 grayscale with round-half-away-from-zero, a +Lanczos3 resize to exactly 32×32, a ×2 scale after each 1-D DCT pass, the +top-left 8×8 block including DC, a strict `>` median threshold, MSB-first +packing. Get one wrong and nothing matches. ## The string `EXEC` is not in the file -Excel 4.0 macros — XLM — predate VBA by years. The macro lives in the cells of a -dedicated *macro sheet* in the BIFF stream, not in a VBA project, so it slips -straight past VBA-only macro detection. That alone made XLM a popular delivery -mechanism long after the format was obsolete. - -The part that breaks naive scanning is *how* the formulas are stored. A macro -sheet is marked by a `BOUNDSHEET` record (`0x0085`) with sheet type 1; the macros -live in `FORMULA` records as `ptg` token streams. Built-in function names — -`EXEC`, `CALL`, `ALERT` — are stored as **numeric ids**. The literal bytes `EXEC` -never appear in the file, so a byte-pattern signature for them cannot match. - -exav's `formats/xlm.rs` walks the BIFF record stream and synthesises an -`xlm_macro` artifact carrying the macro-sheet names, the recovered `ptgStr` -string constants, and each formula's tokens *with the function ids decoded back -to names* — so `Target:2`/`Doc.*` signatures match the real calls and -`--detect macros` fires for XLM the way it does for VBA. - -Adjacent, same theme — content that must be decoded, or re-normalised, before any -signature can see it: - -- **RTF embedded objects** are ASCII hex inside an `\objdata` destination, and - the hex may be split across lines and interleaved with control words. That is - a deliberate parser-differential: the control words' letters (`a`–`f`) would be - eaten as hex digits by a naive scraper, shifting every subsequent nibble. - Decoding needs a real RTF tokenizer (`formats/rtf.rs`), including the rule that - a single trailing space belongs to the control word. -- **VBA line continuations** split identifiers across lines. `Sub _`⏎ - `UserForm_Activate` is the same code as `Sub UserForm_Activate`, but no - signature written against the second matches the first. The source has to be - re-joined before matching — with the extra subtlety that an underscore *inside* - an identifier must be left alone (`formats/vba.rs`). -- **PDF name objects** can hex-escape characters that never needed escaping: - `/J#61vaScript`, `/Ope#6eAction`. The PDF spec only requires `#`-escaping for - whitespace, the delimiters, and bytes outside `0x21..=0x7E`, so escaping a plain - alphanumeric is gratuitous — and the gratuitous escape *is* the signal. - `has_obfuscated_name_object` (`formats/pdf.rs`) fires only when the de-escaped - name is one of a fixed sensitive list (`JavaScript`, `OpenAction`, `Launch`, - `EmbeddedFile`, …), because a lone incidental escape like `/C#31` is common in - benign PDFs. That module also ignores the xref table entirely and byte-scans for - `N G obj`, since a broken xref is normal in malicious PDFs. -- **Microsoft's Script Encoder** (`#@~^` … `^#~@`, shipped as `.vbe`/`.jse` or - inside `"), FileType::Html); // Obfuscated JS with no HTML tags must NOT be Html (so Target:3 HTML - // exploit sigs don't fire on it — the npm-package FP). It is text, so it + // exploit sigs don't fire on it, as in the npm-package FP). It is text, so it // types as Text (covered by Target:7), never Html. assert_eq!( identify(b"const _0x12=_0x37;var a=new Uint32Array(0x10000);for(;;){}"), diff --git a/crates/exav-core/src/fuzzy.rs b/crates/exav-core/src/fuzzy.rs index 1e24c2ba..e6d3f52f 100644 --- a/crates/exav-core/src/fuzzy.rs +++ b/crates/exav-core/src/fuzzy.rs @@ -68,6 +68,11 @@ impl FuzzyDb { self.len() == 0 } + /// Whether any imphash signature is loaded. + pub(crate) fn has_imphash(&self) -> bool { + !self.imphash.is_empty() || !self.imp_sized.is_empty() + } + /// Parse a fuzzy-signature file. One entry per line: /// `imphash:HEX:Name` /// `tlsh:HASH:Name[:MaxDistance]` (default distance 100) @@ -158,10 +163,20 @@ impl FuzzyDb { /// Distance-based TLSH match; returns the closest signature within its /// configured threshold. pub fn match_tlsh(&self, data: &[u8]) -> Option { + self.match_tlsh_source(&data) + } + + /// As [`Self::match_tlsh`], over an object that need not be held in memory. + pub(crate) fn match_tlsh_source(&self, data: &dyn crate::byte_source::ByteSource) -> Option { if self.tlsh.is_empty() { return None; } - let sample = tlsh_of(data)?; + let mut b = TlshDefaultBuilder::new(); + data.chunks(0, data.len(), &mut |_, piece| { + b.update(piece); + true + }); + let sample = b.build()?; let mut best: Option<(i32, &str)> = None; for (t, name, max) in &self.tlsh { let d = sample.diff(t, true); @@ -237,4 +252,16 @@ mod tests { // identical content -> distance 0 -> matches assert_eq!(db.match_tlsh(&data).as_deref(), Some("Family.B")); } + + #[test] + fn a_digest_fed_in_chunks_is_the_one_made_whole() { + use crate::byte_source::{BlockCache, CHUNK}; + let data: Vec = (0..5 * CHUNK + 77).map(|i| ((i * 31 % 251) ^ (i / 97)) as u8).collect(); + let h = tlsh_string(&data).expect("tlsh"); + let mut db = FuzzyDb::new(); + db.extend_from_text(&format!("tlsh:{h}:Family.C:0\n")); + let cache = BlockCache::with_sizes(std::io::Cursor::new(data), 509, 4096).unwrap(); + // Distance 0 only: the chunked digest must be exactly the whole one. + assert_eq!(db.match_tlsh_source(&cache).as_deref(), Some("Family.C")); + } } diff --git a/crates/exav-core/src/hashes.rs b/crates/exav-core/src/hashes.rs index 20eefa1e..11f0c8b4 100644 --- a/crates/exav-core/src/hashes.rs +++ b/crates/exav-core/src/hashes.rs @@ -1,8 +1,5 @@ -//! Hash signatures (`.hdb`/`.hsb`-style) and a streaming -//! tee-hasher that computes MD5/SHA1/SHA256 in the same single pass used -//! for pattern matching — so whole-file hash detection works at any size. - -use std::io::{self, Read}; +//! Hash signatures (`.hdb`/`.hsb`-style) and the MD5/SHA1/SHA256 digests they +//! are matched against, computed in one pass over an object of any size. use md5::Md5; use serde::{Deserialize, Serialize}; @@ -136,9 +133,34 @@ macro_rules! digest_table { fn get(&self, key: &[u8; $n], size: u64) -> Option<(&str, bool)> { self.sized.get(&(size, *key)).or_else(|| self.any.get(key)) } + /// Whether a signature could match an object of `size` bytes: one + /// of any size, or one of exactly that size. + fn wants(&self, size: u64) -> bool { + let keys = &self.sized.keys; + let i = keys.partition_point(|k| k.0 < size); + !self.any.keys.is_empty() || keys.get(i).is_some_and(|k| k.0 == size) + } } }; } + +/// Which digests a lookup can use for an object of one size. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub(crate) struct Want { + pub(crate) md5: bool, + pub(crate) sha1: bool, + pub(crate) sha256: bool, +} + +impl Want { + pub(crate) fn union(self, o: Want) -> Want { + Want { + md5: self.md5 || o.md5, + sha1: self.sha1 || o.sha1, + sha256: self.sha256 || o.sha256, + } + } +} digest_table!(Md5Table, 16); digest_table!(Sha1Table, 20); digest_table!(Sha256Table, 32); @@ -230,6 +252,17 @@ impl HashDb { } } + /// The digests [`Self::lookup`] can match for an object of `size` bytes: + /// only those of an algorithm with a signature of any size, or of that + /// size. Most objects need none of the SHA ones, and some none at all. + pub(crate) fn wants(&self, size: u64) -> Want { + Want { + md5: self.md5.wants(size), + sha1: self.sha1.wants(size), + sha256: self.sha256.wants(size), + } + } + /// Look up computed digests for a file of `size` bytes; returns the matching /// signature's `(clean_name, unofficial)`. A sized signature matches only at /// that exact length; a `*` signature matches any length. @@ -290,7 +323,7 @@ impl SectionHashDb { } /// Parse `.mdb`/`.mdu`/`.msb` lines (`SectionSize:HASH:Name`). The first - /// field is the size, the second the hash — the opposite field order from + /// field is the size, the second the hash: the opposite field order from /// `.hdb`, so this does not reuse `parse_hash_line`. pub fn extend_from_text(&mut self, text: &str) { self.extend_from_text_prov(text, false); @@ -373,55 +406,6 @@ pub struct Digests { pub sha256: String, } -/// A reader wrapper that updates MD5/SHA1/SHA256 with every byte read, -/// so hashing piggybacks on the streaming pattern-match pass. -pub struct TeeHasher { - inner: R, - md5: Md5, - sha1: Sha1, - sha256: Sha256, - len: u64, -} - -impl TeeHasher { - pub fn new(inner: R) -> Self { - Self { - inner, - md5: Md5::new(), - sha1: Sha1::new(), - sha256: Sha256::new(), - len: 0, - } - } - - /// Total bytes hashed so far (the file size once the stream is consumed). - pub fn bytes_read(&self) -> u64 { - self.len - } - - /// Finalize and return hex digests. Consumes the hasher. - pub fn finalize(self) -> Digests { - Digests { - md5: encode_hex(&self.md5.finalize()), - sha1: encode_hex(&self.sha1.finalize()), - sha256: encode_hex(&self.sha256.finalize()), - } - } -} - -impl Read for TeeHasher { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let n = self.inner.read(buf)?; - if n > 0 { - self.md5.update(&buf[..n]); - self.sha1.update(&buf[..n]); - self.sha256.update(&buf[..n]); - self.len += n as u64; - } - Ok(n) - } -} - /// Digests for section-hash matching: always MD5; SHA1/SHA256 only when /// `want_sha` (i.e. `.msb` signatures are loaded), since they are otherwise /// unused. Skipped fields are left empty and their lookups simply miss. @@ -440,30 +424,71 @@ pub fn section_digests(data: &[u8], want_sha: bool) -> Digests { /// Compute digests over a byte slice (used for already-buffered content /// such as extracted archive entries). pub fn digests_of(data: &[u8]) -> Digests { - // Single pass: feed each cache-resident chunk to all three hashers at once, - // rather than three independent full passes over `data`. For large buffers - // (e.g. a big archive member) three passes evict the buffer from cache - // between each, tripling memory traffic; chunking keeps each ~16 KiB slice - // hot in L1/L2 across md5+sha1+sha256. Output is byte-identical. - let mut md5 = Md5::new(); - let mut sha1 = Sha1::new(); - let mut sha256 = Sha256::new(); - for chunk in data.chunks(16 * 1024) { - md5.update(chunk); - sha1.update(chunk); - sha256.update(chunk); - } - Digests { - md5: encode_hex(&md5.finalize()), - sha1: encode_hex(&sha1.finalize()), - sha256: encode_hex(&sha256.finalize()), + digests_of_source(&data) +} + +/// [`digests_of`] an object that need not be held in memory. +pub(crate) fn digests_of_source(data: &dyn crate::byte_source::ByteSource) -> Digests { + let all = Want { + md5: true, + sha1: true, + sha256: true, + }; + digests_wanted(data, all) +} + +/// The digests of `data` that `want` asks for; the others are left empty, and +/// a lookup of them misses. Wanting none reads nothing. +pub(crate) fn digests_wanted(data: &dyn crate::byte_source::ByteSource, want: Want) -> Digests { + let mut h = Hashing::new(want); + if want != Want::default() { + data.chunks(0, data.len(), &mut |_, piece| { + h.update(piece); + true + }); + } + h.finish() +} + +/// The digests [`digests_wanted`] computes, fed an object's bytes in order by +/// a read that serves other searches too. +pub(crate) struct Hashing { + md5: Option, + sha1: Option, + sha256: Option, +} + +impl Hashing { + pub(crate) fn new(want: Want) -> Self { + Hashing { + md5: want.md5.then(Md5::new), + sha1: want.sha1.then(Sha1::new), + sha256: want.sha256.then(Sha256::new), + } + } + + pub(crate) fn update(&mut self, piece: &[u8]) { + // Every hasher over a 16 KiB slice before the next, which stays in + // L1/L2 across them. + for chunk in piece.chunks(16 * 1024) { + self.md5.iter_mut().for_each(|h| h.update(chunk)); + self.sha1.iter_mut().for_each(|h| h.update(chunk)); + self.sha256.iter_mut().for_each(|h| h.update(chunk)); + } + } + + pub(crate) fn finish(self) -> Digests { + Digests { + md5: self.md5.map_or_else(String::new, |h| encode_hex(&h.finalize())), + sha1: self.sha1.map_or_else(String::new, |h| encode_hex(&h.finalize())), + sha256: self.sha256.map_or_else(String::new, |h| encode_hex(&h.finalize())), + } } } #[cfg(test)] mod tests { use super::*; - use std::io::Cursor; #[test] fn known_md5() { @@ -474,18 +499,6 @@ mod tests { assert!(d.sha256.starts_with("e3b0c44298fc1c14")); } - #[test] - fn tee_matches_direct() { - let data = b"the quick brown fox"; - let mut tee = TeeHasher::new(Cursor::new(data.to_vec())); - let mut sink = Vec::new(); - tee.read_to_end(&mut sink).unwrap(); - let teed = tee.finalize(); - let direct = digests_of(data); - assert_eq!(teed.md5, direct.md5); - assert_eq!(teed.sha256, direct.sha256); - } - #[test] fn hashdb_lookup() { let mut db = HashDb::new(); diff --git a/crates/exav-core/src/jsnorm.rs b/crates/exav-core/src/jsnorm.rs index 90f913af..88a7d9ed 100644 --- a/crates/exav-core/src/jsnorm.rs +++ b/crates/exav-core/src/jsnorm.rs @@ -44,6 +44,8 @@ use std::borrow::Cow; use std::collections::HashMap; +use crate::byte_source::{ByteSource, Bytes, Stepper}; + /// Hard ceilings so a hostile script can't blow up time or memory. const MAX_OUTPUT: usize = 32 * 1024 * 1024; /// `eval` layers unrolled, counted from the script itself. Each layer re-reads @@ -77,14 +79,22 @@ enum Tok { /// Normalise a JavaScript/script buffer to its canonical form for matching. /// The flag is true when the output was cut at [`MAX_OUTPUT`]. pub fn normalize(data: &[u8]) -> (Vec, bool) { - run(data).finish() + run(Text::Mem(Cow::Borrowed(data))).finish() +} + +/// [`normalize`] over a script that need not be held in memory. +pub(crate) fn normalize_source(src: &dyn ByteSource) -> (Vec, bool) { + match src.as_slice() { + Some(data) => normalize(data), + None => run(Text::Source(Stepper::new(src))).finish(), + } } /// Read all of `data` through the normaliser, leaving the end unwritten. -fn run(data: &[u8]) -> Normalizer { +fn run(data: Text) -> Normalizer { let mut n = Normalizer::default(); // The script, and above it the text of each `eval` being unrolled. - let mut sources = vec![Lexer::new(Cow::Borrowed(data), 0)]; + let mut sources = vec![Lexer::new(data, 0)]; while let Some(src) = sources.last_mut() { if n.out.cut { break; @@ -124,7 +134,7 @@ fn is_ident_start(b: u8) -> bool { /// whitespace. A `/` is a regex when it appears where a value is expected, /// otherwise a comment (`//`, `/*`) or the division operator. struct Lexer<'a> { - data: Cow<'a, [u8]>, + data: Text<'a>, i: usize, /// "Value expected" position: at the start, or right after an operator / /// opening bracket / keyword: a `/` here begins a regex, not division. @@ -133,8 +143,42 @@ struct Lexer<'a> { depth: u32, } +/// Text a lexer reads: the script, held in memory or read from its source, +/// or the decoded text of an `eval`. +enum Text<'a> { + Mem(Cow<'a, [u8]>), + Source(Stepper<'a>), +} + +impl Bytes for Text<'_> { + fn len(&self) -> usize { + match self { + Text::Mem(d) => d.len(), + Text::Source(s) => s.len(), + } + } + + #[inline] + fn at(&mut self, i: usize) -> u8 { + match self { + Text::Mem(d) => d[i], + Text::Source(s) => s.at(i), + } + } + + fn range(&mut self, from: usize, to: usize) -> Cow<'_, [u8]> { + match self { + Text::Mem(d) => { + let to = to.min(d.len()); + Cow::Borrowed(&d[from.min(to)..to]) + } + Text::Source(s) => s.range(from, to), + } + } +} + impl<'a> Lexer<'a> { - fn new(data: Cow<'a, [u8]>, depth: u32) -> Self { + fn new(data: Text<'a>, depth: u32) -> Self { Lexer { data, i: 0, @@ -144,24 +188,24 @@ impl<'a> Lexer<'a> { } fn next(&mut self) -> Option { - let data: &[u8] = &self.data; + let data = &mut self.data; let n = data.len(); let mut i = self.i; while i < n { - let b = data[i]; + let b = data.at(i); match b { b' ' | b'\t' | b'\r' | b'\n' | 0x0c | 0x0b => { i += 1; } - b'/' if i + 1 < n && data[i + 1] == b'/' => { + b'/' if i + 1 < n && data.at(i + 1) == b'/' => { i += 2; - while i < n && data[i] != b'\n' { + while i < n && data.at(i) != b'\n' { i += 1; } } - b'/' if i + 1 < n && data[i + 1] == b'*' => { + b'/' if i + 1 < n && data.at(i + 1) == b'*' => { i += 2; - while i + 1 < n && !(data[i] == b'*' && data[i + 1] == b'/') { + while i + 1 < n && !(data.at(i) == b'*' && data.at(i + 1) == b'/') { i += 1; } i = (i + 2).min(n); @@ -181,41 +225,43 @@ impl<'a> Lexer<'a> { b'0'..=b'9' => { let start = i; i += 1; - while i < n - && (data[i].is_ascii_alphanumeric() - || data[i] == b'.' - || data[i] == b'_' - || ((data[i] == b'+' || data[i] == b'-') - && matches!(data[i - 1], b'e' | b'E'))) - { + while i < n && { + let c = data.at(i); + c.is_ascii_alphanumeric() + || c == b'.' + || c == b'_' + || ((c == b'+' || c == b'-') && matches!(data.at(i - 1), b'e' | b'E')) + } { i += 1; } self.i = i; self.value_pos = false; - return Some(Tok::Num(data[start..i].to_vec())); + return Some(Tok::Num(data.range(start, i).into_owned())); } - b'.' if i + 1 < n && data[i + 1].is_ascii_digit() => { + b'.' if i + 1 < n && data.at(i + 1).is_ascii_digit() => { let start = i; i += 1; - while i < n - && (data[i].is_ascii_digit() || matches!(data[i], b'e' | b'E' | b'+' | b'-')) - { + while i < n && { + let c = data.at(i); + c.is_ascii_digit() || matches!(c, b'e' | b'E' | b'+' | b'-') + } { i += 1; } self.i = i; self.value_pos = false; - return Some(Tok::Num(data[start..i].to_vec())); + return Some(Tok::Num(data.range(start, i).into_owned())); } _ if is_ident_start(b) => { let start = i; i += 1; - while i < n && is_ident(data[i]) { + while i < n && is_ident(data.at(i)) { i += 1; } self.i = i; + let ident = data.range(start, i).into_owned(); // After an identifier that is a keyword, a value follows. - self.value_pos = ident_is_keyword(&data[start..i]); - return Some(Tok::Ident(data[start..i].to_vec())); + self.value_pos = ident_is_keyword(&ident); + return Some(Tok::Ident(ident)); } _ => { self.i = i + 1; @@ -233,12 +279,12 @@ impl<'a> Lexer<'a> { /// Read a string literal starting at the opening quote `data[i] == quote`. /// Returns the decoded content (without quotes) and the index past the closer. -fn read_string(data: &[u8], mut i: usize, quote: u8) -> (Vec, usize) { +fn read_string(data: &mut B, mut i: usize, quote: u8) -> (Vec, usize) { let n = data.len(); let mut s = Vec::new(); i += 1; // skip opening quote while i < n { - let b = data[i]; + let b = data.at(i); if b == quote { i += 1; break; @@ -261,38 +307,38 @@ fn read_string(data: &[u8], mut i: usize, quote: u8) -> (Vec, usize) { /// Decode one backslash escape whose body starts at `i` (the char after `\`). /// Returns the decoded bytes and the index past the escape. -fn decode_escape(data: &[u8], i: usize) -> (Vec, usize) { +fn decode_escape(data: &mut B, i: usize) -> (Vec, usize) { let n = data.len(); if i >= n { return (vec![b'\\'], i); } - match data[i] { + match data.at(i) { b'x' if i + 2 < n => { - if let Some(v) = hex2(data[i + 1], data[i + 2]) { + if let Some(v) = hex2(data.at(i + 1), data.at(i + 2)) { return (vec![v], i + 3); } - (vec![data[i]], i + 1) + (vec![b'x'], i + 1) } b'u' => { - if i + 1 < n && data[i + 1] == b'{' { + if i + 1 < n && data.at(i + 1) == b'{' { // \u{HHHHHH} let mut j = i + 2; let mut cp: u32 = 0; let mut any = false; - while j < n && data[j] != b'}' { - let Some(d) = (data[j] as char).to_digit(16) else { break }; + while j < n && data.at(j) != b'}' { + let Some(d) = (data.at(j) as char).to_digit(16) else { break }; cp = cp.saturating_mul(16).saturating_add(d); any = true; j += 1; } - if any && j < n && data[j] == b'}' { + if any && j < n && data.at(j) == b'}' { return (encode_cp(cp), j + 1); } (vec![b'u'], i + 1) } else if i + 4 < n { let mut cp: u32 = 0; for k in 1..=4 { - let Some(d) = (data[i + k] as char).to_digit(16) else { + let Some(d) = (data.at(i + k) as char).to_digit(16) else { return (vec![b'u'], i + 1); }; cp = cp * 16 + d; @@ -306,8 +352,8 @@ fn decode_escape(data: &[u8], i: usize) -> (Vec, usize) { // Octal escape, up to 3 digits. let mut j = i; let mut v: u32 = 0; - while j < n && j < i + 3 && (b'0'..=b'7').contains(&data[j]) { - v = v * 8 + (data[j] - b'0') as u32; + while j < n && j < i + 3 && (b'0'..=b'7').contains(&data.at(j)) { + v = v * 8 + (data.at(j) - b'0') as u32; j += 1; } (vec![(v & 0xFF) as u8], j) @@ -319,7 +365,7 @@ fn decode_escape(data: &[u8], i: usize) -> (Vec, usize) { b'f' => (vec![0x0c], i + 1), b'v' => (vec![0x0b], i + 1), b'\n' => (vec![], i + 1), // line continuation - b'\r' if i + 1 < n && data[i + 1] == b'\n' => (vec![], i + 2), + b'\r' if i + 1 < n && data.at(i + 1) == b'\n' => (vec![], i + 2), b'\r' => (vec![], i + 1), other => (vec![other], i + 1), // \" \\ \/ \' → literal char } @@ -348,13 +394,13 @@ fn hex2(a: u8, b: u8) -> Option { } /// Read a regex literal `/.../flags` starting at `data[i] == '/'`. -fn read_regex(data: &[u8], mut i: usize) -> (Vec, usize) { +fn read_regex(data: &mut B, mut i: usize) -> (Vec, usize) { let n = data.len(); let start = i; i += 1; // opening / let mut in_class = false; while i < n { - match data[i] { + match data.at(i) { b'\\' if i + 1 < n => i += 2, b'[' => { in_class = true; @@ -372,10 +418,10 @@ fn read_regex(data: &[u8], mut i: usize) -> (Vec, usize) { _ => i += 1, } } - while i < n && data[i].is_ascii_alphabetic() { + while i < n && data.at(i).is_ascii_alphabetic() { i += 1; // flags } - (data[start..i].to_vec(), i) + (data.range(start, i).into_owned(), i) } // --------------------------------------------------------------------------- @@ -599,7 +645,7 @@ impl Normalizer { }; if matches!(kind, Kind::Eval) { self.held.truncate(call.start); - return Some(Lexer::new(Cow::Owned(text), call.depth + 1)); + return Some(Lexer::new(Text::Mem(Cow::Owned(text)), call.depth + 1)); } percent_decode(&text) } @@ -989,11 +1035,11 @@ mod tests { for k in 0..200_000 { script.extend_from_slice(format!("a{k}=b.c(d,e)+\"f\";").as_bytes()); } - let n = run(&script); + let n = run(Text::Mem(Cow::Borrowed(&script))); assert!(n.peak_held < 2 * FLUSH_BATCH, "held {} tokens", n.peak_held); // Calls nested deeper than the cap: the outermost are given up. let deep = [b"unescape(".repeat(100_000), b"'%41'".to_vec(), b")".repeat(100_000)].concat(); - let n = run(&deep); + let n = run(Text::Mem(Cow::Borrowed(&deep))); assert!(n.peak_held < 4 * MAX_OPEN_CALLS + FLUSH_BATCH, "held {} tokens", n.peak_held); } @@ -1116,4 +1162,32 @@ mod tests { let _ = normalize(bad); // must not panic } } + + #[test] + fn a_script_read_in_blocks_normalizes_as_in_memory() { + use crate::byte_source::{BlockCache, CHUNK}; + let pieces: &[&[u8]] = &[ + b"eval(", b"unescape(", b"'%61%6c'", b")", b";", b" ", b"\n", b"var x=", b"1e+5", b".5e3", + b"/*c*/", b"// line\n", b"/re[/]x/g", b"\"a\\x41\\u0042\\u{43}\\101\"", b"String.fromCharCode(", + b"72,105", b"`t\\`q`", b"a/b", b"(", b"]", b"\\\r\n", b"abc$_\x80", + ]; + let mut state = 0x6A09_E667_F3BC_C908u64; + for round in 0..300 { + let count = if round < 290 { 20 } else { 3 * CHUNK / 4 }; + let mut data = Vec::new(); + for _ in 0..count { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + data.extend_from_slice(pieces[(state % pieces.len() as u64) as usize]); + } + let cache = BlockCache::with_sizes(std::io::Cursor::new(data.clone()), 7, 56).unwrap(); + assert_eq!( + normalize_source(&cache), + normalize(&data), + "{:?}", + String::from_utf8_lossy(&data[..data.len().min(80)]) + ); + } + } } diff --git a/crates/exav-core/src/lib.rs b/crates/exav-core/src/lib.rs index a3b6513a..04cebd61 100644 --- a/crates/exav-core/src/lib.rs +++ b/crates/exav-core/src/lib.rs @@ -1,11 +1,14 @@ //! Streaming, size-unbounded file scanner. //! //! Every input goes through one scan, [`scan_seekable`] ([`scan_path`] opens a -//! file and hands it over). Structural analysis (archive extraction, PE -//! parsing, ML/fuzzy) needs the whole object in memory and so runs for -//! objects within [`ScanOptions::deep_analysis_max`]; a larger one gets the -//! constant-memory literal and whole-file-hash pass and is reported -//! `LimitsExceeded` unless that pass finds something. +//! file and hands it over), and every object in it, the input and whatever is +//! unpacked from it, goes through the same pipeline whatever its size: an +//! object within [`ScanOptions::deep_analysis_max`] is held in memory, a larger +//! one is read through a block cache of bounded size (or, for a decoded member, +//! a spill file), and a container's members are decoded as they are read. The +//! checks that parse an object whole (PE structure, a container format read +//! whole) do not run on one over that limit, and a scan that finds nothing is +//! then reported `LimitsExceeded`. //! //! # Anti-evasion invariants //! @@ -15,7 +18,7 @@ //! are therefore security properties rather than ergonomics: //! //! 1. **A detection always beats a limit.** If a signature matches, the verdict -//! is [`Verdict::Infected`] — never downgraded to `LimitsExceeded`/`Clean` +//! is [`Verdict::Infected`], never downgraded to `LimitsExceeded`/`Clean` //! because some *other* part of the input tripped a budget. Limits bound //! work; they never suppress a hit already found. //! 2. **Never refuse by size without scanning.** An input over @@ -25,9 +28,9 @@ //! we fall back to a limit verdict. (Otherwise `cat malware huge.pad > evil` //! is a one-line bypass.) //! 3. **Not-fully-scanned is never `Clean`.** Anything we couldn't fully examine -//! — a size/ratio/depth/scan-byte limit ([`Verdict::LimitsExceeded`]) or a -//! recognised-but-undecodable container ([`Verdict::Unscannable`], e.g. an -//! unsupported codec or encryption) — yields a distinct non-clean verdict. +//! (a size/ratio/depth/scan-byte limit, [`Verdict::LimitsExceeded`], or a +//! recognised-but-undecodable container, [`Verdict::Unscannable`], e.g. an +//! unsupported codec or encryption) yields a distinct non-clean verdict. //! Callers must treat both as suspicious, never as a pass. //! //! Sizes and offsets are 64-bit throughout. @@ -40,7 +43,7 @@ // ---- Stable public API surface ------------------------------------------- // These modules are covered by semver. `filetype` is public because it is // returned by `Scanner::identify`. -/// Authenticode (PE code-signing) triage without RSA — recompute the PE hash and +/// Authenticode (PE code-signing) triage without RSA: recompute the PE hash and /// compare it to the signature's embedded digest, and extract signer-cert fields. pub mod authenticode; pub mod database; @@ -54,6 +57,7 @@ pub mod loader; pub mod phishing; pub mod profile; pub mod source; +pub mod spill; /// Temp files for the test suite, so testing needs no temp-file dependency. #[cfg(test)] mod tmpfile; @@ -73,7 +77,7 @@ macro_rules! engine_internals { pub mod $m; // Without the feature these modules are crate-private. Many of their // `pub` items exist for the unstable-internals surface / examples and - // so are unused within the lean build — don't warn on that (they are + // so are unused within the lean build; don't warn on that (they are // exercised again once the feature re-exposes them). Scoped to the // engine internals only, so real dead code elsewhere still warns. #[cfg(not(feature = "unstable-internals"))] @@ -87,32 +91,36 @@ engine_internals!( normalize, patterns, pe, ); +mod byte_source; +mod stream_regex; + // YARA rule support. The native engine (compiler/scanner/modules) lives in-tree // under `src/yara/`. This module is `pub` (not an `engine_internals!` member) so // that a consumer who wants YARA alone can reach the compiler and scanner -// without going through a scan. It is ALWAYS compiled — the `yara::YaraDb` type -// is part of the -// on-disk database format regardless of the `yara` feature — while the actual -// compiler/matcher submodules inside it are gated on `feature = "yara"`. +// without going through a scan. It is ALWAYS compiled: the `yara::YaraDb` type +// is part of the on-disk database format regardless of the `yara` feature, +// while the actual compiler/matcher submodules inside it are gated on `feature = "yara"`. pub mod yara; use std::fs::File; use std::io::{self, Read, Seek, SeekFrom}; use std::path::Path; +use byte_source::ByteSource; use filetype::FileType; use fuzzy::FuzzyDb; -use hashes::{digests_of, HashDb, SectionHashDb, TeeHasher}; +#[cfg(test)] +use hashes::digests_of; +use hashes::{HashDb, SectionHashDb}; use ml::Model; -use patterns::PatternSet; use unpack::Budget; /// Smallest object a whole-file hash signature (`.hdb`/`.hsb`) may be matched /// against. /// -/// ClamAV refuses to scan any object of 5 bytes or fewer at all — `fmap->len <= +/// ClamAV refuses to scan any object of 5 bytes or fewer at all (`fmap->len <= /// 5` in `cli_magic_scan` and its four sibling entry points (libclamav -/// `scanners.c`, checked against 1.4.3) — so every hash signature it ships with +/// `scanners.c`, checked against 1.4.3)), so every hash signature it ships with /// a smaller declared size is unreachable in ClamAV, standalone file or nested /// layer alike. Six such signatures exist in a current main+daily set. /// @@ -136,12 +144,12 @@ use unpack::Budget; /// itself can never fire, so nothing can depend on it. const MIN_HASH_MATCH_BYTES: u64 = 6; -/// What a scan reports when a per-buffer bound — the step pool, or the -/// repeated-anchor cap — ended the signature search before it finished. +/// What a scan reports when a per-buffer bound (the step pool, or the +/// repeated-anchor cap) ended the signature search before it finished. /// /// Shared by the single-verdict and all-match paths so the two cannot drift /// apart in what they call the same condition. -const SEARCH_INCOMPLETE: &str = "scan budget exhausted — signature search incomplete"; +const SEARCH_INCOMPLETE: &str = "scan budget exhausted, signature search incomplete"; /// Whether a whole-file hash signature may be matched against an object of /// this size. See [`MIN_HASH_MATCH_BYTES`]. @@ -214,10 +222,177 @@ fn unpack_format(ft: FileType) -> Option { }) } +/// What the scan of one object works out about all of it, once however many +/// steps ask. Each costs a read of the whole object, which for one not held in +/// memory is a read of its whole source: for such an object, the first step to +/// ask makes one read for all of them. +pub(crate) struct Facts<'a> { + data: &'a dyn ByteSource, + /// What [`unpack::detect`] makes of it: for an object with no magic at its + /// start, detection searches all of it. + fmt: std::cell::OnceCell>, + /// Its whole-object digests, for the allowlist and the hash signatures. + digests: std::cell::OnceCell, + /// What carving finds in it. + embedded: std::cell::RefCell>, + /// For an object not held in memory, what the one read is for, until it + /// is made. + shared: std::cell::Cell>, + /// What that read found for detection, until detection asks. + prescan: std::cell::RefCell>, +} + +/// What the one read of an object not held in memory computes: the digests +/// `want` names, detection's search, and carving's when `carve`. +#[derive(Clone, Copy)] +struct Shared { + want: hashes::Want, + carve: bool, +} + +impl<'a> Facts<'a> { + /// The facts of `data`, each worked out by a read of its own when asked. + pub(crate) fn new(data: &'a dyn ByteSource) -> Self { + Facts { + data, + fmt: std::cell::OnceCell::new(), + digests: std::cell::OnceCell::new(), + embedded: std::cell::RefCell::new(None), + shared: std::cell::Cell::new(None), + prescan: std::cell::RefCell::new(None), + } + } + + /// The facts of `data` scanned by `db`, carved when `carve`: when `data` + /// is not held in memory, all worked out in one read of it. + fn for_scan(data: &'a dyn ByteSource, db: &Scanner, carve: bool) -> Self { + let facts = Facts::new(data); + if data.as_slice().is_none() { + facts.shared.set(Some(Shared { + want: digests_wanted_by(db, data.len() as u64), + carve, + })); + } + facts + } + + pub(crate) fn format(&self) -> Option { + *self.fmt.get_or_init(|| { + if self.shared.get().is_none() && self.prescan.borrow().is_none() { + return unpack::detect(self.data); + } + unpack::detect_prescanned(self.data, &|| { + self.read_once(); + // None only when the source gave nothing to read. + self.prescan + .take() + .unwrap_or_else(|| unpack::Prescan::new(b"")) + }) + }) + } + + /// The digests the allowlist and the hash signatures of `db` can match at + /// this object's size, all computed in one read of it, and none that no + /// signature there could use. + fn digests(&self, db: &Scanner) -> &hashes::Digests { + // Wanting none reads nothing: no reason to make the read yet. + if self + .shared + .get() + .is_some_and(|s| s.want != hashes::Want::default()) + { + self.read_once(); + } + self.digests.get_or_init(|| { + hashes::digests_wanted(self.data, digests_wanted_by(db, self.data.len() as u64)) + }) + } + + /// What carving finds in the object, taken: carving runs once. + fn embedded(&self) -> pe::Embedded { + self.read_once(); + self.embedded + .take() + .unwrap_or_else(|| pe::embedded_in(self.data)) + } + + /// The one read of an object not held in memory, the first time a step + /// asks, if it was not made yet. + fn read_once(&self) { + let Some(Shared { want, carve }) = self.shared.take() else { + return; + }; + let search = self.fmt.get().is_none() && unpack::Prescan::needed(self.data.len()); + if want == hashes::Want::default() && !carve && !search { + return; + } + let data = self.data; + let len = data.len(); + let mut hashing = (want != hashes::Want::default()).then(|| hashes::Hashing::new(want)); + let mut carving = carve.then(pe::Carving::new); + let mut prescan = None; + let overlap = unpack::Prescan::OVERLAP.max(pe::Carving::OVERLAP); + let mut at = 0; + while at < len { + let w = data.window(at, (len - at).min(byte_source::CHUNK)); + if w.is_empty() { + break; + } + if at == 0 && search { + prescan = Some(unpack::Prescan::new(&w)); + } + let searching = prescan.as_ref().is_some_and(|p| !p.done()) + || carving.as_ref().is_some_and(|c| !c.full()); + if hashing.is_none() && !searching { + break; + } + let last = at + w.len() >= len; + let owned = if last { + w.len() + } else { + w.len().saturating_sub(overlap).max(1) + }; + if let Some(h) = &mut hashing { + h.update(&w[..owned]); + } + if let Some(c) = &mut carving { + c.feed(data, at, &w, owned); + } + if let Some(p) = &mut prescan { + p.feed(at, &w, last); + } + at += owned; + } + if let Some(h) = hashing { + let _ = self.digests.set(h.finish()); + } + if let Some(c) = carving { + *self.embedded.borrow_mut() = Some(c.found); + } + *self.prescan.borrow_mut() = prescan; + } +} + +/// The digests the allowlist and the hash signatures of `db` can match on an +/// object of `size` bytes. +fn digests_wanted_by(db: &Scanner, size: u64) -> hashes::Want { + let mut want = db.allow.wants(size); + if hash_matchable(size) { + want = want.union(db.hashes.wants(size)); + } + want +} + /// Like [`unpack_format`] but content-aware: also recognises a UPX-packed /// executable (which is classified as a PE/ELF/Mach-O, not a container) so its /// embedded original is decompressed and scanned. -fn unpack_target(ft: FileType, data: &[u8], restrict: bool) -> Option { +fn unpack_target( + ft: FileType, + facts: &Facts, + restrict: bool, + opts: &ScanOptions, +) -> Option { + let data = facts.data; if let Some(fmt) = unpack_format(ft) { // When restricted to ClamAV's extractor set, skip the formats stock // ClamAV lacks so a differential run doesn't count exav's extra reach as @@ -230,14 +405,15 @@ fn unpack_target(ft: FileType, data: &[u8], restrict: bool) -> Option Option Option Option Option { +fn container_cltype(fmt: unpack::Format, data: &dyn ByteSource) -> Option { use engine::ClType; use unpack::Format; Some(match fmt { @@ -312,7 +495,7 @@ fn container_cltype(fmt: unpack::Format, data: &[u8]) -> Option // parts carry `CL_TYPE_MHTML` rather than `CL_TYPE_MAIL`. The two are // mutually exclusive. Format::Email => { - if filetype::looks_like_mhtml(data) { + if filetype::looks_like_mhtml(&data.window(0, 8192)) { ClType::Mhtml } else { ClType::Mail @@ -338,7 +521,7 @@ fn container_cltype(fmt: unpack::Format, data: &[u8]) -> Option Format::Rtf => ClType::Rtf, // A Windows executable acting as a container: an SFX stub with an // archive appended, a runtime-packed image, or an installer. All are - // `CL_TYPE_MSEXE` to the signature format — the members' parent is the + // `CL_TYPE_MSEXE` to the signature format: the members' parent is the // executable, whatever wrapped them inside it. Format::Sfx | Format::PePacked | Format::Upx | Format::Inno => ClType::MsExe, _ => return None, @@ -348,8 +531,8 @@ fn container_cltype(fmt: unpack::Format, data: &[u8]) -> Option /// The container type of a markup document that carries embedded base64 assets, /// or `None` if this buffer is not one. /// -/// The two flat-XML Office types are single-file documents — no ZIP, so nothing -/// an unpacker would open — identified by the processing instruction Office +/// The two flat-XML Office types are single-file documents (no ZIP, so nothing +/// an unpacker would open), identified by the processing instruction Office /// writes at the top. They are NOT the zipped `.docx`/`.xlsx`, which carry their /// own `CL_TYPE_OOXML_*` types. #[cfg(feature = "base64scan")] @@ -377,7 +560,7 @@ fn markup_cltype(ft: FileType, data: &[u8]) -> Option { /// name; `None` means the caller keeps whatever container it already had. /// /// Its one caller sits behind `base64scan`, so this is dead in a build without -/// it — a configuration that decodes nothing out of a carrier in the first place. +/// it: a configuration that decodes nothing out of a carrier in the first place. #[cfg_attr(not(feature = "base64scan"), allow(dead_code))] fn carrier_cltype(ft: FileType) -> Option { use engine::ClType; @@ -389,12 +572,6 @@ fn carrier_cltype(ft: FileType) -> Option { }) } -/// Bytes read from the front of a ZIP to detect its OOXML sub-type: the OOXML -/// part names (`[Content_Types].xml`, `word/document.xml`, …) sit in the leading -/// local file headers, so a bounded prefix reliably classifies real Office docs -/// without buffering a large archive. -const OOXML_DETECT_PREFIX: usize = 1 << 20; - /// Detect whether a ZIP is an Office Open XML document and which kind, by the /// member names present in its central directory. A `.docx`/`.xlsx`/`.pptx` is /// the `CL_TYPE_OOXML_*` container type (not plain `CL_TYPE_ZIP`) in the @@ -402,7 +579,7 @@ const OOXML_DETECT_PREFIX: usize = 1 << 20; /// way keeps such sigs firing on real Office documents while staying off plain /// ZIPs. The marker is the conventional top-level part for each kind plus the /// OOXML `[Content_Types].xml` package descriptor. -fn ooxml_subtype(data: &[u8]) -> Option { +fn ooxml_subtype(data: &dyn ByteSource) -> Option { use engine::ClType; // Cheap necessary condition: every OOXML package carries this part. if !contains_window(data, b"[Content_Types].xml") { @@ -421,8 +598,8 @@ fn ooxml_subtype(data: &[u8]) -> Option { /// Substring search over raw bytes (the OOXML member names appear verbatim in /// the ZIP central-directory file-name fields, which are stored uncompressed). -fn contains_window(haystack: &[u8], needle: &[u8]) -> bool { - memchr::memmem::find(haystack, needle).is_some() +fn contains_window(haystack: &dyn ByteSource, needle: &[u8]) -> bool { + haystack.find(needle, 0, haystack.len()).is_some() } /// Produce the final reported signature name from a matcher's CLEAN name plus @@ -487,7 +664,7 @@ pub enum Verdict { reason: String, }, /// A container/member was recognised but could not be decoded for a - /// NON-limit reason — an unsupported compression method (e.g. RAR PPMd). + /// NON-limit reason: an unsupported compression method (e.g. RAR PPMd). /// Distinct from `LimitsExceeded` (not a resource issue) and from `Clean` /// (we know there is content we couldn't examine). The `reason` names what /// was skipped. @@ -496,7 +673,7 @@ pub enum Verdict { }, /// A member is encrypted: we recognised it but can't read its content /// without a password. Distinct from `Unscannable` because it is - /// *actionable* — a caller can prompt for a password and re-scan with + /// *actionable*: a caller can prompt for a password and re-scan with /// [`ScanOptions::passwords`] set. Takes precedence over `Unscannable` when /// both occur (it's the one the user can do something about). PasswordProtected { @@ -506,8 +683,8 @@ pub enum Verdict { /// Coarse classification of a [`Verdict`], the single source of truth for /// summary counters and exit codes across every front-end (one-shot CLI, the -/// clamd daemon, and the daemon client). Keep counting/exit logic keyed on this -/// — never on the rendered string — so the three surfaces can't drift apart. +/// clamd daemon, and the daemon client). Keep counting/exit logic keyed on this, +/// never on the rendered string, so the three surfaces can't drift apart. /// Deliberately NOT `#[non_exhaustive]`, unlike [`Verdict`]. This enum exists /// so that exit codes and output shape are decided in one place, and the value /// of that is the compiler refusing to build until every consumer has handled a @@ -524,7 +701,7 @@ pub enum VerdictCategory { /// Work happened and stopped short of the end: a budget ran out, a /// container would not decode, or the content is encrypted. Reported /// `PARTIAL` under one of the three categories - /// ([`Verdict::status_tag`]), exit `3` — never a silent pass. + /// ([`Verdict::status_tag`]), exit `3`: never a silent pass. /// /// Distinct from an *error*, which is exav failing to do its job at all /// (an unreadable path, a database that would not load) and exits `2`. @@ -596,7 +773,7 @@ pub(crate) type PhishingPartsOwned = (Vec, Vec<(String, String)>, Vec<(S /// Full result of a scan. /// -/// New fields may appear in any release. Build with [`ScanReport::new`] — +/// New fields may appear in any release. Build with [`ScanReport::new`], /// never with a struct literal, which a new field would break. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[non_exhaustive] @@ -628,17 +805,6 @@ impl ScanReport { } } - /// Build an infected report from a core hit (clean name + provenance), - /// applying the `.UNOFFICIAL` suffix at report time when `suffix` is set. - fn infected_hit(hit: CoreHit, suffix: bool, findings: Vec) -> Self { - let (sig, offset, method, unofficial) = hit; - Self::infected( - report_name(&sig, unofficial, suffix), - offset, - method, - findings, - ) - } fn limits(reason: String, findings: Vec) -> Self { Self { verdict: Verdict::LimitsExceeded { @@ -685,7 +851,7 @@ fn one_line(reason: String) -> String { /// Options controlling a scan and its limits. /// /// New fields may appear in any release. Build with -/// `ScanOptions::default()` and assign the fields you care about — never with +/// `ScanOptions::default()` and assign the fields you care about, never with /// a struct literal, which a new field would break. #[derive(Clone)] #[non_exhaustive] @@ -693,10 +859,12 @@ pub struct ScanOptions { /// Max bytes for a single top-level file. `None` = unlimited (default). /// Exceeding yields `LimitsExceeded`. pub max_scan_size: Option, - /// Max object size eligible for in-memory structural/deep analysis - /// (archive extraction, PE parse, ML/fuzzy). Larger objects are still - /// stream-scanned (pattern+hash); the skipped structural step is - /// reported as a finding. Default 256 MiB. + /// Largest object held in memory (`--max-object-bytes`). A larger one gets + /// the same scan, read through a block cache or, for a decoded member, a + /// spill file; the checks that parse an object whole do not run on it, and + /// if nothing is found it is reported `LimitsExceeded`. A decoded member + /// over this limit with no spill to go to is not scanned, and reported. + /// A scan also holds `limits.max_buffer_bytes` to it. Default 256 MiB. pub deep_analysis_max: u64, /// Enable exav's *exclusive* structural heuristics: TLSH fuzzy matching, the /// static suspicion scorer (`Heuristics.Static.Suspect.*`), and the packed-with-injection @@ -706,7 +874,7 @@ pub struct ScanOptions { pub heuristics: bool, /// The heuristics stock ClamAV runs *by default* and that carry exact /// ClamAV-compatible names: `Heuristics.PDF.ObfuscatedNameObject` and imphash - /// (`.imp`) matching. **On by default** — these are FP-safe (imphash is an + /// (`.imp`) matching. **On by default**: these are FP-safe (imphash is an /// exact DB signature; the PDF check counts only gratuitous escapes), so exav /// out of the box does every reasonable ClamAV-default match. Kept separate /// from [`heuristics`](Self::heuristics) so this parity subset stays on under @@ -717,7 +885,7 @@ pub struct ScanOptions { /// the same fact differently. Set by [`ScanOptions::clamav_compat`]. /// /// This changes NAMES, never what is detected. exav does not withhold a - /// finding in either mode, and does not adopt a check it believes is wrong — + /// finding in either mode, and does not adopt a check it believes is wrong: /// compat is about feature scope and the strings a drop-in replacement must /// emit, not about reproducing another engine's judgement. /// @@ -730,10 +898,10 @@ pub struct ScanOptions { pub limits: unpack::Limits, /// Restrict exav's unpacking reach to stock ClamAV's, so a differential run /// against `clamscan` doesn't count exav's extra reach as a disagreement. - /// Narrows two things to ClamAV's scope: (1) archive extractors — skip the + /// Narrows two things to ClamAV's scope: (1) archive extractors, skipping the /// formats stock ClamAV lacks: `ar` (Unix archive / `.deb` / `.a`), `lzip`, /// Inno Setup, and the magic-dispatched disk-image / Unix `compress` formats - /// (verified against 1.4.x, which handles cpio/xar natively); (2) UPX — to PE + /// (verified against 1.4.x, which handles cpio/xar natively); (2) UPX: to PE /// only (ClamAV's UPX unpacker runs only from its PE path, never ELF/Mach-O). /// /// The other PE runtime packers stay **on**: ClamAV unpacks those too, and @@ -741,11 +909,11 @@ pub struct ScanOptions { /// it miss a packed dropper. Compat narrows *scope and naming*, never /// coverage of content that is there to be found. Default off. This /// deliberately *reduces* exav's detection capability for reproducibility, so - /// it is a diff-testing aid, **not for production** — it is one of the + /// it is a diff-testing aid, **not for production**. It is one of the /// behaviours the CLI's `--clamav-compat` turns on. pub restrict_extractors: bool, /// Decode long base64 blobs found in text/script buffers and rescan any that - /// decode to a real executable (PE/ELF/Mach-O/OLE) — catching a PE stashed as + /// decode to a real executable (PE/ELF/Mach-O/OLE), catching a PE stashed as /// a base64 string in a PowerShell/JS/VBS dropper or an RTF body, invisible to /// a signature that matches the decoded bytes. **On by default** (exav-exclusive /// reach beyond stock ClamAV); off under `--clamav-compat` and via `--no-base64`. @@ -753,14 +921,14 @@ pub struct ScanOptions { pub decode_base64: bool, /// Append `.UNOFFICIAL` to signature names that come from unofficial /// databases, matching stock `clamscan`'s output. Purely - /// cosmetic — it changes only how a detection is *named*, never whether it + /// cosmetic: it changes only how a detection is *named*, never whether it /// fires. Default off. (The other behaviour `--clamav-compat` turns on.) pub unofficial_suffix: bool, /// Password(s) to try when decrypting encrypted archive members. Empty by /// default. When a scan returns [`Verdict::PasswordProtected`], a caller can /// set this and re-scan. pub passwords: Vec, - /// Verify archive checksums (CRCs) during extraction. **Off by default** — a + /// Verify archive checksums (CRCs) during extraction. **Off by default**: a /// scanner scans decompressed content regardless of integrity metadata, so a /// corrupted checksum can't hide a payload from detection (this matches /// ClamAV, which ignores CRCs when scanning). Only has effect when exav-core @@ -810,12 +978,12 @@ pub struct ScanOptions { /// `Heuristics.Broken.Media.*` for a structurally invalid image/media file /// (GIF, PNG, TIFF and JPEG container validation). Off by default. pub alert_broken_media: bool, - /// Opt-in heuristic (`--alert-packed`): report `Heuristics.Packed.*` for an + /// Opt-in heuristic (`exav --detect packed`): report `Heuristics.Packed.*` for an /// executable behind a packer or protector exav cannot unpack. /// /// Reported IN ADDITION TO the unscannable signal, never instead of it. The - /// two say different things — "this is VMProtect" and "its original code was - /// not recovered" — and a scanner that emits only the second leaves every + /// two say different things ("this is VMProtect" and "its original code was + /// not recovered"), and a scanner that emits only the second leaves every /// commercially-protected binary on the ERROR line, where gateways read it as /// scanner failure rather than as a fact about the file. The heuristic also /// stays correct once a given packer becomes unpackable: the file was still @@ -829,7 +997,7 @@ pub struct ScanOptions { pub alert_phishing: bool, /// Opt-in Authenticode heuristic: report `Heuristics.Authenticode.HashMismatch` /// when a code-signed PE's embedded digest does NOT cover the current file - /// bytes (i.e. the file was modified or had data appended after signing — a + /// bytes (i.e. the file was modified or had data appended after signing, a /// classic trojanized-signed-binary trick). Triage only: exav does not verify /// the RSA signature (see `authenticode`). Off by default. pub alert_broken_authenticode: bool, @@ -841,6 +1009,11 @@ pub struct ScanOptions { /// undefined (and such rules do not match). It never affects any non-YARA /// detection. pub filename: Option, + /// Where a scan may write what it makes of an object too large to hold in + /// memory (its normalised text, a member too large to buffer), to read it + /// back at any offset. `None`: nowhere, and what needed it is reported as + /// not fully scanned. See [`spill`]. + pub spill: Option>, } impl Default for ScanOptions { @@ -850,7 +1023,7 @@ impl Default for ScanOptions { deep_analysis_max: 256 * 1024 * 1024, heuristics: false, // ClamAV-default matchings (PDF obfuscation, imphash) are on out of the - // box — FP-safe and part of a faithful default scan. The exav-exclusive + // box: FP-safe and part of a faithful default scan. The exav-exclusive // TLSH/ML heuristics above stay opt-in. clamav_heuristics: true, clamav_compat: false, @@ -872,6 +1045,7 @@ impl Default for ScanOptions { alert_phishing: false, alert_broken_authenticode: false, filename: None, + spill: None, } } } @@ -903,7 +1077,7 @@ impl ScanOptions { decode_base64: false, unofficial_suffix: true, passwords: Vec::new(), - // ClamAV ignores CRCs when scanning — match it. + // ClamAV ignores CRCs when scanning; match it. verify_checksums: false, structured_cc_count: None, structured_ssn_count: None, @@ -917,6 +1091,7 @@ impl ScanOptions { alert_phishing: false, alert_broken_authenticode: false, filename: None, + spill: None, } } } @@ -928,10 +1103,7 @@ impl ScanOptions { /// are crate-private (not part of the stable API); their layout may change /// between releases. pub struct Scanner { - /// Literal patterns (EICAR + literal `.ndb`), also used on the streaming - /// path where wildcard verification isn't possible. - pub(crate) patterns: PatternSet, - /// Full `.ndb`/`.ldb` matcher (wildcards, logical sigs); in-memory only. + /// The `.ndb`/`.ldb`/`.db` signature matcher. pub(crate) engine: engine::SigEngine, pub(crate) hashes: HashDb, /// `.mdb`/`.mdu`/`.msb` PE section-hash signatures. @@ -961,10 +1133,10 @@ pub struct Scanner { pub(crate) crb: authenticode::CrbDb, /// Passwords loaded from `.pwdb` files (ClamAV password database). Tried /// (unioned with [`ScanOptions::passwords`]) when decrypting encrypted - /// archive members. The official CVDs ship none — this is user-supplied. + /// archive members. The official CVDs ship none; this is user-supplied. pub(crate) passwords: Vec, /// Version number and build-time of the newest loaded `.cvd`/`.cld` container - /// (the one with the highest version — the daily set in a normal ClamAV DB + /// (the one with the highest version: the daily set in a normal ClamAV DB /// dir), for the clamd-compatible daemon `VERSION`/`VERSIONCOMMANDS` replies. /// `None` when only loose signatures or the built-in baseline were loaded. pub(crate) db_version: Option<(u32, String)>, @@ -984,7 +1156,7 @@ impl Scanner { } /// Assemble a [`Scanner`] from individually-built subsystems over the - /// [`Scanner::builtin`] baseline (which supplies `patterns`, `sections`, + /// [`Scanner::builtin`] baseline (which supplies `sections`, /// `yara`, `model`, `ftm`, `bytecode`, …). For fuzzing / low-level tooling /// that constructs sub-databases directly; not part of the stable API. #[allow(clippy::too_many_arguments)] @@ -1019,12 +1191,9 @@ impl Scanner { /// Built-in baseline database (EICAR pattern, heuristic model). pub fn builtin() -> Self { - // EICAR goes in the engine (used by in-memory scans) and in the - // streaming PatternSet (used by stdin/pipe scans). let mut eb = engine::EngineBuilder::new(); eb.add_literal("Exav.Test.EICAR", patterns::eicar()); Self { - patterns: PatternSet::builtin(), engine: eb.build(), hashes: HashDb::new(), sections: SectionHashDb::new(), @@ -1058,28 +1227,44 @@ impl Scanner { /// loaded `.ftm` magic rules only when content detection is inconclusive /// (`Unknown`), so native typing is never overridden. pub fn identify(&self, data: &[u8]) -> FileType { - profile::timed("filetype", data.len() as u64, || self.identify_inner(data)) + self.identify_source(&data) + } + + /// As [`Self::identify`], over an object that need not be held in memory. + pub(crate) fn identify_source(&self, data: &dyn ByteSource) -> FileType { + self.identify_facts(&Facts::new(data)) + } + + /// As [`Self::identify_source`], sharing what detection finds with the + /// other steps that ask `facts`. + pub(crate) fn identify_facts(&self, facts: &Facts) -> FileType { + profile::timed("filetype", facts.data.len() as u64, || { + self.identify_inner(facts) + }) } - fn identify_inner(&self, data: &[u8]) -> FileType { - let mut ft = filetype::identify(data); + fn identify_inner(&self, facts: &Facts) -> FileType { + let data = facts.data; + let mut ft = filetype::identify_source_with(data, || facts.format()); if ft == FileType::Unknown { - if let Some(f) = self.ftm.identify(data) { + if let Some(f) = self.ftm.identify_source(data) { ft = f; } } - // The bzip2 (`BZh`), CAB (`MSCF`) and gzip (`1f 8b`) file-type magics are - // short and collide with ordinary binary data; a false hit (e.g. a `BZh4…`, - // `MSCF…` or `1f8b08…` byte-run inside an ISO member or PE overlay) would - // be routed to that decoder, fail deep in parsing, and report the whole - // object UNSCANNABLE / LIMITS-EXCEEDED. `.ftm` rules match on those weak - // prefixes, so confirm each against the extractor's stronger magic check - // (block magic for bzip2, zero `reserved1` for CAB, deflate CM + flag bits - // for gzip) before trusting the typing; a false hit is scanned as raw bytes. + // The bzip2 (`BZh`), CAB (`MSCF`), gzip (`1f 8b`) and ARJ (`60 ea`) + // file-type magics are short and collide with ordinary binary data; a + // false hit (e.g. a `BZh4…`, `MSCF…` or `1f8b08…` byte-run inside an ISO + // member or PE overlay) would be routed to that decoder, fail deep in + // parsing, and report the whole object UNSCANNABLE / LIMITS-EXCEEDED. + // `.ftm` rules match on those weak prefixes, so confirm each against the + // extractor's stronger magic check (block magic for bzip2, zero + // `reserved1` for CAB, deflate CM + flag bits for gzip, the header CRC + // for ARJ) before trusting the typing; a false hit is scanned as raw bytes. let false_archive = match ft { - FileType::Bzip2 => unpack::detect(data) != Some(unpack::Format::Bzip2), - FileType::Cab => unpack::detect(data) != Some(unpack::Format::Cab), - FileType::Gzip => unpack::detect(data) != Some(unpack::Format::Gzip), + FileType::Bzip2 => facts.format() != Some(unpack::Format::Bzip2), + FileType::Cab => facts.format() != Some(unpack::Format::Cab), + FileType::Gzip => facts.format() != Some(unpack::Format::Gzip), + FileType::Arj => facts.format() != Some(unpack::Format::Arj), _ => false, }; if false_archive { @@ -1088,8 +1273,7 @@ impl Scanner { ft } - /// Distinct signatures loaded. The streaming literal set is not counted: it - /// is a copy of literals the engine also holds. + /// Distinct signatures loaded. pub fn signature_count(&self) -> usize { self.engine.signature_count() + self.hashes.len() @@ -1116,13 +1300,6 @@ impl Scanner { /// Source signatures that could not be loaded (e.g. unsupported `.ndb` /// wildcards, PCRE/bytecode subsignatures). pub fn unsupported_count(&self) -> usize { - // The ENGINE is the authoritative loader: a signature it compiled is - // loaded and will match, whatever the streaming set did with it. - // `patterns.unsupported` counts lines the streaming literal set could - // not carry — overwhelmingly wildcard bodies the engine handles fine — - // so adding it here both invented gaps that don't exist and - // double-counted the actually malformed lines (both loaders see the - // same `.ndb` text). Report the engine's count alone. self.engine.unsupported } } @@ -1134,13 +1311,13 @@ impl Scanner { /// /// # Errors /// -/// The `io::Error` covers only reaching the file — opening, reading, seeking. +/// The `io::Error` covers only reaching the file: opening, reading, seeking. /// **Nothing about the scan's outcome is reported this way.** A file that could /// not be decoded, that exhausted a budget, or that turned out to be encrypted /// is a successful call returning a [`ScanReport`] whose [`Verdict`] says so. /// -/// Treating `Err` as "not infected" is therefore safe, and treating `Ok` as -/// "clean" is not: check the verdict. +/// An `Err` means the file was not scanned: report it as an error, never as +/// clean. An `Ok` is not "clean" either until its verdict says so. /// /// # Panics /// @@ -1148,7 +1325,7 @@ impl Scanner { /// rather than unwinding. Two failure modes are outside that boundary and will /// take the process down: an allocation large enough to abort, and stack /// exhaustion from a deeply self-nested file. A caller that must survive -/// arbitrary input needs an out-of-process bound — see `SECURITY.md`. +/// arbitrary input needs an out-of-process bound; see `SECURITY.md`. pub fn scan_path(db: &Scanner, path: &Path, opts: &ScanOptions) -> io::Result { // Supply the scanned file's identity to the YARA external variables // (`filepath`/`filename`/`extension`) unless the caller already set one. @@ -1169,102 +1346,21 @@ pub fn scan_path(db: &Scanner, path: &Path, opts: &ScanOptions) -> io::Result ScanReport { - let reason = format!( - "{} is {size} bytes, over the {}-byte deep-analysis limit \ - (--max-object-bytes); only literal signatures and whole-file hashes \ - were matched", - ft.as_str(), - opts.deep_analysis_max - ); - report_of_outcome( - limits_outcome(opts, unpack::LimitKind::MaxFileSize, reason), - Vec::new(), - opts, - ) -} - -/// The report for a file that was under the deep-analysis limit when sized and -/// over it by the time it was read (it grew, or it is a special file). -fn grew_past_deep_analysis_limit(opts: &ScanOptions) -> ScanReport { - let reason = format!( - "input grew past the {}-byte deep-analysis limit (--max-object-bytes) \ - while it was read", - opts.deep_analysis_max - ); - report_of_outcome( - limits_outcome(opts, unpack::LimitKind::MaxFileSize, reason), - Vec::new(), - opts, - ) -} - -/// Read into `buf` until it is full or EOF, retrying short/interrupted reads. -/// Returns the number of bytes filled. Used so type identification never sees -/// a truncated prefix from a single short `read`. -fn fill_prefix(reader: &mut R, buf: &mut [u8]) -> io::Result { - let mut filled = 0; - while filled < buf.len() { - match reader.read(&mut buf[filled..]) { - Ok(0) => break, - Ok(n) => filled += n, - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, - Err(e) => return Err(e), - } - } - Ok(filled) -} - -/// The whole of a seekable source, or `None` when it is larger than `cap`. -/// Leaves the reader at the start either way. -fn read_bounded(reader: &mut R, cap: u64) -> io::Result>> { - let size = reader.seek(SeekFrom::End(0))?; - reader.seek(SeekFrom::Start(0))?; - if size > cap { - return Ok(None); - } - let mut buf = Vec::new(); - reader.by_ref().take(cap).read_to_end(&mut buf)?; - reader.seek(SeekFrom::Start(0))?; - Ok(Some(buf)) -} - -/// Whether `data`'s whole-file hash is on the `.fp`/`.sfp` allowlist. -fn allowlisted(db: &Scanner, data: &[u8]) -> bool { +/// Whether the object's whole-file hash is on the `.fp`/`.sfp` allowlist. +fn allowlisted(db: &Scanner, facts: &Facts) -> bool { !db.allow.is_empty() && db .allow - .lookup(&digests_of(data), data.len() as u64) + .lookup(facts.digests(db), facts.data.len() as u64) .is_some() } /// Build every structure a scan initialises lazily (engine automata, compiled -/// YARA rules, the constant-memory pass's automaton), so a process that forks -/// workers after loading shares them copy-on-write instead of each worker -/// building its own. +/// YARA rules), so a process that forks workers after loading shares them +/// copy-on-write instead of each worker building its own. pub fn warm_up(db: &Scanner) { const PROBE: &[u8] = b"MZ\x90\x00\x00\x00\x00\x00"; let _ = analyze(db, PROBE, &ScanOptions::default()); - let _ = stream_core(db, PROBE, false); -} - -/// The constant-memory pass alone, for the tests that pin it down. -#[cfg(test)] -pub(crate) fn scan_stream(db: &Scanner, reader: R) -> io::Result { - engine::reset_scan_truncated(); - if let Some(hit) = stream_core(db, reader, false)? { - return Ok(suppress_name( - db, - ScanReport::infected_hit(hit, false, Vec::new()), - )); - } - Ok(ScanReport::clean(Vec::new())) } thread_local! { @@ -1297,7 +1393,7 @@ impl Drop for MatchPathGuard { /// Record the current member path as the detection location, unless one is /// already recorded (first detection wins, matching first-match scan -/// semantics). A no-op at top level (empty path) — a top-level detection has no +/// semantics). A no-op at top level (empty path): a top-level detection has no /// member location. /// /// Must be called at EVERY site that returns a detection, including ones that @@ -1368,10 +1464,9 @@ pub fn scan_seekable_located( /// only hold the first `max_scan_size` bytes: those get the whole scan, and /// with no detection in them the input is reported over the limit. /// -/// A ZIP, tar or gzip is walked member by member off the reader, so only the -/// parts actually scanned are read and a range-backed reader never fetches the -/// rest. Anything else within `deep_analysis_max` is buffered and analyzed; a -/// larger one gets the constant-memory literal and whole-file-hash pass. +/// An input within `deep_analysis_max` is read into memory; a larger one is +/// read through a block cache of bounded size, by offset, as the scan needs it. +/// Either way it gets the same scan. pub fn scan_seekable( db: &Scanner, mut reader: R, @@ -1402,33 +1497,6 @@ fn scan_within( size: u64, opts: &ScanOptions, ) -> io::Result { - let mut prefix = [0u8; 4096]; - let n = fill_prefix(&mut reader, &mut prefix)?; - reader.seek(SeekFrom::Start(0))?; - let mut ft = profile::timed("filetype", n as u64, || filetype::identify(&prefix[..n])); - // An input that fits is typed again from all of it by `analyze`. - if size > opts.deep_analysis_max && !streams_natively(ft) { - ft = type_unbuffered(&mut reader, ft, opts)?; - } - - // Natively-streaming containers (ZIP/tar/gzip) are walked member-by-member - // off the reader: the whole container is never buffered and no - // `deep_analysis_max` cap applies to the walk, so an arbitrarily large - // archive is scanned in bounded memory (one member at a time). - if streams_natively(ft) { - // A container that fits under `deep_analysis_max` gets the full - // engine over its own bytes inside `scan_container_stream`. A larger - // one cannot be held, so it gets the constant-memory pass here, and - // the walk reports what it could not run. - if size > opts.deep_analysis_max { - if let Some(hit) = stream_core(db, &mut reader, opts.unofficial_suffix)? { - return Ok(core_hit_report(db, hit, opts)); - } - reader.seek(SeekFrom::Start(0))?; - } - return Ok(scan_container_stream(db, reader, opts, ft)); - } - if size <= opts.deep_analysis_max { // Bounded, so a source whose size under-reports its length cannot // stream unbounded. @@ -1436,62 +1504,19 @@ fn scan_within( (&mut reader) .take(opts.deep_analysis_max.saturating_add(1)) .read_to_end(&mut data)?; - if data.len() as u64 > opts.deep_analysis_max { - return Ok(grew_past_deep_analysis_limit(opts)); + if data.len() as u64 <= opts.deep_analysis_max { + return Ok(analyze(db, &data, opts)); } - return Ok(analyze(db, &data, opts)); - } - - // Too large to buffer. The constant-memory pass still runs, so a literal - // signature or a whole-file hash anywhere in the input is found. - if let Some(hit) = stream_core(db, &mut reader, opts.unofficial_suffix)? { - return Ok(core_hit_report(db, hit, opts)); - } - Ok(over_deep_analysis_limit(ft, size, opts)) -} - -/// Head an input too large to buffer is typed from: through sector 31, the last -/// a UDF recognition sequence is looked for in. An ISO 9660 descriptor sits -/// earlier, at 32 KiB. -const UNBUFFERED_TYPING_HEAD: usize = 32 * 2048 + 6; - -/// The type of an input too large to buffer, when more of it than the 4 KiB -/// `ft` was read from names a container walked off the reader: an ISO 9660 or -/// UDF image, or a self-extractor, wherever in the first `max_buffer_bytes` its -/// archive starts. Either would otherwise get the literal pass alone. -fn type_unbuffered( - reader: &mut R, - ft: FileType, - opts: &ScanOptions, -) -> io::Result { - let mut head = vec![0u8; UNBUFFERED_TYPING_HEAD]; - let n = fill_prefix(reader, &mut head)?; - reader.seek(SeekFrom::Start(0))?; - let head = &head[..n]; - let wide = filetype::identify(head); - if streams_natively(wide) { - return Ok(wide); - } - // In memory an executable is walked as an SFX when `detect` says so, which - // it does only after ruling out the installers it knows. One named in the - // head is left to that. - if wide.is_executable() && matches!(unpack::detect(head), None | Some(unpack::Format::Sfx)) { - let sfx = unpack::is_sfx(reader, opts.limits.max_buffer_bytes) - .map_err(|h| io::Error::other(h.reason))?; + // It grew while it was read: scanned as the larger object it now is. reader.seek(SeekFrom::Start(0))?; - if sfx { - return Ok(FileType::Sfx); - } } - Ok(ft) -} - -/// The report for a hit of the constant-memory pass. -fn core_hit_report(db: &Scanner, hit: CoreHit, opts: &ScanOptions) -> ScanReport { - suppress_name( - db, - ScanReport::infected_hit(hit, opts.unofficial_suffix, Vec::new()), - ) + let cache = byte_source::BlockCache::new(reader)?; + let report = analyze_source(db, &cache, opts); + // A read that failed part way leaves bytes the scan never saw. + if let Some(e) = cache.read_error() { + return Err(io::Error::other(e)); + } + Ok(report) } /// The first `len` bytes of a seekable source, as a seekable source of its own. @@ -1546,7 +1571,11 @@ fn scan_budget(db: &Scanner, opts: &ScanOptions) -> Budget { pool.push(pw.clone()); } } - let mut b = Budget::with_passwords(opts.limits.clone(), pool); + // Nothing is held in memory past the deep-analysis limit, a container its + // format reads whole included. + let mut limits = opts.limits.clone(); + limits.max_buffer_bytes = limits.max_buffer_bytes.min(opts.deep_analysis_max); + let mut b = Budget::with_passwords(limits, pool); b.set_verify_checksums(opts.verify_checksums); b } @@ -1576,7 +1605,7 @@ fn limits_alert_name(kind: unpack::LimitKind) -> Option<&'static str> { /// The report for a top-level file larger than `max_scan_size`. /// /// This is ClamAV's `MaxFileSize` condition, so under `--partial-as found` it -/// reports under ClamAV's own name for it — `Heuristics.Limits.Exceeded.*`, +/// reports under ClamAV's own name for it, `Heuristics.Limits.Exceeded.*`, /// which a ClamAV-shaped pipeline matches, rather than a synthesised /// `Heuristics.Exav.*` that nothing does. Both entry points that enforce the /// ceiling come through here, so the two cannot name one condition two ways, @@ -1624,674 +1653,284 @@ fn outcome_for_hit(hit: unpack::LimitHit, opts: &ScanOptions) -> DeepOutcome { } } -/// Container formats scanned member-by-member off the raw seekable source -/// **without holding the whole container in RAM**. Two mechanisms sit behind -/// this: every [`unpack::is_streamable`] format uses the low-level reader API -/// ([`unpack::stream_members`]), so a member is decoded on demand and a -/// multi-gigabyte member is never materialized (see [`member_stream_scan`]). -/// Either way the `deep_analysis_max` buffer does not cap the container size — -/// a multi-gigabyte `.tar`/`.tar.gz`/`.zip`/`.gz` on disk is scanned in bounded -/// memory. Every other container still buffers (its decoder needs random access -/// over a slice), so it stays on the size-capped `analyze` path. -fn streams_natively(ft: FileType) -> bool { - matches!( - ft, - FileType::Zip - | FileType::Tar - | FileType::Gzip - | FileType::Bzip2 - | FileType::Xz - | FileType::Zstd - | FileType::Lzip - | FileType::Lha - | FileType::Ar - | FileType::Cpio - | FileType::Machofat - | FileType::Pyc - | FileType::Sfx - | FileType::Tnef - | FileType::Partition - | FileType::Iso - | FileType::OneNote - | FileType::Swf - | FileType::Szdd - | FileType::SevenZip - | FileType::Cab - ) +/// A member's bytes, once read. +enum Body { + /// Nothing to scan: the member was not decoded, or could not be read. + None, + Mem(Vec), + /// Too large to hold, and written to the host's spill. + Spilled(spill::Spilled), } -/// The outcome of reading a streamed member into memory, capped. +/// Read a member decoded as it is read: into memory up to the deep-analysis +/// limit, else to the host's spill, the prefix already read first. What could +/// not be read is noted on the tally; `Err` means the walk must stop. /// -/// In every case the bytes already consumed come back: a stream does not -/// rewind, so discarding them would lose content that was decoded and then -/// never scanned. -enum Capped { - /// The whole member fitted under the cap. - Whole(Vec), - /// It did not fit. `.0` is the prefix consumed; chain it ahead of the - /// reader rather than dropping it. - TooBig(Vec), - /// The read itself failed — a disk error, a dropped connection, a range - /// request the server would not honour. `.0` is what arrived before the - /// failure. This is NOT truncation: the rest of the member exists and we - /// did not get it, so the caller must report rather than treat the prefix - /// as the member. - Failed(Vec, io::Error), -} - -/// Read at most `cap` bytes from a streamed member. -fn read_capped(rdr: &mut dyn Read, cap: u64) -> Capped { +/// A decode error keeps what was decoded before it (a truncated gzip or tar +/// must not hide the payload in its readable part), and `failed` says why the +/// rest is missing. +fn read_member( + cx: &MemberCtx<'_>, + tally: &mut MemberTally, + rdr: &mut dyn Read, + verify_checksums: bool, + failed: &mut Option, +) -> Result { + let cap = cx.opts.deep_analysis_max; + // The walk reports the budget that ran out once the member is left. + let over_budget = || DeepOutcome::Limits("member exceeds the scan budget".to_string()); let mut buf = Vec::new(); // One byte past the cap: reading exactly `cap` cannot tell a member that - // just fits from one that is about to overflow. - if let Err(e) = rdr.take(cap.saturating_add(1)).read_to_end(&mut buf) { - return Capped::Failed(buf, e); + // just fits from one that does not. + let mut head = rdr.take(cap.saturating_add(1)); + match head.read_to_end(&mut buf) { + Err(e) if unpack::is_budget_overflow(&e) => return Err(over_budget()), + Err(e) if verify_checksums || buf.is_empty() => { + tally + .unscannable + .get_or_insert_with(|| format!("member decode error: {e}")); + return Ok(Body::None); + } + Err(e) => { + buf.truncate(cap as usize); + *failed = Some(e); + return Ok(Body::Mem(buf)); + } + Ok(_) if buf.len() as u64 <= cap => return Ok(Body::Mem(buf)), + Ok(_) => {} } - if buf.len() as u64 > cap { - Capped::TooBig(buf) - } else { - Capped::Whole(buf) + let rest = head.into_inner(); + let not_scanned = |tally: &mut MemberTally, why: &str| { + let reason = format!( + "a member is over the {cap}-byte deep-analysis limit \ + (--max-object-bytes) and could not be spilled to disk ({why}): \ + it was not scanned" + ); + match limits_outcome(cx.opts, unpack::LimitKind::MaxFileSize, reason) { + DeepOutcome::Limits(r) => { + tally.limits.get_or_insert(r); + Ok(Body::None) + } + alert => Err(alert), + } + }; + let Some(spill) = &cx.opts.spill else { + return not_scanned(tally, "spilling to disk is off"); + }; + match spill::spill_stream(spill.as_ref(), &mut io::Cursor::new(buf).chain(rest)) { + Ok(spilled) => Ok(Body::Spilled(spilled)), + Err(spill::SpillStreamError::Read(e)) if unpack::is_budget_overflow(&e) => { + Err(over_budget()) + } + Err(spill::SpillStreamError::Read(e)) => { + tally + .unscannable + .get_or_insert_with(|| format!("member stream error: {e}")); + Ok(Body::None) + } + Err(spill::SpillStreamError::Spill(e)) => not_scanned(tally, &e), } } -/// Scan one member's *bytes* when they arrive as a **reader** rather than a -/// slice — the streaming counterpart of [`member_content_scan`], reporting to -/// the same sink and tally so the two produce identical results. -/// -/// A bounded prefix — up to `deep_analysis_max` — is buffered here in -/// `exav-core` (the highest layer that can decide) so slice-based structural -/// analysis runs on members that fit; those hand straight to -/// [`member_content_scan`]. A member larger than the cap is *not* materialized: -/// its buffered prefix is chained with the still-streaming tail and pattern+hash -/// scanned end to end (so a signature anywhere in a 2 GiB member is found), -/// while structural/ML analysis is skipped — reported, never a silent Clean. -/// -/// The over-cap branch can report only one detection: finding a second would -/// mean re-reading a stream that has already been consumed. That is the one -/// place `--all-matches` is bounded by how the member arrived, and it is the -/// bound the cap itself imposes, not a difference between the walks. -fn member_stream_scan( +/// One member of a container walk: its metadata, then its bytes, scanned as an +/// object one level down. `Some` means the walk must stop. +#[allow(clippy::too_many_arguments)] +fn visit_member( cx: &MemberCtx<'_>, tally: &mut MemberTally, - reader: &mut dyn Read, + volumes: &mut unpack::volume::Collector, + container: &dyn ByteSource, + meta: &unpack::MemberMeta, + content: Option>, budget: &mut Budget, findings: &mut Vec, sink: &mut Sink, ) -> Option { - let cap = cx.opts.deep_analysis_max; - let mut buf = Vec::new(); - let mut head = reader.take(cap.saturating_add(1)); - if let Err(e) = head.read_to_end(&mut buf) { - if unpack::is_budget_overflow(&e) { - return Some(DeepOutcome::Limits( - "member exceeds per-member decompression budget".to_string(), - )); - } - // A decode error still leaves the bytes decoded *before* the error in - // `buf` — `Read::read_to_end` appends them. Scan that salvaged prefix so - // malware in the recoverable part is caught (a truncated gzip/tar must - // not hide its payload — observed on real samples). - if !budget.should_verify_checksums() && !buf.is_empty() { - buf.truncate(cap as usize); - let scanned = member_content_scan(cx, tally, &buf, budget, findings, sink); - // We scanned every byte the stream yielded. If it simply ran out of - // input (truncation: the missing tail is *absent*, not hidden) or - // decoded in full and failed its checksum, a clean result is a real - // Clean: exav scans for malware, it is not a file-integrity - // validator. A mid-stream *corruption* (undecodable bytes still - // present) keeps the not-fully-scanned verdict on a clean salvage. - if scanned.is_none() && unpack::decode_error_hides_content(&e) { - tally.unscannable.get_or_insert_with(|| { - format!( - "member decode error (salvaged {} B, no match): {e}", - buf.len() - ) - }); + let mut failed = None; + let body = match content { + None => Body::None, + Some(unpack::Member::Bytes(data)) => Body::Mem(data), + Some(unpack::Member::Stream(rdr)) => { + let verify = budget.should_verify_checksums(); + match read_member(cx, tally, rdr, verify, &mut failed) { + Ok(body) => body, + Err(stop) => return Some(stop), } - return scanned; } - tally - .unscannable - .get_or_insert_with(|| format!("member decode error: {e}")); - return None; + }; + // A member byte-identical to its container is a **fixed point**: typing it + // re-detects the same format, which yields the same member, forever. It is + // never a real member (nothing was unwrapped), and following it burns the + // whole recursion budget on one buffer, so the content that actually needed + // those levels never gets reached. Skipping loses nothing: these exact + // bytes are already being scanned, as the container. + // + // A FAT boot sector read as an MBR produces one, and a gzip can be built + // to decompress to itself, so the guard lives here rather than in any + // extractor. The length test is false for essentially every real member, + // so the comparison is only reached by a genuine fixed point. + if let Body::Mem(data) = &body { + if data.len() == container.len() && *container.window(0, data.len()) == data[..] { + return None; + } } - if buf.len() as u64 <= cap { - // Whole member in hand → full structural analysis on the bounded buffer. - return member_content_scan(cx, tally, &buf, budget, findings, sink); + let size_real = match &body { + Body::Mem(data) => data.len() as u64, + Body::Spilled(spilled) => spilled.len() as u64, + Body::None => meta.comp_size, + }; + if let Some(o) = member_metadata_scan(cx, tally, meta, size_real, sink) { + return Some(o); } - // Member exceeds the structural-buffer cap: stream the whole thing through - // the constant-memory pattern+hash core (prefix + tail chained so a match - // straddling the boundary is still caught). No full-member buffer is held. - let rest = head.into_inner(); - match stream_core( - cx.db, - std::io::Cursor::new(&buf).chain(rest), - cx.opts.unofficial_suffix, - ) { - Ok(Some((sig, off, method, unofficial))) => sink.hit( - report_name(&sig, unofficial, cx.opts.unofficial_suffix), - off, - method, - ), - Ok(None) => { - let reason = format!( - "a member is over the {cap}-byte deep-analysis limit \ - (--max-object-bytes); only literal signatures and hashes were \ - matched in it" - ); - match limits_outcome(cx.opts, unpack::LimitKind::MaxFileSize, reason) { - DeepOutcome::Limits(r) => { - tally.limits.get_or_insert(r); - None + match body { + Body::None => None, + // Nothing decoded: the metadata above is all this member has. + Body::Mem(data) if data.is_empty() && meta.unsupported.is_some() => None, + Body::Mem(data) => { + if let Some(e) = failed { + // Every byte that decoded was scanned. A member that ran out + // of input has its tail absent, not hidden, and a clean scan + // of it is a real Clean: exav scans for malware, it is not a + // file-integrity validator. Undecodable bytes still present + // keep the not-fully-scanned verdict. + let o = member_content_scan(cx, tally, &data, budget, findings, sink); + if o.is_none() && unpack::decode_error_hides_content(&e) { + tally.unscannable.get_or_insert_with(|| { + format!( + "member decode error (salvaged {} B, no match): {e}", + data.len() + ) + }); } - alert => Some(alert), + return o; } + // A part of a byte-split set is a fragment of a file that only + // exists once the set is rejoined, so it is held rather than + // scanned on its own. A part cut short above is not offered: it + // would splice a hole into the rejoined archive. + let data = match volumes.offer(&meta.name, data) { + unpack::volume::Offer::Held => return None, + unpack::volume::Offer::PassThrough { data, .. } => data, + }; + member_content_scan(cx, tally, &data, budget, findings, sink) } - Err(ref e) if unpack::is_budget_overflow(e) => Some(DeepOutcome::Limits( - "member exceeds per-member decompression budget".to_string(), - )), - Err(e) => { - tally - .unscannable - .get_or_insert_with(|| format!("member stream error: {e}")); - None + Body::Spilled(spilled) => { + let o = member_content_scan(cx, tally, &spilled, budget, findings, sink); + if let Some(e) = spilled.read_error() { + tally + .unscannable + .get_or_insert_with(|| format!("spilled member unreadable: {e}")); + } + o } } } -/// Scan a natively-streaming container ([`streams_natively`]) member-by-member -/// off a seekable source, stopping on the first detection. The container is -/// never fully buffered. Every [`unpack::is_streamable`] format goes through the -/// low-level reader API so a member of any size is decoded on demand (see -/// [`member_stream_scan`]); the rest use the seekable [`unpack::Archive`] walk, -/// which materializes one member at a time. Incomplete-scan signals (encrypted / -/// undecodable members) are accumulated and only surfaced after the whole -/// container is walked, so an early bad member can't mask a malicious sibling — -/// and "not fully scanned is never Clean" holds. -fn scan_container_stream( +/// Walk a container's members, each scanned as an object one level down. +/// +/// What does not stop the walk (a member encrypted, undecodable, or cut short +/// by a limit) is kept on a tally and decides the verdict once every member has +/// been seen, so an early bad member cannot mask a malicious sibling, and "not +/// fully scanned is never Clean" holds. +#[allow(clippy::too_many_arguments)] +fn walk_members( db: &Scanner, - reader: R, - opts: &ScanOptions, + data: &dyn ByteSource, + obj: Obj<'_>, ft: FileType, -) -> ScanReport { - let mut findings = vec![Finding::new("type", ft.as_str())]; - // Every type that reaches this walk came from `filetype_of_format`, so the - // inverse exists — `scan_dispatch_covers_every_format` asserts the - // round-trip over `Format::ALL`. Defaulting to some format anyway would - // parse, say, a RAR as a ZIP: the walk finds nothing, the file reports - // clean, and the mistake looks exactly like an empty archive. If the two - // mappings ever drift, say so instead. - let Some(fmt) = unpack_format(ft) else { - return ScanReport::unscannable( - format!("no extractor is mapped for {}", ft.as_str()), - findings, - ); - }; - let mut reader = reader; - - // The container's own bytes get the full matching core (wildcard and - // logical signatures, type-scoped ones, YARA, bytecode, hashes), as they - // do when the same container is a member of another one - // (`member_content_scan`). Without it a top-level archive was matched by - // literal patterns at best, and a signature on the archive itself fired - // only once the archive was nested. Held up to `deep_analysis_max`, the - // bound every in-memory pass has; past it the walk reports what it skipped. - let whole = match read_bounded(&mut reader, opts.deep_analysis_max) { - Ok(w) => w, - Err(e) => return ScanReport::unscannable(format!("container unreadable: {e}"), findings), - }; - if whole.as_deref().is_some_and(|w| allowlisted(db, w)) { - return ScanReport::clean(findings); - } - let outcome = walk_container(db, reader, opts, fmt, ft, whole.as_deref(), &mut findings); - // The members first, so a hit in one is attributed to it rather than to the - // container that happens to hold its bytes stored. Anything short of a - // detection still gets this pass: a detection beats a limit. - let outcome = match outcome { - hit @ DeepOutcome::Infected { .. } => hit, - other => whole - .as_deref() - .and_then(|w| core_scan(db, w, None, None, opts, &mut Sink::First(&db.ignored))) - .unwrap_or(other), - }; - report_of_outcome(outcome, findings, opts) -} - -/// Walk a top-level container's members for [`scan_container_stream`], with the -/// container on the ancestry chain for exactly as long as its members are -/// being scanned. -fn walk_container( - db: &Scanner, - reader: R, - opts: &ScanOptions, fmt: unpack::Format, - ft: FileType, - whole: Option<&[u8]>, + opts: &ScanOptions, + budget: &mut Budget, findings: &mut Vec, + sink: &mut Sink, ) -> DeepOutcome { - let mut reader = reader; - let mut budget = scan_budget(db, opts); - - // Container `CL_TYPE_*` for `Container:`-scoped signatures, so each member - // inherits it. A ZIP is sub-typed as OOXML Word/Excel/PowerPoint by its part - // names; read a bounded prefix (the OOXML part names sit in the leading local - // file headers) and rewind, so streamed OOXML members are scanned with the - // right container context (e.g. `Container:CL_TYPE_OOXML_WORD` sigs fire). - let container = if let Some(w) = whole { - container_cltype(fmt, w) - } else if fmt == unpack::Format::Zip && db.engine.has_ooxml_container_sigs() { - // Seek to the start FIRST: an earlier raw pass (`stream_core`) may have - // consumed the reader to EOF, so reading without rewinding yields nothing. - // Gated on the DB actually having `Container:CL_TYPE_OOXML_*` sigs: the - // probe reads a leading prefix, which over a range-fetched source (HTTP) - // would otherwise pull megabytes for a distinction nothing depends on. - let _ = reader.seek(std::io::SeekFrom::Start(0)); - let mut prefix = vec![0u8; OOXML_DETECT_PREFIX]; - let n = fill_prefix(&mut reader, &mut prefix).unwrap_or(0); - let _ = reader.seek(std::io::SeekFrom::Start(0)); - container_cltype(fmt, &prefix[..n]) - } else { - container_cltype(fmt, &[]) - }; - - // Members are one layer below this container, so it joins their ancestry for - // as long as we are walking it (`Intermediates:`). Both walks below need it: - // a nested archive reached through the streaming path would otherwise show a - // chain one link short, and a signature keyed on `A>B` would not fire on the - // very nesting it describes. - let _ag = engine::AncestryGuard::enter(container); - if unpack::is_streamable(fmt) { - return scan_streamed_container( - db, - reader, + if obj.depth >= budget.limits().max_recursion { + return limits_outcome( opts, - fmt, - ft, - container, - findings, - &mut budget, - 0, - whole, - &mut Sink::First(&db.ignored), + unpack::LimitKind::MaxRecursion, + format!("recursion depth exceeds {}", budget.limits().max_recursion), ); } - - // Non-streamable Archive-walkable containers: seekable member walk. Members - // are still materialized one at a time by the Archive layer, but the container - // is never buffered. - // Container size for `.cdb` `ContainerSize` matching (seek to end, rewind). - // - // A failure here is reported rather than absorbed. This path is only taken - // for a source that seeks, so the error is the file becoming unreadable - // mid-scan, not a stream that never could. Substituting a size would keep - // the scan running against a number that is not the container's: a `.cdb` - // signature keyed on a size range would then quietly fail to match, and the - // file would be called clean because of an I/O error nobody saw. - let container_size = match reader - .seek(std::io::SeekFrom::End(0)) - .and_then(|n| reader.seek(std::io::SeekFrom::Start(0)).map(|_| n)) - { - Ok(n) => n, - Err(e) => return DeepOutcome::Unscannable(format!("container size unreadable: {e}")), - }; - let mut archive = match unpack::Archive::open(reader) { - Ok(a) => a, - Err(h) => return outcome_for_hit(h, opts), - }; - // The same context, tally and per-member checks the other walks use. A - // container reached this way is a large one that was never buffered, which - // is a statement about its size — not a reason for it to be scanned by - // different rules than the same archive would get at any other size. + // The type the members belong to, so signatures scoped with + // `Container:CL_TYPE_*` fire only inside their intended container. A ZIP + // is sub-typed as OOXML Word/Excel/PowerPoint by its part names. + let member_container = container_cltype(fmt, data); + // The members are one layer below this container, so it is on their + // ancestry (`Intermediates:`) for as long as they are scanned. + let _ag = engine::AncestryGuard::enter(member_container); let cx = MemberCtx { db, opts, - container_size, + container_size: data.len() as u64, container_is_ole: fmt == unpack::Format::Ole, - member_container: container, + member_container, ft, fmt, - depth: 0, + depth: obj.depth, }; let mut tally = MemberTally::new(); + // Parts of a byte-split set (`x.7z.001`, `.002`, …) are held here and + // rejoined once the walk has ended. let mut volumes = unpack::volume::Collector::new(budget.limits().max_buffer_bytes); - let sink = &mut Sink::First(&db.ignored); - let mut terminal: Option = None; - // Inclusive of the member scans, as in the other walks. - let walk = profile::timed("unpack", container_size, || { - loop { - let mut entry = match archive.extract_next(&mut budget) { - Ok(Some(e)) => e, - Ok(None) => break, - Err(h) => return Err(h), - }; - // Track this member on the location stack for the duration of its scan, - // so a detection inside it (or a nested member) reports the full path. - let _mpg = MatchPathGuard::enter(&entry.name); - let size_real = entry.data.len() as u64; - let facts = MemberFacts { - name: &entry.name, - comp_size: entry.comp_size, - size_real, - encrypted: entry.encrypted, - unsupported: entry.unsupported, - }; - if let Some(o) = member_metadata_scan(&cx, &mut tally, facts, sink) { - terminal = Some(o); - break; - } - // Nothing decoded: the metadata above is all this member has. A - // member that decoded part way is still scanned for what it has; - // its `unsupported` reason is already in the tally. - if entry.unsupported.is_some() && entry.data.is_empty() { - continue; - } - // A part of a byte-split set is held and rejoined below, not scanned as - // the fragment it is. - let body = std::mem::take(&mut entry.data); - let data = match volumes.offer(&entry.name, body) { - unpack::volume::Offer::Held => continue, - unpack::volume::Offer::PassThrough { data, .. } => data, - }; - if let Some(o) = - member_content_scan(&cx, &mut tally, &data, &mut budget, findings, sink) - { - terminal = Some(o); - break; - } - } - Ok(()) + // Inclusive wall time: this drives the member scans, so `unpack_us` + // contains the member matchers' time too, and `emu_us` splits the + // emulator's share of it out. + let walk = profile::timed("unpack", data.len() as u64, || { + unpack::walk(fmt, data, budget, &mut |meta, content, budget| { + // Track this member on the location stack for its scan, so a + // detection in it, or deeper, reports the full path. + let _mpg = MatchPathGuard::enter(&meta.name); + visit_member( + &cx, + &mut tally, + &mut volumes, + data, + meta, + content, + budget, + findings, + sink, + ) + }) }); - if let Err(h) = walk { - return outcome_for_hit(h, opts); - } - // Reassembly happens only now: completeness is knowable only once no further - // member can arrive. Every part held above is still scanned, rejoined or - // not — bytes withheld and then dropped are the silent clean this scanner - // exists to prevent. + // Reassembly happens only now. Nothing in a byte-split set's names says + // how many parts it has, so `.001`+`.002` looks contiguous even when + // `.003` follows: joining on arrival would emit a truncated prefix that + // still parses as the archive and would then be scanned as if whole. let held = volumes.finish(); + let mut terminal = match walk { + Ok(o) => o, + Err(hit) => Some(outcome_for_hit(hit, opts)), + }; + // Rejoined files first, then the parts that could not be joined. Bytes + // withheld from the scan and then dropped would be exactly the silent + // clean this scanner exists to prevent. for (name, buf, incomplete) in held.into_scannable() { if terminal.is_some() { break; } + // A set with a gap in it can no longer be read by anything, not by us + // and not by the tool that wrote it. if let Some(reason) = incomplete { tally.unscannable.get_or_insert_with(|| reason.to_string()); } - // Nothing else accounts for these bytes: the parts were charged as they - // were extracted, but this is a buffer the collector made. + // The parts were charged as they were read; this is a buffer the + // collector made. if let Err(h) = budget.charge_scan(buf.len() as u64) { - return outcome_for_hit(h, opts); + terminal = Some(limits_outcome(opts, h.kind, h.reason)); + break; } let _mpg = MatchPathGuard::enter(&name); - terminal = member_content_scan(&cx, &mut tally, &buf, &mut budget, findings, sink); + terminal = member_content_scan(&cx, &mut tally, &buf, budget, findings, sink); } terminal.unwrap_or_else(|| tally.verdict()) } -/// Drive the low-level reader API ([`unpack::stream_members`]): each member is a -/// reader decoded on demand, scanned via [`scan_stream_member`] with no -/// full-member buffer. The visitor returns `Some(())` to halt the walk on a -/// terminal outcome (detection / limit), leaving the report in `terminal`. -#[allow(clippy::too_many_arguments)] -fn scan_streamed_container( - db: &Scanner, - reader: R, - opts: &ScanOptions, - fmt: unpack::Format, - ft: FileType, - container: Option, - findings: &mut Vec, - budget: &mut Budget, - depth: u32, - // The container's bytes when the caller already holds them, so they are - // not read a second time for the whole-object checks below. - prefetched: Option<&[u8]>, - sink: &mut Sink, -) -> DeepOutcome { - // Container size for `.cdb` `ContainerSize` matching (seek to end, rewind). - // - // Reported rather than absorbed, on the same reasoning as the seekable walk: - // every caller here hands in a source that seeks, so an error is the file - // becoming unreadable mid-scan. A substituted size is a number that is not - // the container's, and a `.cdb` signature keyed on a size range would then - // quietly fail to match — the file called clean because of an I/O error - // nobody saw. - let mut reader = reader; - let container_size = match reader - .seek(std::io::SeekFrom::End(0)) - .and_then(|n| reader.seek(std::io::SeekFrom::Start(0)).map(|_| n)) - { - Ok(n) => n, - Err(e) => return DeepOutcome::Unscannable(format!("container size unreadable: {e}")), - }; - // Only the container-level checks below record here; everything a member - // produces goes on the tally. - let mut unscannable: Option = None; - let mut limits: Option = None; - let mut terminal: Option = None; - - // The checks that need the container's own bytes rather than a member's. - // This walk holds one member at a time, so getting them means reading the - // container again. - // - // That second read is deliberate. exav optimises for MEMORY, not bandwidth: - // the cap below is what bounds the working set, and a re-fetch over a - // range-reading HTTP source is accepted as the price of scanning the same - // way regardless of how the file arrived. A caller who would rather pay in - // memory than in transfer can download the object and hand exav the file. - // - // Over the cap the container is not held at all, and that is REPORTED — - // scanning less because of how a file arrived, without saying so, is the - // silent clean this scanner exists to prevent. - if container_size <= opts.deep_analysis_max { - let mut owned = Vec::new(); - let read = match prefetched { - Some(bytes) => Ok(bytes), - None => { - let r = reader - .by_ref() - .take(opts.deep_analysis_max) - .read_to_end(&mut owned); - let _ = reader.seek(std::io::SeekFrom::Start(0)); - r.map(|_| &owned[..]) - } - }; - match read { - Ok(whole) => { - if let Some(o) = whole_buffer_heuristics(whole, ft, opts, sink) { - return o; - } - } - // The source failed mid-read — a disk error, a dropped connection, a - // range request the server would not honour. The bytes exist and we - // did not get them, so the checks did not run: say so rather than - // carrying on as though they had. - Err(e) => { - unscannable.get_or_insert_with(|| { - format!("could not re-read the container for whole-object checks: {e}") - }); - } - } - } else { - let reason = format!( - "container is {container_size} bytes, over the {}-byte deep-analysis \ - limit (--max-object-bytes): its members were scanned, but the full \ - signature engine and whole-container checks did not run on its own bytes", - opts.deep_analysis_max - ); - match limits_outcome(opts, unpack::LimitKind::MaxFileSize, reason) { - DeepOutcome::Limits(r) => limits = Some(r), - alert => return alert, - } - } - - // The same context and running tally the buffered walk keeps, so the shared - // member checks behave identically here. - let cx = MemberCtx { - db, - opts, - container_size, - // OLE2 is not a streamable format, so a member reached this way is never - // an OLE stream needing the forced type. - container_is_ole: false, - member_container: container, - ft, - fmt, - // This container's own depth: members recurse at `depth + 1`, which is - // what bounds nesting. A constant here would restart the count and let a - // deeply nested chain of streamable containers run past the limit. - depth, - }; - let mut tally = MemberTally::new(); - // Parts of a byte-split set are held here and rejoined after the walk — see - // the note at `finish()` below for why the join cannot happen sooner. - let mut volumes = unpack::volume::Collector::new(budget.limits().max_buffer_bytes); - let walk = { - let findings = &mut *findings; - let terminal = &mut terminal; - let cx = &cx; - let tally = &mut tally; - let volumes = &mut volumes; - let mut visit = |meta: &unpack::MemberMeta, - rdr: Option<&mut dyn Read>, - budget: &mut Budget| - -> Option<()> { - // Track this member on the location stack for its scan (see - // [`MatchPathGuard`]) so a detection reports the member path. - let _mpg = MatchPathGuard::enter(&meta.name); - // `.cdb` container-metadata match on the member's name/size/pos/ - // encryption (fires even when the body can't be decoded — the streamed - // path is how ZIP/OOXML members reach this, so filename sigs like - // `Archive.Filetype.*` are matched here). - let facts = MemberFacts { - name: &meta.name, - comp_size: meta.comp_size, - // Nothing is decoded at this point, so the compressed size is - // the only size this walk can offer for both fields. - size_real: meta.comp_size, - encrypted: meta.encrypted, - unsupported: meta.unsupported, - }; - if let Some(o) = member_metadata_scan(cx, tally, facts, sink) { - *terminal = Some(o); - return Some(()); - } - if meta.unsupported.is_some() { - // Nothing decoded: the metadata above is all this member has. - return None; - } - let rdr = rdr?; - // A part of a byte-split set is a fragment of a file that only - // exists once the set is rejoined, so it is buffered and held - // instead of being scanned on its own. This is the only place this - // walk materializes a member — everything else still streams — and - // the hold is bounded by `max_buffer_bytes`. - let is_part = - unpack::volume::parse(&meta.name).is_some_and(|v| v.scheme.is_byte_split()); - let outcome = if is_part { - match read_capped(&mut *rdr, budget.limits().max_buffer_bytes) { - Capped::Whole(buf) => match volumes.offer(&meta.name, buf) { - unpack::volume::Offer::Held => return None, - // Past the collector's budget: it will not be joined, so - // scan it where it stands like any other member. - unpack::volume::Offer::PassThrough { data, .. } => { - member_content_scan(cx, tally, &data, budget, findings, sink) - } - }, - // Too large to hold. The prefix is already consumed and a - // stream does not rewind, so it is chained back in front of - // the tail — dropping it would skip the bytes. - Capped::TooBig(prefix) => { - let mut whole = std::io::Cursor::new(prefix).chain(rdr); - member_stream_scan(cx, tally, &mut whole, budget, findings, sink) - } - // The read failed. Scan what did arrive, then report — a - // short buffer must NOT be offered to the collector, which - // would treat it as a complete volume and splice a truncated - // part into the rejoined archive. - Capped::Failed(prefix, e) => { - let got = member_content_scan(cx, tally, &prefix, budget, findings, sink); - if got.is_none() { - tally.unscannable.get_or_insert_with(|| { - format!("member read failed after {} B: {e}", prefix.len()) - }); - } - got - } - } - } else { - member_stream_scan(cx, tally, rdr, budget, findings, sink) - }; - // `Some` means this walk must stop: a first-match detection, or a - // limit. Everything that does not stop the walk — a nested - // unscannable member, an encrypted one, an all-match hit — has - // already been recorded on the tally or the sink. - match outcome { - Some(o) => { - *terminal = Some(o); - Some(()) - } - None => None, - } - }; - profile::timed("unpack", container_size, || { - unpack::stream_members(fmt, reader, budget, &mut visit) - }) - }; - // Reassembly happens only now. Nothing in a byte-split set's names says how - // many parts it has, so `.001`+`.002` looks contiguous even when `.003` - // follows: joining on arrival would emit a truncated prefix that still - // parses as the archive and would then be scanned as if it were whole. - let held = volumes.finish(); - if let Err(h) = walk { - return outcome_for_hit(h, opts); - } - // Rejoined files first, then the parts that could not be joined. Bytes - // withheld from the scan and then dropped would be exactly the silent clean - // this scanner exists to prevent, so every part held above still gets - // scanned on its own here. - for (name, buf, incomplete) in held.into_scannable() { - if terminal.is_some() { - break; - } - // A set with a gap in it is reported: the archive those bytes belong to - // can no longer be read by anything — not by us and not by the tool that - // wrote it. - if let Some(reason) = incomplete { - tally.unscannable.get_or_insert_with(|| reason.to_string()); - } - // Nothing else accounts for these bytes: the parts were charged as they - // were read, but this is a buffer the collector made. - if let Err(h) = budget.charge_scan(buf.len() as u64) { - return outcome_for_hit(h, opts); - } - let _mpg = MatchPathGuard::enter(&name); - terminal = member_content_scan(&cx, &mut tally, &buf, budget, findings, sink); - } - if let Some(r) = terminal { - return r; - } - // Fold in what the shared member checks recorded. They keep their findings - // on the tally, and a member that is encrypted or undecodable is noted there - // rather than here — dropping it would turn a PASSWORD-PROTECTED archive - // into a clean one. - // - // A note made *about the container* — its whole-object checks could not run - // — outranks a member's, because it says something was not looked at at all - // rather than that one member could not be decoded. - if let Some(r) = unscannable { - tally.unscannable = Some(r); - } - if let Some(r) = limits { - tally.limits = Some(r); - } - tally.verdict() -} - /// The verdict of a multi-volume archive, reported against one of its parts. #[derive(Debug, Clone)] #[non_exhaustive] pub struct VolumeSetVerdict { - /// The part this verdict is attributed to — one of the names given. + /// The part this verdict is attributed to: one of the names given. pub name: String, /// The name of the archive the part belongs to (`big.7z` for `big.7z.001`). pub set: String, @@ -2299,7 +1938,7 @@ pub struct VolumeSetVerdict { } /// Scan the **multi-volume archives** spread across a group of files that -/// arrived together — a directory, a client's multi-file request. +/// arrived together: a directory, a client's multi-file request. /// /// A set like `big.7z.001`, `.002`, `.003` is one archive cut into pieces at /// arbitrary byte offsets. Handed to a scanner one file at a time, no piece @@ -2312,13 +1951,13 @@ pub struct VolumeSetVerdict { /// /// A part that cannot be read, or that the collector will not hold, takes its /// whole set with it. Byte-split naming records no part count, so a set missing -/// its LAST part still looks contiguous and would join into a truncated prefix — +/// its LAST part still looks contiguous and would join into a truncated prefix, /// which parses as the archive and scans clean. The set is reported /// `Unscannable` instead. The dropped part's own scan, which the caller does /// separately, still covers that part's bytes. /// /// Returns one entry per *part*, each carrying the verdict of the archive that -/// part belongs to — including `Unscannable` for a set with a hole in it, whose +/// part belongs to, including `Unscannable` for a set with a hole in it, whose /// bytes belong to an archive nothing can read. Callers report these against /// the part's own name: one result per file, and a piece of an infected archive /// is not a clean file. @@ -2326,7 +1965,7 @@ pub struct VolumeSetVerdict { /// Format-aware volumes (RAR `.partN`, ZIP `.zNN`) are **not** handled here. /// Each of those carries its own headers and a member's data resumes past the /// next volume's header, so concatenating them yields garbage that still looks -/// like an archive — the join has to be done by the format's own decoder. +/// like an archive: the join has to be done by the format's own decoder. pub fn analyze_volume_sets( db: &Scanner, names: &[String], @@ -2355,7 +1994,7 @@ where continue; }; any = true; - // `PassThrough` means the collector declined to hold it — the held-bytes + // `PassThrough` means the collector declined to hold it: the held-bytes // cap, or a second part claiming an index another part already holds. if let unpack::volume::Offer::PassThrough { name, .. } = collector.offer(name, data) { broken.insert( @@ -2390,7 +2029,7 @@ where } } for u in held.unjoined { - // A lone numbered file is not a set — plenty of ordinary files end in + // A lone numbered file is not a set: plenty of ordinary files end in // `.001`, and the caller's own scan of it is the right answer. let Some(reason) = u.incomplete_set else { continue; @@ -2419,70 +2058,42 @@ fn volume_set_name(part: &str) -> String { .unwrap_or_else(|| part.to_string()) } -/// Full in-memory analysis of a bounded buffer: core detection, then -/// recursive unpacking and (optionally) structural/ML/fuzzy heuristics. +/// Scan an object held in memory: [`scan_seekable`] without the reader. pub fn analyze(db: &Scanner, data: &[u8], opts: &ScanOptions) -> ScanReport { - engine::reset_scan_truncated(); - let report = analyze_inner(db, data, opts); - suppress(db, data, report) + analyze_source(db, &data, opts) } -fn analyze_inner(db: &Scanner, data: &[u8], opts: &ScanOptions) -> ScanReport { - if let Some((sig, off, method, unofficial)) = - scan_bytes_core(db, data, opts.filename.as_deref(), opts.unofficial_suffix) - { - return ScanReport::infected( - report_name(&sig, unofficial, opts.unofficial_suffix), - off, - method, - Vec::new(), - ); - } +/// As [`analyze`], over an object that need not be held in memory. +fn analyze_source(db: &Scanner, data: &dyn ByteSource, opts: &ScanOptions) -> ScanReport { + engine::reset_scan_truncated(); let mut findings = Vec::new(); let mut budget = scan_budget(db, opts); - match deep_analyze( + let outcome = scan_object( db, data, + Obj::top(opts), opts, &mut budget, - 0, - None, - true, &mut findings, &mut Sink::First(&db.ignored), - ) { - DeepOutcome::Infected { - signature, - offset, - method, - } => ScanReport::infected(signature, offset, method, findings), - DeepOutcome::Limits(reason) => ScanReport::limits(reason, findings), - // "Not-fully-scanned is never Clean" holds in EVERY mode — exav never - // downgrades a real safety verdict to `Clean` to mimic clam's silent - // `OK`. compat matches clam's capabilities and naming, not this. (A diff - // harness should bucket exav-`UNSCANNABLE` vs clam-`OK` as an expected - // capability difference, not have exav lie.) - DeepOutcome::Unscannable(reason) => ScanReport::unscannable(reason, findings), - DeepOutcome::PasswordProtected(reason) => ScanReport::password_protected(reason, findings), - // Cardinal rule: a would-be `Clean` is downgraded to `LimitsExceeded` if - // any wildcard verification was skipped because a per-buffer scan bound - // ran out (the step pool or the repeated-anchor cap) — the search did - // not fully complete, so we must not report `OK`. - DeepOutcome::Clean if engine::scan_was_truncated() => { - ScanReport::limits(SEARCH_INCOMPLETE.into(), findings) - } - DeepOutcome::Clean => ScanReport::clean(findings), - } + ); + report_of_outcome(outcome, findings, opts) } /// Turn a walk's outcome into the report a caller sees. /// -/// The walk speaks one currency — [`DeepOutcome`] — whatever entry point drove +/// The walk speaks one currency, [`DeepOutcome`], whatever entry point drove /// it; this is the single place that becomes a [`ScanReport`]. +/// +/// "Not-fully-scanned is never Clean" holds in EVERY mode: exav never +/// downgrades a real safety verdict to `Clean` to mimic clam's silent `OK`. +/// compat matches clam's capabilities and naming, not this. (A diff harness +/// should bucket exav-`UNSCANNABLE` vs clam-`OK` as an expected capability +/// difference, not have exav lie.) fn report_of_outcome( outcome: DeepOutcome, findings: Vec, - _opts: &ScanOptions, + opts: &ScanOptions, ) -> ScanReport { match outcome { DeepOutcome::Infected { @@ -2493,50 +2104,28 @@ fn report_of_outcome( DeepOutcome::Limits(reason) => ScanReport::limits(reason, findings), DeepOutcome::Unscannable(reason) => ScanReport::unscannable(reason, findings), DeepOutcome::PasswordProtected(reason) => ScanReport::password_protected(reason, findings), - // A would-be `Clean` is downgraded when wildcard verification ran out of - // its per-buffer scan bounds: the search did not complete, so `OK` would - // be a claim the scan cannot support. - DeepOutcome::Clean if engine::scan_was_truncated() => { - ScanReport::limits(SEARCH_INCOMPLETE.into(), findings) - } - DeepOutcome::Clean => ScanReport::clean(findings), + DeepOutcome::Clean => match unfinished(opts) { + Some(outcome) => report_of_outcome(outcome, findings, opts), + None => ScanReport::clean(findings), + }, } } -/// True if a detection of `signature` on `data` should be suppressed: the -/// name is on the ignore list (`.ign`/`.ign2`) or `data`'s whole-file hash is -/// allowlisted (`.fp`/`.sfp`). -fn is_suppressed(db: &Scanner, data: &[u8], signature: &str) -> bool { - db.ignored.contains(signature) || allowlisted(db, data) -} - -/// Clear a detection if it is suppressed (see [`is_suppressed`]). -fn suppress(db: &Scanner, data: &[u8], report: ScanReport) -> ScanReport { - if let Verdict::Infected { signature, .. } = &report.verdict { - if is_suppressed(db, data, signature) { - return ScanReport::clean(report.findings); - } +/// What a walk that found nothing amounts to when a check did not finish: +/// the size limit, when an object too large to hold kept one from running, +/// else an incomplete search (a per-buffer bound ran out). `None` when the +/// walk was complete and `Clean` is the true answer. +fn unfinished(opts: &ScanOptions) -> Option { + if let Some(reason) = engine::scan_over_size() { + return Some(limits_outcome(opts, unpack::LimitKind::MaxFileSize, reason)); } - report -} - -/// Name-only suppression for the streaming paths, where the whole object is -/// not buffered so the `.fp`/`.sfp` hash allowlist cannot be evaluated (it -/// would require reading the entire stream, defeating the early-exit). The -/// `.ign`/`.ign2` name ignore-list needs no data and is always applied. -fn suppress_name(db: &Scanner, report: ScanReport) -> ScanReport { - if let Verdict::Infected { signature, .. } = &report.verdict { - if db.ignored.contains(signature) { - return ScanReport::clean(report.findings); - } - } - report + engine::scan_was_truncated().then(|| DeepOutcome::Limits(SEARCH_INCOMPLETE.into())) } /// Every detection on `data` or its (recursively unpacked) members, -/// de-duplicated by name — the data behind `--all-matches`. +/// de-duplicated by name: the data behind `--all-matches`. /// -/// This is `deep_analyze`, the same walk a normal scan uses, driven by a sink +/// This is [`scan_object`], the same walk a normal scan uses, driven by a sink /// that collects instead of stopping. Every heuristic, decoder and recursion /// step is therefore shared by construction: all-match cannot see less than a /// normal scan, because it *is* a normal scan that declines to stop. @@ -2554,13 +2143,49 @@ pub fn analyze_all_with_outcome( db: &Scanner, data: &[u8], opts: &ScanOptions, +) -> (Vec<(String, Method)>, AllMatchOutcome) { + analyze_all_source(db, &data, opts) +} + +/// As [`analyze_all_with_outcome`], over a seekable input of `size` bytes. +/// One too large to hold is read through a block cache of bounded size, so +/// every detection is listed whatever the input's size. +pub fn analyze_all_seekable( + db: &Scanner, + mut reader: R, + size: u64, + opts: &ScanOptions, +) -> io::Result<(Vec<(String, Method)>, AllMatchOutcome)> { + if size <= opts.deep_analysis_max { + let mut data = Vec::new(); + (&mut reader) + .take(opts.deep_analysis_max.saturating_add(1)) + .read_to_end(&mut data)?; + if data.len() as u64 <= opts.deep_analysis_max { + return Ok(analyze_all_with_outcome(db, &data, opts)); + } + reader.seek(SeekFrom::Start(0))?; + } + let cache = byte_source::BlockCache::new(reader)?; + let found = analyze_all_source(db, &cache, opts); + // A read that failed part way leaves bytes the scan never saw. + if let Some(e) = cache.read_error() { + return Err(io::Error::other(e)); + } + Ok(found) +} + +fn analyze_all_source( + db: &Scanner, + data: &dyn ByteSource, + opts: &ScanOptions, ) -> (Vec<(String, Method)>, AllMatchOutcome) { // The flag is thread-local and sticky, so it has to be cleared per scan the // way `analyze` clears it. Before this function read it the omission was // invisible; with the read below, a stale `true` from an earlier scan on // this thread would report a complete search as truncated. engine::reset_scan_truncated(); - let (names, outcome) = analyze_all_raw(db, data, opts); + let (mut names, outcome) = analyze_all_raw(db, data, opts); let outcome = match outcome { Some(DeepOutcome::Limits(r)) => AllMatchOutcome::LimitsExceeded(one_line(r)), Some(DeepOutcome::Unscannable(r)) => AllMatchOutcome::Unscannable(one_line(r)), @@ -2568,24 +2193,29 @@ pub fn analyze_all_with_outcome( // The same cardinal rule the single-verdict paths apply, which this one // was missing: a search that did not finish must not be presented as a // complete one. There it downgrades a would-be `Clean`; here there is no - // verdict to downgrade — the detections stand — so it becomes the + // verdict to downgrade (the detections stand), so it becomes the // outcome that travels alongside them, and the caller reports both. - _ if engine::scan_was_truncated() => { - AllMatchOutcome::LimitsExceeded(SEARCH_INCOMPLETE.into()) - } - _ => AllMatchOutcome::Complete, + _ => match unfinished(opts) { + Some(DeepOutcome::Infected { + signature, method, .. + }) => { + if !names.iter().any(|(n, _)| *n == signature) { + names.push((signature, method)); + } + AllMatchOutcome::Complete + } + Some(DeepOutcome::Limits(r)) => AllMatchOutcome::LimitsExceeded(one_line(r)), + _ => AllMatchOutcome::Complete, + }, }; (names, outcome) } fn analyze_all_raw( db: &Scanner, - data: &[u8], + data: &dyn ByteSource, opts: &ScanOptions, ) -> (Vec<(String, Method)>, Option) { - if allowlisted(db, data) { - return (Vec::new(), None); - } let mut out = Vec::new(); let mut seen = std::collections::HashSet::new(); let mut findings = Vec::new(); @@ -2595,20 +2225,12 @@ fn analyze_all_raw( seen: &mut seen, ignored: &db.ignored, }; - // The walk core-scans every buffer it *reaches* — members, carved images, - // decoded payloads — but not the one it is handed, because on the - // first-match path `analyze_inner` has already scanned that itself before - // calling in. So the top-level buffer is scanned here, or a signature on the - // file's own bytes is never looked for. - core_scan(db, data, None, None, opts, &mut sink); - let outcome = deep_analyze( + let outcome = scan_object( db, data, + Obj::top(opts), opts, &mut budget, - 0, - None, - true, &mut findings, &mut sink, ); @@ -2629,7 +2251,7 @@ pub fn analyze_all(db: &Scanner, data: &[u8], opts: &ScanOptions) -> Vec<(String /// The partial outcome of an all-match scan, when there is one. /// /// All-match and a normal scan may legitimately differ in HOW MANY signatures -/// they list. They must never differ on whether the file was fully scanned — +/// they list. They must never differ on whether the file was fully scanned: /// that is a property of the walk, not of how many results it was asked for. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AllMatchOutcome { @@ -2643,21 +2265,35 @@ pub enum AllMatchOutcome { PasswordProtected(String), } -/// The checks that need the WHOLE object at once, rather than a member of it. -/// -/// Separated because that requirement is what divides the two ways an object -/// reaches the scanner. A buffered walk always has the bytes; a streamed walk -/// holds only the member it is on, so it can run these only when the container -/// itself fits in memory — and must say so when it does not, rather than -/// quietly scanning less. +/// The structural checks on an object as a whole, rather than on what is +/// inside it. Those that parse it whole do not run on an object over the +/// deep-analysis limit, and the scan says so (see [`whole`]). /// /// `Some(outcome)` means the walk must stop and return it. fn whole_buffer_heuristics( - data: &[u8], + src: &dyn ByteSource, ft: FileType, + embedded: bool, opts: &ScanOptions, sink: &mut Sink, ) -> Option { + // Read once, and only if a check that applies needs it. + let all = std::cell::OnceCell::new(); + let whole = || { + all.get_or_init(|| whole(src, opts, "the whole-file structural heuristics")) + .as_deref() + }; + let head = src.window(0, 16); + let executable = [ + &b"MZ"[..], + b"\x7fELF", + b"\xcf\xfa\xed\xfe", + b"\xfe\xed\xfa\xcf", + b"\xce\xfa\xed\xfe", + b"\xfe\xed\xfa\xce", + ] + .iter() + .any(|m| head.starts_with(m)); // Overlapping ZIP local file records: a parser-confusion technique where two // readers disagree about where a member starts, so the archive shows one // file to the scanner and another to the tool that opens it. ClamAV alerts @@ -2666,7 +2302,7 @@ fn whole_buffer_heuristics( // many. if (opts.clamav_heuristics || opts.heuristics) && ft == FileType::Zip { const OVERLAP_THRESHOLD: usize = 5; - if unpack::overlapping_local_records(data) > OVERLAP_THRESHOLD { + if unpack::overlapping_local_records(src) > OVERLAP_THRESHOLD { if let Some(o) = sink.hit( "Heuristics.Zip.OverlappingFiles".to_string(), 0, @@ -2683,7 +2319,9 @@ fn whole_buffer_heuristics( // it off; exav already refuses to allocate past its cap, so this reports the // condition rather than letting it surface as an opaque decode failure. if ft == FileType::Xz { - if let Some(dict) = unpack::xz_declared_dict_size(data) { + // The stream header and the first block header, which is at most + // 1024 bytes. + if let Some(dict) = unpack::xz_declared_dict_size(&src.window(0, 12 + 1024 + 64)) { if dict > unpack::XZ_MAX_DICT { if let Some(o) = sink.hit( "Heuristics.XZ.DicSizeLimit".to_string(), @@ -2698,21 +2336,39 @@ fn whole_buffer_heuristics( // Images whose container does not hold together. Opt-in, matching ClamAV: // a viewer renders a truncated GIF happily, and that forgiveness is what - // exploit writers aim at — so the mismatch between "renders" and "parses" is + // exploit writers aim at, so the mismatch between "renders" and "parses" is // itself the signal. - if opts.alert_broken_media { - if let Some(name) = unpack::broken_media_alert(data) { + // Checked only for the formats `broken_media_alert` knows. + let media = [ + &b"GIF87a"[..], + b"GIF89a", + b"\x89PNG", + b"II\x2a\x00", + b"MM\x00\x2a", + b"\xff\xd8", + ] + .iter() + .any(|m| head.starts_with(m)); + if opts.alert_broken_media && media { + if let Some(name) = whole().and_then(unpack::broken_media_alert) { if let Some(o) = sink.hit(name.to_string(), 0, Method::Heuristic) { return Some(o); } } } - // A file that claims to be an executable and does not parse as one. The - // signal is the contradiction: ordinary software ships well-formed headers, - // while truncation, corruption and droppers that lean on a forgiving loader - // do not. - if opts.alert_broken && pe::looks_broken(data) { + // A file that claims to be an executable and whose headers describe a + // layout no loader could map. The signal is the contradiction: ordinary + // software ships well-formed headers, while truncation, corruption and + // droppers that lean on a forgiving loader do not. Reads only the headers, + // so it runs whatever the object's size. + // + // Neither this nor the stripped-ELF check below runs on an executable + // carved out of its host: that is a candidate found by its magic, and one + // that does not parse is a fragment or a false hit, not a broken file. + // clamscan `--alert-broken` agrees: it reports a broken PE or ELF alone and + // not embedded in another file. + if opts.alert_broken && executable && !embedded && pe::looks_broken_in(src) { if let Some(o) = sink.hit( "Heuristics.Broken.Executable".to_string(), 0, @@ -2722,7 +2378,7 @@ fn whole_buffer_heuristics( } } // An ELF whose section-header table has been stripped. Detected in BOTH - // modes — only the name differs. It is not breakage (the program headers are + // modes; only the name differs. It is not breakage (the program headers are // intact and the binary runs), but no toolchain zeroes the entry size, so it // is a deliberate anti-analysis step worth reporting under its own name. // @@ -2730,7 +2386,7 @@ fn whole_buffer_heuristics( // actually found, and under `--clamav-compat` says what ClamAV would, because // a gateway filtering on ClamAV's exact string has to keep matching. What is // reported never changes; only the vocabulary does. - if opts.alert_broken && pe::elf_section_headers_stripped(data) { + if opts.alert_broken && !embedded && pe::elf_section_headers_stripped_in(src) { let name = if opts.clamav_compat { "Heuristics.Broken.Executable" } else { @@ -2746,7 +2402,7 @@ fn whole_buffer_heuristics( // whatever scans it. Parser confusion one layer below the ZIP case above. // Opt-in, matching ClamAV's `--alert-partition-intersection`. if opts.alert_partition_intersection { - if let Some(name) = unpack::partition_intersection_alert(data) { + if let Some(name) = unpack::partition_intersection_alert(src) { if let Some(o) = sink.hit(name.to_string(), 0, Method::Heuristic) { return Some(o); } @@ -2755,12 +2411,12 @@ fn whole_buffer_heuristics( // ClamAV `Heuristics.PDF.ObfuscatedNameObject`: a PDF whose name objects // hex-escape plain alphanumerics (`/J#61vaScript`) to hide keywords from - // naive scanners. Structural and FP-safe — only gratuitous escapes count. + // naive scanners. Structural and FP-safe: only gratuitous escapes count. // Applied to the raw document, before the PDF is unpacked. #[cfg(feature = "pdf")] if (opts.clamav_heuristics || opts.heuristics) && ft == FileType::Pdf - && unpack::has_obfuscated_name_object(data) + && whole().is_some_and(unpack::has_obfuscated_name_object) { if let Some(o) = sink.hit( "Heuristics.PDF.ObfuscatedNameObject".to_string(), @@ -2777,7 +2433,7 @@ fn whole_buffer_heuristics( /// /// A normal scan and `--all-matches` are the same traversal answering one question /// differently: *is this hit enough?* That is the only difference, so it is the -/// only thing parameterised — the traversal reports to a sink rather than +/// only thing parameterised: the traversal reports to a sink rather than /// returning a verdict, and there is exactly one traversal. /// /// Two walks cannot be kept in step by discipline. Each capability added to one @@ -2789,7 +2445,7 @@ enum Sink<'a> { /// rest of the object unscanned on the strength of a match the operator /// asked to have ignored. First(&'a std::collections::HashSet), - /// Collect every distinct detection and keep walking — `--all-matches`. + /// Collect every distinct detection and keep walking (`--all-matches`). All { out: &'a mut Vec<(String, Method)>, seen: &'a mut std::collections::HashSet, @@ -2842,7 +2498,7 @@ enum DeepOutcome { }, Limits(String), /// A member was recognised but couldn't be decoded (unsupported method). - /// Unlike `Limits` it does NOT stop scanning sibling members — it's + /// Unlike `Limits` it does NOT stop scanning sibling members; it's /// remembered and surfaced only if nothing infected is found. Unscannable(String), /// An encrypted member: like `Unscannable` but actionable (re-scan with a @@ -2853,9 +2509,9 @@ enum DeepOutcome { /// Everything a container-member scan needs that does not vary between the /// members of one container. /// -/// Bundled because the same per-member logic runs from two places: the -/// `extract_each` visitor as members arrive, and — after it returns — over the -/// files rejoined from a multi-volume set, which have no visitor to run in. +/// Bundled because the same per-member logic runs from two places: the walk's +/// visitor as members arrive, and, after the walk, over the files rejoined +/// from a multi-volume set. struct MemberCtx<'a> { db: &'a Scanner, opts: &'a ScanOptions, @@ -2869,29 +2525,6 @@ struct MemberCtx<'a> { depth: u32, } -/// What the metadata checks need to know about a member, independent of how it -/// was produced. -/// -/// The buffered walk has an `unpack::Entry` with the bytes already decoded; the -/// streamed walk has a `MemberMeta` and a reader it has not touched yet. Both -/// know these five things, and the checks that run on them — `.cdb` container -/// signatures, the macro heuristic, the encrypted-member heuristic — care about -/// nothing else. Taking the facts rather than either struct is what lets one -/// implementation serve both. -struct MemberFacts<'a> { - name: &'a str, - /// Size within the container, for `.cdb` `ContainerSize` matching. - comp_size: u64, - /// Decompressed size when known. The streamed walk has not decoded the - /// member yet, so it passes `comp_size` — the same value the walk used - /// before this was shared. - size_real: u64, - encrypted: bool, - /// `Some(reason)` when the member was recognised but its content could not - /// be decoded. - unsupported: Option<&'static str>, -} - /// Outcomes gathered across a container's members that do not stop the loop. /// They decide the verdict once every member has been seen. struct MemberTally { @@ -2926,16 +2559,18 @@ impl MemberTally { } } -/// The checks that read a member's *metadata* — name, size, position, -/// encryption — rather than its content. +/// The checks that read a member's *metadata* (name, size, position, +/// encryption) rather than its content. /// /// Run as each member arrives, before any decision about its bytes, so /// positions stay in container order even for members whose content is held -/// back to be rejoined. The caller owns the [`MatchPathGuard`]. +/// back to be rejoined. `size_real` is its decoded size, or its size in the +/// container when it was not decoded. The caller owns the [`MatchPathGuard`]. fn member_metadata_scan( cx: &MemberCtx<'_>, tally: &mut MemberTally, - e: MemberFacts<'_>, + e: &unpack::MemberMeta, + size_real: u64, sink: &mut Sink, ) -> Option { // `.cdb` container-metadata signatures match on the member's @@ -2956,13 +2591,13 @@ fn member_metadata_scan( // succeeding at decryption ERASE the report: a member exav cracks with a pool // password (`VelvetSweatshop` for Office, the malware-convention list for // ZIP) reaches here with `encrypted: false, unsupported: None`, so a rule - // keyed on undecodable content would never see it — exav would decrypt the + // keyed on undecodable content would never see it: exav would decrypt the // document, scan the plaintext, and say nothing about it having been // encrypted at all. // // Decrypting stays a genuine advantage over clamd: the recovered plaintext is // still scanned for real signatures. It just no longer costs us the fact. - // A member we could NOT read reports here and now — there is no content + // A member we could NOT read reports here and now: there is no content // coming that could outrank it. A member we DECRYPTED is different: its // plaintext is about to be scanned, and a real signature in that plaintext // must win over "this was encrypted", so under first-match the heuristic @@ -2979,11 +2614,11 @@ fn member_metadata_scan( return Some(o); } } - // Opt-in (`--alert-packed`): name the packer as well as reporting that its - // payload went unread. Same rule as the encryption case above — two facts, + // Opt-in (`--detect packed`): name the packer as well as reporting that its + // payload went unread. Same rule as the encryption case above: two facts, // both true, both reported. if cx.opts.alert_packed { - if let Some(n) = packed_heuristic_name(e.name) { + if let Some(n) = packed_heuristic_name(&e.name) { if let Some(o) = sink.hit(n, 0, Method::Heuristic) { return Some(o); } @@ -2993,7 +2628,7 @@ fn member_metadata_scan( if e.encrypted { // Only an encrypted member we could NOT read is password-blocked. // A decrypted one has its content and must not degrade the verdict - // to PasswordProtected — this is the one place the two facts stay + // to PasswordProtected; this is the one place the two facts stay // deliberately separate. tally.password.get_or_insert_with(|| r.to_string()); } else { @@ -3001,14 +2636,14 @@ fn member_metadata_scan( } } // Opt-in ClamAV heuristic (`--alert-macros`): an OLE2 document carrying a - // VBA project surfaces `vba_project*` artifacts from the OLE extractor — + // VBA project surfaces `vba_project*` artifacts from the OLE extractor; // their presence means the document has macros. if cx.opts.alert_macros && cx.container_is_ole { - // ClamAV suffixes the macro dialect — `.VBA` for a VBA project, `.XLM` + // ClamAV suffixes the macro dialect: `.VBA` for a VBA project, `.XLM` // for an Excel 4.0 macro sheet. Emitting the bare name looked harmless // and is not: a gateway filtering on ClamAV's exact string matches // neither of ours. - if let Some(kind) = macro_dialect(e.name) { + if let Some(kind) = macro_dialect(&e.name) { if let Some(o) = sink.hit( format!("Heuristics.OLE2.ContainsMacros.{kind}"), 0, @@ -3020,9 +2655,9 @@ fn member_metadata_scan( } if !cx.db.cdb.is_empty() { let member = container::Member { - name: e.name, + name: &e.name, size_in_container: e.comp_size, - size_real: e.size_real, + size_real, encrypted: e.encrypted, pos: member_pos, }; @@ -3041,62 +2676,49 @@ fn member_metadata_scan( None } -/// Scan one member's *bytes*: the pattern/hash core, then recursion into it. +/// Scan one member's bytes, as an object one level below its container. /// /// Split from [`member_metadata_scan`] because a member of a multi-volume set /// has its metadata read on arrival but its content scanned only after the /// container ends, once the set has been rejoined. The caller owns the -/// [`MatchPathGuard`] — this is entered under the member's own name in the +/// [`MatchPathGuard`]: this is entered under the member's own name in the /// visitor and under the rejoined file's name afterwards. fn member_content_scan( cx: &MemberCtx<'_>, tally: &mut MemberTally, - data: &[u8], + data: &dyn ByteSource, budget: &mut Budget, findings: &mut Vec, sink: &mut Sink, ) -> Option { - // A member whose own whole-file hash is allowlisted (`.fp`/`.sfp`) is - // content the operator has vouched for: neither it nor anything nested - // inside it is a detection. Checked once, here, so an allowlist entry means - // the same thing for a member of any container — the allowlist is keyed on - // the member's bytes, which do not depend on how the member was produced. - if allowlisted(cx.db, data) { - return None; - } - // Textual content extracted from an OLE2 document is scanned in OLE + // Textual content extracted from an OLE2 document is matched in OLE // context: its type is forced to MSOLE2 so `Target:2` macro sigs apply and - // `Target:7` (ascii-text) sigs do NOT — without this a generic text macro + // `Target:7` (ascii-text) sigs do NOT. Without this a generic text macro // sig (e.g. `Doc.Downloader.Macro-25` on the standard `Name="Project"…` // PROJECT stream) false-positives on benign macro documents. Binary streams // (an embedded PE, etc.) keep their own type so embedded-executable - // detection is preserved. Either way the member carries its container type - // so `Container:`-scoped sigs are gated correctly. - let ft_override = if cx.container_is_ole + // detection is preserved. + let core_type = if cx.container_is_ole && is_textual_type(profile::timed("filetype", data.len() as u64, || { - filetype::identify(data) + filetype::identify_source(data) })) { Some(FileType::Ole) } else { None }; - if let Some(o) = core_scan(cx.db, data, ft_override, cx.member_container, cx.opts, sink) { - return Some(o); - } - match deep_analyze( - cx.db, - data, - cx.opts, - budget, - cx.depth + 1, - cx.member_container, - true, - findings, - sink, - ) { + let obj = Obj { + depth: cx.depth + 1, + container: cx.member_container, + core_type, + filename: None, + core: true, + carve: true, + embedded: false, + }; + match scan_object(cx.db, data, obj, cx.opts, budget, findings, sink) { DeepOutcome::Clean => None, // Nested unscannable/encrypted members are remembered, not propagated as - // a stop — keep scanning the rest of this container. + // a stop: keep scanning the rest of this container. DeepOutcome::Unscannable(r) => { tally.unscannable.get_or_insert(r); None @@ -3115,103 +2737,190 @@ fn member_content_scan( } } -/// Recursive structural analysis of an in-memory buffer. -#[allow(clippy::too_many_arguments)] -fn deep_analyze( - db: &Scanner, - data: &[u8], - opts: &ScanOptions, - budget: &mut Budget, +/// Where an object sits in the scan, and what its container decided about it. +#[derive(Clone, Copy)] +struct Obj<'a> { + /// Nesting depth; the input is at 0. depth: u32, - // Container type this buffer sits inside (for `Container:CL_TYPE_*` scoping), - // threaded so embedded/nested content inherits it rather than over-matching. + /// The type of the container the object sits in, for `Container:`-scoped + /// signatures. Inherited by what is carved or decoded out of the object, + /// rather than over-matching. container: Option, - // Whether to enumerate embedded PE/ELF/Mach-O images in this buffer. False - // when we were *reached by* embedded-carving: the parent already enumerated - // every embedded offset in the larger buffer (a carved suffix is a subset), - // so re-carving here would rescan the same overlapping regions — the source - // of large scan-amplification. We still extract archives + run heuristics on - // the carved image (so an appended archive in a dropper is not missed). + /// The type the matching core takes the object for, when its container + /// decides that (a textual OLE stream is matched as OLE); else its own. + core_type: Option, + /// The input's name, for YARA's filename externals. Nothing inside the + /// input has one. + filename: Option<&'a str>, + /// Whether to match the object's own bytes. Off for an archive carved out + /// of an object whose own match already covered them. + core: bool, + /// Whether to carve embedded executables and archives out of the object. + /// Off for what carving or decoding produced: its parent already + /// enumerated every offset in it, and carving again would rescan the same + /// overlapping regions, the source of a large scan amplification. carve: bool, + /// Whether the object was carved out of its host at an offset, found by a + /// magic rather than typed as a whole. It runs to the host's end, whatever + /// its headers say. + embedded: bool, +} + +impl<'a> Obj<'a> { + /// The input. + fn top(opts: &'a ScanOptions) -> Self { + Obj { + depth: 0, + container: None, + core_type: None, + filename: opts.filename.as_deref(), + core: true, + carve: true, + embedded: false, + } + } + + /// Something found inside this object, one level down, in `container`. + fn inner(self, container: Option) -> Self { + Obj { + depth: self.depth + 1, + container, + core_type: None, + filename: None, + core: true, + carve: true, + embedded: false, + } + } +} + +/// Scan one object and everything inside it: the one pipeline every object +/// goes through, the input and whatever is unpacked, decoded or carved out of +/// it, whatever its size and wherever it sits. +/// +/// An archive's members are scanned before its own bytes, so a detection in a +/// member is attributed to it rather than to the container that happens to +/// hold its bytes stored. Anything else is matched first, then analysed. +fn scan_object( + db: &Scanner, + data: &dyn ByteSource, + obj: Obj<'_>, + opts: &ScanOptions, + budget: &mut Budget, findings: &mut Vec, - // Where detections go, and whether finding one ends the walk. This is the - // ONLY difference between a normal scan and `--all-matches`; see [`Sink`]. sink: &mut Sink, ) -> DeepOutcome { - // A `HandlerType:` signature that matched this buffer during the pattern - // scan just above says "treat this as type T" — it is programmable file-type - // identification, filling in where the magic tables cannot: a PDF exploit - // recognised by its object layout rather than a `%PDF` header still gets - // opened as a PDF. Re-typing to what we already decided would be a no-op, so - // only a genuine change is taken, which also makes a loop impossible. - let ft = db.identify(data); - let ft = match engine::take_retype(data).filter(|t| *t != ft) { - None => ft, - Some(retyped) => { - // The re-type is only worth anything if the buffer is scanned AS the - // new type: that is what brings the type's signatures and its - // extractor into play. Rescan here, then carry `retyped` through the - // rest of this frame so unpacking dispatches on it too. - if let Some(o) = core_scan(db, data, Some(retyped), container, opts, sink) { - return o; - } - retyped + // Content the operator has vouched for (`.fp`/`.sfp`): neither it nor + // anything inside it is a detection. Keyed on the object's bytes, so an + // entry means the same thing wherever the object is found. + let carve = obj.carve && obj.depth < budget.limits().max_recursion; + let facts = Facts::for_scan(data, db, carve); + if allowlisted(db, &facts) { + return DeepOutcome::Clean; + } + let ft = db.identify_facts(&facts); + match unpack_target(ft, &facts, opts.restrict_extractors, opts) { + Some(fmt) if !ft.is_executable() => { + scan_archive(db, &facts, obj, ft, fmt, opts, budget, findings, sink) } - }; - if depth == 0 { + fmt => scan_file(db, &facts, obj, ft, fmt, opts, budget, findings, sink), + } +} + +/// [`scan_object`] for an archive: what is inside it, then its own bytes. +#[allow(clippy::too_many_arguments)] +fn scan_archive( + db: &Scanner, + facts: &Facts, + obj: Obj<'_>, + ft: FileType, + fmt: unpack::Format, + opts: &ScanOptions, + budget: &mut Budget, + findings: &mut Vec, + sink: &mut Sink, +) -> DeepOutcome { + let data = facts.data; + if obj.depth == 0 { findings.push(Finding::new("type", ft.as_str())); } + if let Some(o) = scan_carried(db, data, obj, ft, opts, budget, findings, sink) { + return o; + } + let members = walk_members(db, data, obj, ft, fmt, opts, budget, findings, sink); + // Matched whatever stopped the walk short of a detection: a detection beats + // a limit. + if matches!(members, DeepOutcome::Infected { .. }) || !obj.core { + return members; + } + let mut outcome = members; + let rest = match core(db, facts, obj.core_type.unwrap_or(ft), obj, opts, sink) { + Ok(rest) => rest, + Err(stop) => return stop, + }; + scan_unpacked( + db, + rest.unpacked, + obj, + opts, + budget, + findings, + sink, + &mut outcome, + ) + .unwrap_or(outcome) +} +/// What an object carries encoded in its own bytes, and the checks that read +/// it as a whole: the steps before unpacking that archives and other objects +/// share. `Some` means the walk must stop. +#[allow(clippy::too_many_arguments)] +#[cfg_attr(not(feature = "base64scan"), allow(unused_variables, clippy::ptr_arg))] +fn scan_carried( + db: &Scanner, + data: &dyn ByteSource, + obj: Obj<'_>, + ft: FileType, + opts: &ScanOptions, + budget: &mut Budget, + findings: &mut Vec, + sink: &mut Sink, +) -> Option { // Embedded base64-encoded executables. Scripts/RTF/HTML carriers stash a // PE/ELF as a long base64 string (PowerShell reflective loaders, JS/VBS // droppers) that is invisible to a signature matching the decoded bytes. // Decode such blobs (in a text-ish buffer) and rescan any that decode to a - // real executable. Run BEFORE `unpack_target` so it also fires on carriers - // that are themselves containers (RTF/HTML/email return early below). Bounded - // by recursion depth and the scan budget; exav-exclusive, so off under - // `--clamav-compat` and `--no-base64`. + // real executable. Bounded by recursion depth and the scan budget; + // exav-exclusive, so off under `--clamav-compat` and `--no-base64`. #[cfg(feature = "base64scan")] - if opts.decode_base64 && depth < budget.limits().max_recursion && mostly_text(data) { + if opts.decode_base64 + && obj.depth < budget.limits().max_recursion + && mostly_text(&data.window(0, 8192)) + { for (b64ix, payload) in unpack::base64_payloads(data, budget.limits().max_buffer_bytes) .into_iter() .enumerate() { // Name the decoded run so a hit inside it is attributable. Without // this the detection reports no location and reads as a match on the - // carrier's own bytes — which is how a correct `Win.Trojan.Mimikatz` + // carrier's own bytes, which is how a correct `Win.Trojan.Mimikatz` // hit, on a PE base64-encoded inside an RTF, looked like a PE-only // signature firing on an RTF. let _mpg = MatchPathGuard::enter(&format!("base64-payload-{}", b64ix + 1)); - if let Err(h) = budget.charge_scan(payload.len() as u64) { - return limits_outcome(opts, h.kind, h.reason); - } // The payload's container is the CARRIER, not whatever the carrier - // itself sits in — a `data:` URI image inside an HTML page has + // itself sits in: a `data:` URI image inside an HTML page has // `Container:CL_TYPE_HTML`, which is how a family of phishing // signatures scopes a logo's perceptual hash so it fires on a page - // and not on the same image standing alone. Passing the carrier's - // own container instead left those signatures unsatisfiable. - let payload_container = carrier_cltype(ft).or(container); + // and not on the same image standing alone. + let payload_container = carrier_cltype(ft).or(obj.container); let _ag = engine::AncestryGuard::enter(payload_container); - // Pattern/hash-scan the decoded executable (this is where a signature - // like `Win.Trojan.Mimikatz` matches), then recurse structurally. - if let Some(o) = core_scan(db, &payload, None, payload_container, opts, sink) { - return o; - } - match deep_analyze( - db, - &payload, - opts, - budget, - depth + 1, - payload_container, - false, - findings, - sink, - ) { - inf @ DeepOutcome::Infected { .. } => return inf, - lim @ DeepOutcome::Limits(_) => return lim, - _ => {} + let inner = Obj { + carve: false, + ..obj.inner(payload_container) + }; + let o = scan_found(db, &payload, inner, opts, budget, findings, sink); + if matches!(o, DeepOutcome::Infected { .. } | DeepOutcome::Limits(_)) { + return Some(o); } } } @@ -3219,14 +2928,14 @@ fn deep_analyze( // Assets embedded straight into a markup document: a `data:` URI image on // an HTML page, a base64 element body in a Word/Excel 2003 flat-XML file. // Distinct from the base64-executable pass above, which only decodes runs - // starting with an executable magic — here the payload is usually the lure + // starting with an executable magic: here the payload is usually the lure // IMAGE, and it is the image that signatures key on. Extracting it is what // makes `Container:CL_TYPE_HTML`/`_XML_WORD`/`_XML_XL` satisfiable at all: // those constraints exist to separate "this image, in a document" from // "this image, on its own". #[cfg(feature = "base64scan")] - if opts.decode_base64 && depth < budget.limits().max_recursion { - if let Some(mc) = markup_cltype(ft, data) { + if opts.decode_base64 && obj.depth < budget.limits().max_recursion { + if let Some(mc) = markup_cltype(ft, &data.window(0, 4096)) { let _ag = engine::AncestryGuard::enter(Some(mc)); for (ix, payload) in unpack::markup_embedded_payloads(data, budget.limits().max_buffer_bytes) @@ -3234,255 +2943,202 @@ fn deep_analyze( .enumerate() { let _mpg = MatchPathGuard::enter(&format!("embedded-asset-{}", ix + 1)); - if let Err(h) = budget.charge_scan(payload.len() as u64) { - return limits_outcome(opts, h.kind, h.reason); - } - if let Some(o) = core_scan(db, &payload, None, Some(mc), opts, sink) { - return o; - } - match deep_analyze( - db, - &payload, - opts, - budget, - depth + 1, - Some(mc), - false, - findings, - sink, - ) { - inf @ DeepOutcome::Infected { .. } => return inf, - lim @ DeepOutcome::Limits(_) => return lim, - _ => {} + let inner = Obj { + carve: false, + ..obj.inner(Some(mc)) + }; + let o = scan_found(db, &payload, inner, opts, budget, findings, sink); + if matches!(o, DeepOutcome::Infected { .. } | DeepOutcome::Limits(_)) { + return Some(o); } } } } - if let Some(o) = whole_buffer_heuristics(data, ft, opts, sink) { - return o; - } + whole_buffer_heuristics(data, ft, obj.embedded, opts, sink) +} - // A verdict from unpacking a *packed executable* that has not been reported - // yet, because the file is also a carrier and the carving below still has to - // run. `Clean` until one is produced. - let mut deferred = DeepOutcome::Clean; +/// [`scan_object`] for content found inside another object rather than +/// decoded from a container, charged to the scan budget (a container's +/// members are charged as they are decoded). +fn scan_found( + db: &Scanner, + data: &dyn ByteSource, + obj: Obj<'_>, + opts: &ScanOptions, + budget: &mut Budget, + findings: &mut Vec, + sink: &mut Sink, +) -> DeepOutcome { + if let Err(h) = budget.charge_scan(data.len() as u64) { + return limits_outcome(opts, h.kind, h.reason); + } + scan_object(db, data, obj, opts, budget, findings, sink) +} - // Archives (and UPX-packed executables) are unpacked regardless of the - // heuristics flag. - if let Some(fmt) = unpack_target(ft, data, opts.restrict_extractors) { - if depth >= budget.limits().max_recursion { - return limits_outcome( +/// Scan the buffers a bytecode unpacker made of an object, each as an object of +/// its own. `Some` means the walk must stop; an outcome that does not stop it +/// goes to `deferred`. +#[allow(clippy::too_many_arguments)] +fn scan_unpacked( + db: &Scanner, + bufs: Vec>, + obj: Obj<'_>, + opts: &ScanOptions, + budget: &mut Budget, + findings: &mut Vec, + sink: &mut Sink, + deferred: &mut DeepOutcome, +) -> Option { + for (i, buf) in bufs.into_iter().enumerate() { + // The unpacker gives the buffer no name, so its index is its identity. + let _mpg = MatchPathGuard::enter(&format!("bytecode-unpacked-{}", i + 1)); + // An unpacker that fires again on its own output would never stop. + if obj.depth >= budget.limits().max_recursion { + return Some(limits_outcome( opts, unpack::LimitKind::MaxRecursion, format!("recursion depth exceeds {}", budget.limits().max_recursion), - ); - } - // A container whose members can be STREAMED is walked that way at every - // depth, not just when it is the file handed in. - // - // Otherwise nesting changes what a scan finds. Measured on the same - // 6 MB deflated member with `--max-object-bytes 1M`: as `lv0.zip` it was - // FOUND, and as `tar > lv0.zip` it was "archive member exceeds size - // budget". Identical bytes, different answer, purely because of depth — - // the buffered `extract_each` path caps a member at `max_buffer_bytes` - // while the streamed walk does not have to hold it at all. - if unpack::is_streamable(fmt) { - let member_container = container_cltype(fmt, data); - let _ag = engine::AncestryGuard::enter(member_container); - return scan_streamed_container( - db, - std::io::Cursor::new(data), - opts, - fmt, - ft, - member_container, - findings, - budget, - depth, - Some(data), - sink, - ); + )); } - // Stream members one at a time: scan + recurse into each, stopping (and - // not decompressing the rest) on the first detection. `pos` is the - // member's 1-based position in this container (for `.cdb` matching). - let container_size = data.len() as u64; - let container_is_ole = fmt == unpack::Format::Ole; - // The container type each extracted member belongs to, so signatures - // scoped with `Container:CL_TYPE_*` fire only inside their intended - // container (computed once per container; a ZIP is sub-typed as OOXML - // Word/Excel/PowerPoint when applicable). - let member_container = container_cltype(fmt, data); - let _ag = engine::AncestryGuard::enter(member_container); - // Members we recognised but couldn't decode are remembered here — they - // must not be reported clean, but they also must not stop us scanning - // the remaining members. Encrypted members are tracked separately so the - // (actionable) PasswordProtected verdict can take precedence. - let mut tally = MemberTally::new(); - let cx = MemberCtx { + match scan_found( db, + &buf, + obj.inner(obj.container), opts, - container_size, - container_is_ole, - member_container, - ft, - fmt, - depth, + budget, + findings, + sink, + ) { + DeepOutcome::Clean => {} + o @ (DeepOutcome::Infected { .. } | DeepOutcome::Limits(_)) => return Some(o), + o => defer(deferred, o), + } + } + None +} + +/// Keep the first outcome that does not stop the walk. +fn defer(slot: &mut DeepOutcome, outcome: DeepOutcome) { + if matches!(slot, DeepOutcome::Clean) { + *slot = outcome; + } +} + +/// [`scan_object`] for anything but an archive: its own bytes, then what is +/// inside it (a packed or installer executable's payload, carved images) and +/// the structural heuristics. +#[allow(clippy::too_many_arguments)] +fn scan_file( + db: &Scanner, + facts: &Facts, + obj: Obj<'_>, + ft: FileType, + fmt: Option, + opts: &ScanOptions, + budget: &mut Budget, + findings: &mut Vec, + sink: &mut Sink, +) -> DeepOutcome { + let data = facts.data; + let (depth, container) = (obj.depth, obj.container); + let mut unpacked = Vec::new(); + let (ft, fmt) = if obj.core { + let rest = match core(db, facts, obj.core_type.unwrap_or(ft), obj, opts, sink) { + Ok(rest) => rest, + Err(stop) => return stop, }; - // Members whose names mark them as parts of a byte-split set - // (`x.7z.001`, `.002`, …) are held here and rejoined once the container - // has ended — see the note at `finish()` below for why not sooner. - let mut volumes = unpack::volume::Collector::new(budget.limits().max_buffer_bytes); - // Inclusive wall time: this drives nested member scans, so `unpack_us` - // contains the member matchers' time too — pure extraction/dispatch is - // `unpack_us` minus the nested matcher columns, and `emu_us` splits the - // emulator's share of it out. - let outcome = profile::timed("unpack", data.len() as u64, || { - unpack::extract_each(fmt, data, budget, &mut |mut e: unpack::Entry, - budget: &mut Budget| - -> Option { - // Track this member on the location stack for its scan (see - // [`MatchPathGuard`]); a detection here or in a nested member - // reports the full path. - let _mpg = MatchPathGuard::enter(&e.name); - // A member byte-identical to its container is a **fixed point**: - // typing it re-detects the same format, which yields the same - // member, forever. It is never a real member — nothing was - // unwrapped — and following it burns the whole recursion budget - // on one buffer, so the content that actually needed those levels - // never gets reached. Skipping loses nothing: these exact bytes - // are already being scanned here, by this call. - // - // A FAT boot sector read as an MBR produces one, and any - // extractor can grow one by accident, so the guard lives here - // rather than in whichever extractor is responsible. - // - // The length test carries the cost: it is O(1) and false for - // essentially every real member, so the byte comparison — which - // short-circuits on the first difference — is only ever reached - // by a genuine fixed point. - if e.data.len() == data.len() && e.data == data { - return None; + unpacked = rest.unpacked; + // A `HandlerType:` signature that matched says "treat this as type T": + // programmable file-type identification, filling in where the magic + // tables cannot (a PDF exploit recognised by its object layout rather + // than a `%PDF` header still gets opened as a PDF). Only a genuine + // change is taken, which also makes a loop impossible. + match rest.retype.filter(|t| *t != ft) { + None => (ft, fmt), + Some(retyped) => { + // Worth something only if the object is matched AS the new + // type, and unpacked as it. + match core(db, facts, retyped, obj, opts, sink) { + Ok(rest) => unpacked.extend(rest.unpacked), + Err(stop) => return stop, } - let facts = MemberFacts { - name: &e.name, - comp_size: e.comp_size, - size_real: e.data.len() as u64, - encrypted: e.encrypted, - unsupported: e.unsupported, - }; - if let Some(o) = member_metadata_scan(&cx, &mut tally, facts, sink) { - return Some(o); - } - // Nothing decoded: the metadata above is all this member has. - // A member that decoded part way is still scanned. - if e.unsupported.is_some() && e.data.is_empty() { - return None; - } - // A part of a byte-split set is held, not scanned — its bytes - // are a fragment of a file that only exists once the set is - // rejoined. Everything else scans exactly as before. - let data = match volumes.offer(&e.name, std::mem::take(&mut e.data)) { - unpack::volume::Offer::Held => return None, - unpack::volume::Offer::PassThrough { data, .. } => data, - }; - member_content_scan(&cx, &mut tally, &data, budget, findings, sink) - }) - }); - // Reassembly happens only now. Nothing in a byte-split set's names says - // how many parts it has, so `.001`+`.002` looks contiguous even when - // `.003` follows: joining on arrival would emit a truncated prefix that - // still parses as the archive and would then be scanned as if whole. - // Completeness is only knowable once no further member can arrive. - let held = volumes.finish(); - let mut terminal = match outcome { - Ok(o) => o, - Err(hit) => Some(outcome_for_hit(hit, opts)), - }; - // Rejoined files first, then the parts that could not be joined. Bytes - // withheld from the scan and then dropped would be exactly the silent - // clean this scanner exists to prevent, so every part held above still - // gets scanned on its own here. - for (name, buf, incomplete) in held.into_scannable() { - if terminal.is_some() { - break; + let fmt = unpack_target(retyped, facts, opts.restrict_extractors, opts); + (retyped, fmt) } - // A set with a gap in it is reported: the archive those bytes belong - // to can no longer be read by anything — not by us and not by the - // tool that wrote it. - if let Some(reason) = incomplete { - tally.unscannable.get_or_insert_with(|| reason.to_string()); - } - // Nothing else accounts for these bytes: the parts were charged as - // they were extracted, but this is a buffer the collector made. - if let Err(h) = budget.charge_scan(buf.len() as u64) { - terminal = Some(limits_outcome(opts, h.kind, h.reason)); - break; - } - let _mpg = MatchPathGuard::enter(&name); - terminal = member_content_scan(&cx, &mut tally, &buf, budget, findings, sink); } - let verdict = match terminal { - Some(o) => o, - None => tally.verdict(), - }; + } else { + (ft, fmt) + }; + if depth == 0 { + findings.push(Finding::new("type", ft.as_str())); + } + // A verdict that does not stop the walk and has not been reported yet, + // because what follows still has to run. `Clean` until one is produced. + let mut deferred = DeepOutcome::Clean; + if let Some(o) = scan_unpacked( + db, + unpacked, + obj, + opts, + budget, + findings, + sink, + &mut deferred, + ) { + return o; + } + if let Some(o) = scan_carried(db, data, obj, ft, opts, budget, findings, sink) { + return o; + } + + // A packed or installer executable, or an object a `HandlerType:` + // signature retyped as a container. + if let Some(fmt) = fmt { + let verdict = walk_members(db, data, obj, ft, fmt, opts, budget, findings, sink); // A packed executable is not only a container, it is also a *carrier*. // Unpacking accounts for the image the stub rebuilds; it accounts for - // nothing appended to the file — and stapling an archive or a second PE + // nothing appended to the file, and stapling an archive or a second PE // onto the end of a packed dropper is one of the commonest shapes there - // is. For every other format, returning here is right: an archive's - // bytes are its members. For these two, a clean result falls through to - // the carving below, carrying any `UNSCANNABLE`/`PASSWORD-PROTECTED` - // verdict with it so that it is still reported if nothing is carved. + // is. For every other format, returning here is right. For these two, a + // clean result falls through to the carving below, carrying any + // `UNSCANNABLE`/`PASSWORD-PROTECTED` verdict with it so that it is + // still reported if nothing is carved. let packed_executable = matches!(fmt, unpack::Format::Upx | unpack::Format::PePacked); match verdict { DeepOutcome::Infected { .. } | DeepOutcome::Limits(_) => return verdict, - other if !packed_executable => return other, - other => deferred = other, + other => defer(&mut deferred, other), + } + if !packed_executable { + return deferred; } } // Embedded executables: scan PE/ELF images appended/embedded at a non-zero - // offset (file-infectors, droppers, self-extractors — on Windows via PE, on - // Linux via ELF). Each carved image is run through the pattern/hash core (so - // its section hashes match) and then recursed. Structural, not heuristic, so - // it runs regardless of the flag — bounded by recursion depth and the - // embedded-image cap. - if carve && depth < budget.limits().max_recursion { - let embedded = pe::embedded_pe_offsets(data) + // offset (file-infectors, droppers, self-extractors; on Windows via PE, on + // Linux via ELF). Each carved image is scanned as an object of its own (so + // its PE-relative offsets resolve and its section hashes match), charged + // against the cumulative scan budget so a crafted disk image (e.g. a 58 MB + // VHD full of PEs) trips `LimitsExceeded` instead of running for hours. + // Structural, not heuristic, so it runs regardless of the flag, bounded by + // recursion depth and the embedded-image cap. The carved image inherits the + // container its host sits in. + if obj.carve && depth < budget.limits().max_recursion { + let carved = Obj { + carve: false, + embedded: true, + ..obj.inner(container) + }; + // Every candidate of every kind, found in one read of the object. + let embedded = facts.embedded(); + let images = embedded + .pe .into_iter() - .chain(pe::embedded_elf_offsets(data)) - .chain(pe::embedded_macho_offsets(data)); - for off in embedded { - let sub = &data[off..]; - // Each carved suffix is scanned in full (PE-relative offsets resolve - // only when the image sits at position 0). Charge it against the - // cumulative scan budget so a crafted disk image (e.g. a 58 MB VHD - // full of PEs) trips `LimitsExceeded` instead of running for hours. - if let Err(h) = budget.charge_scan(sub.len() as u64) { - return limits_outcome(opts, h.kind, h.reason); - } - // The carved image inherits the container its host sits in. - if let Some(o) = core_scan(db, sub, None, container, opts, sink) { - return o; - } - // Recurse with carve=false: the anchored scan above already matched - // this image, and the embedded offsets within it are a subset of the - // ones this level enumerated — so we recurse only to extract an - // appended archive / unpack a packed stub, NOT to re-carve (which - // would rescan the same overlapping regions, the amplification bug). - match deep_analyze( - db, - sub, - opts, - budget, - depth + 1, - container, - false, - findings, - sink, - ) { + .chain(embedded.elf) + .chain(embedded.macho); + for off in images { + let sub = &byte_source::Sub::new(data, off, data.len() - off); + match scan_found(db, sub, carved, opts, budget, findings, sink) { DeepOutcome::Clean => {} other => return other, } @@ -3491,30 +3147,23 @@ fn deep_analyze( // Embedded archives: a ZIP/CAB/7z/RAR/GZIP/XZ appended to or stapled // inside a carrier (droppers, SFX stubs, PE overlays). The offset-0 scan // never types these, so carve each candidate whose magic is confirmed by - // the unpacker's own `detect` (skips coincidental byte-runs) and recurse - // to extract it. carve=false on recursion: the archive's own members are - // handled by extraction, not by re-carving overlapping regions. - for off in pe::embedded_archive_offsets(data) { - let sub = &data[off..]; - let Some(fmt) = unpack::detect(sub) else { + // the unpacker's own checks (skips coincidental byte-runs) and unpack + // it. Those read the candidate's first bytes: a coincidental magic + // costs no search through the rest of the object. The carrier's own + // match already covered the archive's bytes, so only what is inside it + // is new. + for off in embedded.archives { + let sub = &byte_source::Sub::new(data, off, data.len() - off); + let Some(fmt) = unpack::detect_archive_start(sub) else { continue; }; - if let Err(h) = budget.charge_scan(sub.len() as u64) { - return limits_outcome(opts, h.kind, h.reason); - } - match deep_analyze( - db, - sub, - opts, - budget, - depth + 1, - container, - false, - findings, - sink, - ) { + let archive = Obj { + core: false, + ..carved + }; + match scan_found(db, sub, archive, opts, budget, findings, sink) { // A real detection in an appended/stapled archive is the whole - // point of carving — surface it. A genuine resource limit (e.g. a + // point of carving: surface it. A genuine resource limit (e.g. a // decompression bomb in a real appended archive) still matters. inf @ DeepOutcome::Infected { .. } => return inf, lim @ DeepOutcome::Limits(_) => return lim, @@ -3527,7 +3176,8 @@ fn deep_analyze( pw @ DeepOutcome::PasswordProtected(_) if !ft.is_executable() && (fmt != unpack::Format::Zip - || unpack::zip_directory_is_consistent(sub)) => + || whole(sub, opts, "the check of an appended ZIP's directory") + .is_some_and(|w| unpack::zip_directory_is_consistent(&w))) => { deferred = pw; } @@ -3536,7 +3186,7 @@ fn deep_analyze( // collides with ordinary binary data (a false `1f8b08` / `PK\x03\x04` // byte-run inside a PE), so it is not really an archive here. The // carrier buffer is already fully pattern-scanned, so a failed guess - // must NOT poison it as UNSCANNABLE — that is a false positive + // must NOT poison it as UNSCANNABLE: that is a false positive // against `clamscan`, which reports such carriers clean. Move on. _ => {} } @@ -3565,12 +3215,17 @@ fn deep_analyze( // Authenticode inspection (independent of `--detect exav-heuristics`, like phishing/DLP). // One PE parse serves both checks: - // * `.crb` certificate block-list — a signed PE carrying a blocked signer + // * `.crb` certificate block-list: a signed PE carrying a blocked signer // cert is reported (always on when a `.crb` DB is loaded); - // * opt-in `alert_broken_authenticode` — the embedded digest does not cover + // * opt-in `alert_broken_authenticode`: the embedded digest does not cover // the file (tampered with / appended-to after signing). - if opts.alert_broken_authenticode || !db.crb.is_empty() { - if let Some(sig) = authenticode::analyze_pe(data) { + // Only a PE carries one, and it is parsed from all of the file. + let signed = + (opts.alert_broken_authenticode || !db.crb.is_empty()) && data.window(0, 2)[..] == *b"MZ"; + if signed { + if let Some(sig) = + whole(data, opts, "the Authenticode checks").and_then(|w| authenticode::analyze_pe(&w)) + { for cert in &sig.certs { if let Some(name) = db.crb.blocked(cert) { if let Some(o) = sink.hit(name.to_string(), 0, Method::Hash) { @@ -3599,18 +3254,26 @@ fn deep_analyze( // TLSH fuzzy matching is exav-exclusive (ClamAV has no TLSH), so it stays // behind the full `--detect exav-heuristics` flag and off under `--clamav-compat`. if opts.heuristics { - if let Some(hit) = profile::timed("fuzzy", data.len() as u64, || db.fuzzy.match_tlsh(data)) - { + if let Some(hit) = profile::timed("fuzzy", data.len() as u64, || { + db.fuzzy.match_tlsh_source(data) + }) { if let Some(o) = sink.hit(hit, 0, Method::Fuzzy) { return o; } } } - if ft == FileType::Pe { + // Parsed from all of the PE, so read only when something below uses it. + let image = if ft == FileType::Pe && (db.fuzzy.has_imphash() || opts.heuristics) { + whole(data, opts, "the PE import and static-model checks") + } else { + None + }; + if let Some(image) = image { + let data: &[u8] = ℑ if let Some(info) = pe::analyze(data) { - // imphash (`.imp`) matching is a ClamAV default — matched whenever the - // loaded DB carries `.imp` sigs — so it runs under `clamav_heuristics` + // imphash (`.imp`) matching is a ClamAV default (matched whenever the + // loaded DB carries `.imp` sigs), so it runs under `clamav_heuristics` // (i.e. under `--clamav-compat` too). The exav-exclusive ML scorer, // packed-injection heuristic, and the `-v` diagnostic findings below // stay behind the full `--detect exav-heuristics` flag. @@ -3679,7 +3342,7 @@ fn deep_analyze( /// `Heuristics.Packed.` for a member the PE-packer extractor surfaced as /// an image it could not unpack, or `None` for anything else. /// -/// The packer name arrives already spelled the way it should be reported — +/// The packer name arrives already spelled the way it should be reported: /// `pepack::packer_name` owns that vocabulary and writes it into the member name /// as `"-packed image"`. Nothing is re-derived here, so adding a packer /// there makes it reportable here with no second table to forget to update. @@ -3702,7 +3365,7 @@ fn encrypted_heuristic_name(fmt: unpack::Format) -> &'static str { Format::SevenZip => "Heuristics.Encrypted.7Zip", Format::Pdf => "Heuristics.Encrypted.PDF", // `OLE2`, not `Doc`. ClamAV's *config option* is `AlertEncryptedDoc`, - // but the *signature name* it emits is `Heuristics.Encrypted.OLE2` — + // but the *signature name* it emits is `Heuristics.Encrypted.OLE2`, // observed 468 times against 0 for `.Doc` over a corpus run. A gateway // filtering on ClamAV's exact string never matched ours. Format::Ole => "Heuristics.Encrypted.OLE2", @@ -3723,15 +3386,27 @@ fn macro_dialect(name: &str) -> Option<&'static str> { /// is met, else `None`. Only runs when a threshold is set, on textual buffers. /// The counters are linear and allocate nothing, so the whole buffer is counted. #[cfg(feature = "dlp")] -fn structured_data_scan(data: &[u8], opts: &ScanOptions, sink: &mut Sink) -> Option { +fn structured_data_scan( + data: &dyn ByteSource, + opts: &ScanOptions, + sink: &mut Sink, +) -> Option { if opts.structured_cc_count.is_none() && opts.structured_ssn_count.is_none() { return None; } - if !normalize::is_textual(data) { + if !normalize::is_textual(&data.window(0, normalize::SAMPLE)) { return None; } + let count_cards = || match data.as_slice() { + Some(d) => dlp::count_credit_cards(d), + None => dlp::count_credit_cards_in(&mut byte_source::Stepper::new(data)), + }; + let count_ssns = || match data.as_slice() { + Some(d) => dlp::count_ssns(d, dlp::SsnMode::Both), + None => dlp::count_ssns_in(&mut byte_source::Stepper::new(data), dlp::SsnMode::Both), + }; if let Some(threshold) = opts.structured_cc_count { - if dlp::count_credit_cards(data) >= threshold as usize { + if count_cards() >= threshold as usize { if let Some(o) = sink.hit( "Heuristics.Structured.CreditCardNumber".to_string(), 0, @@ -3742,7 +3417,7 @@ fn structured_data_scan(data: &[u8], opts: &ScanOptions, sink: &mut Sink) -> Opt } } if let Some(threshold) = opts.structured_ssn_count { - if dlp::count_ssns(data, dlp::SsnMode::Both) >= threshold as usize { + if count_ssns() >= threshold as usize { if let Some(o) = sink.hit( "Heuristics.Structured.SSN".to_string(), 0, @@ -3762,51 +3437,59 @@ fn structured_data_scan(data: &[u8], opts: &ScanOptions, sink: &mut Sink) -> Opt #[cfg(feature = "phishing")] fn phishing_scan( db: &Scanner, - data: &[u8], + data: &dyn ByteSource, opts: &ScanOptions, sink: &mut Sink, ) -> Option { if !opts.alert_phishing { return None; } - if !normalize::is_textual(data) { + if !normalize::is_textual(&data.window(0, normalize::SAMPLE)) { return None; } - let (found, complete) = phishing::scan_complete(data, &db.phishing); + let (found, complete) = phishing::scan_complete_source(data, &db.phishing); if !complete { engine::mark_scan_truncated(); } sink.hit(found?.signature().to_string(), 0, Method::Heuristic) } -/// Core detection over an in-memory buffer: the full `.ndb`/`.ldb` engine -/// (literals, wildcards, logical sigs — including EICAR), section hashes, then -/// whole-file hashes. The streaming literal automaton isn't used here; the -/// engine already covers every literal, so it stays unbuilt for file scans. -/// A core detection: clean signature name, match offset, method, and whether the -/// matched signature is from an unofficial database (so the report layer can add -/// `.UNOFFICIAL` in compat mode). The name is ALWAYS clean here. -type CoreHit = (String, u64, Method, bool); +/// All of a PE, for the checks that parse its structure (see [`whole`]). +fn pe_image<'a>( + ft: FileType, + data: &'a dyn ByteSource, + opts: &ScanOptions, +) -> Option> { + if ft == FileType::Pe { + whole(data, opts, "the PE layout, icon and section checks") + } else { + None + } +} -fn scan_bytes_core( - db: &Scanner, - data: &[u8], - filename: Option<&str>, - suffix: bool, -) -> Option { - // Total bytes of bytecode-extracted (unpacked) content this scan may - // re-scan, across the whole recursion — bounds an extraction bomb where a - // (trusted) unpacker, driven by a hostile input, emits many/large buffers. - let mut extract_budget = MAX_BC_EXTRACT_TOTAL; - scan_bytes_depth( - db, - data, - 0, - &mut extract_budget, - None, - None, - filename, - suffix, +/// All of `data`, for a check that parses an object whole: the object itself +/// when it is held in memory, or read into memory when it is at most +/// `deep_analysis_max`. `None` otherwise, and the scan is then reported as +/// over the size limit: `what` applied and did not run. +fn whole<'a>( + data: &'a dyn ByteSource, + opts: &ScanOptions, + what: &str, +) -> Option> { + let whole = data.materialize(opts.deep_analysis_max as usize); + if whole.is_none() { + engine::mark_over_size(|| over_size(data.len(), opts, &format!("{what} did not run"))); + } + whole +} + +/// Why something was skipped on an object of `len` bytes: it is over the +/// deep-analysis limit, and `consequence`. +fn over_size(len: usize, opts: &ScanOptions, consequence: &str) -> String { + format!( + "object is {len} bytes, over the {}-byte deep-analysis limit \ + (--max-object-bytes): {consequence}", + opts.deep_analysis_max ) } @@ -3851,39 +3534,110 @@ fn mostly_text(data: &[u8]) -> bool { printable * 100 >= sample.len() * 90 } -/// True if `data` looks like JavaScript / generic script and is worth running -/// the JS normaliser over (in addition to the HTML/text normalisers). Covers -/// shebang scripts and HTML (`FileType::Script` / `FileType::Html`), any text -/// embedding a `(ft: FileType, data: &'a [u8]) -> Vec Vec + 'a>> { - let mut v: Vec Vec + 'a>> = vec![ - Box::new(move || normalize::html(data)), - Box::new(move || normalize::text(data)), - ]; - if looks_like_script(ft, data) { - v.push(Box::new(move || normalize::javascript(data))); - v.push(Box::new(move || { - let (view, cut) = jsnorm::normalize(data); - // The raw bytes are still scanned in full; only this view is short. - if cut { - engine::mark_scan_truncated(); - } - view - })); +/// inside them. Making one at a time keeps the peak at one copy without +/// changing which views are scanned or in what order. +fn normalizations(ft: FileType, data: &dyn ByteSource) -> Vec { + let mut v = vec![ViewKind::Html, ViewKind::Text]; + if looks_like_script(ft, &data.window(0, 8192)) { + v.push(ViewKind::Javascript); + v.push(ViewKind::Deobfuscated); } v } +/// A normalised view of an object: [`normalize`]'s three, and [`jsnorm`]'s. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ViewKind { + Html, + Text, + Javascript, + Deobfuscated, +} + +/// A normalised view, held in memory or read back from a spill file. +enum View { + Mem(Vec), + Spilled(spill::Spilled), +} + +impl View { + fn source(&self) -> &dyn ByteSource { + match self { + View::Mem(v) => v, + View::Spilled(s) => s, + } + } +} + +/// `kind`'s view of `data`: made in memory when `data` is held there, else +/// written to the host's spill. `None` when it cannot be made, and the scan is +/// then marked incomplete. +fn make_view(kind: ViewKind, data: &dyn ByteSource, opts: &ScanOptions) -> Option { + if kind == ViewKind::Deobfuscated { + // Bounded by its own cap, so held in memory either way. + let (view, cut) = jsnorm::normalize_source(data); + // The raw bytes are still scanned in full; only this view is short. + if cut { + engine::mark_scan_truncated(); + } + return Some(View::Mem(view)); + } + if let Some(d) = data.as_slice() { + return Some(View::Mem(match kind { + ViewKind::Html => normalize::html(d), + ViewKind::Text => normalize::text(d), + _ => normalize::javascript(d), + })); + } + let not_made = |why: &str| { + engine::mark_over_size(|| { + over_size( + data.len(), + opts, + &format!("its normalised text views were not scanned: {why}"), + ) + }) + }; + let file = match &opts.spill { + Some(spill) => spill.create(), + None => Err("spilling to disk is off".to_string()), + }; + let file = match file { + Ok(file) => file, + Err(e) => { + not_made(&e); + return None; + } + }; + let mut out = spill::SpillOut::new(file); + let mut src = byte_source::Stepper::new(data); + match kind { + ViewKind::Html => normalize::html_into(&mut src, &mut out), + ViewKind::Text => normalize::text_into(&mut src, &mut out), + _ => normalize::javascript_into(&mut src, &mut out), + } + match out.finish() { + Ok(spilled) => Some(View::Spilled(spilled)), + Err(e) => { + not_made(&e); + None + } + } +} + +/// True if `data` looks like JavaScript / generic script and is worth running +/// the JS normaliser over (in addition to the HTML/text normalisers). Covers +/// shebang scripts and HTML (`FileType::Script` / `FileType::Html`), any text +/// embedding a ` bool { if matches!(ft, FileType::Script | FileType::Html) { return true; @@ -3898,443 +3652,198 @@ fn looks_like_script(ft: FileType, data: &[u8]) -> bool { contains_ci(b", - container: Option, - suffix: bool, -) -> Option { - let mut extract_budget = MAX_BC_EXTRACT_TOTAL; - // A container member has its own identity (not the outer file's), which exav - // does not track, so YARA filename externals stay undefined for members. - scan_bytes_depth( - db, - data, - 0, - &mut extract_budget, - ft_override, - container, - None, - suffix, - ) -} - -/// Run the matching core over one buffer and report every hit to `sink`. -/// -/// The all-match half of [`scan_bytes_member`]: the same pattern / section-hash -/// / whole-file-hash / bytecode passes, but gathering all matches instead of -/// returning the first. Only this function branches on `Sink::wants_all` — the -/// traversal above it does not, which is what the split buys: one walk, and a -/// single place where first-match and all-match differ. -/// -/// Returns the buffers a bytecode unpacker produced, for the caller to recurse -/// into; they are content that exists nowhere else. -fn core_scan_all( - db: &Scanner, - data: &[u8], - ft: FileType, - container: Option, - unofficial_suffix: bool, - sink: &mut Sink, -) -> Vec> { - let layout = if ft == FileType::Pe { - pe::layout(data) - } else { - None - }; - let icon_metrics = if ft == FileType::Pe && !db.icons.is_empty() { - icon::pe_icon_metrics(data) - } else { - Vec::new() - }; - let icon_ctx = engine::IconCtx::new(&db.icons, &icon_metrics); - let mut eng = Vec::new(); - db.engine.scan_all_with_icons( - data, - ft, - layout.as_ref(), - container, - Some(&icon_ctx), - &mut eng, - ); - // `is_textual_type(ft)` cannot be dropped, and `scan_bytes_depth` carries - // the same guard: `normalize::is_textual` counts 0x80..=0xff as text and most - // packed binaries carry few NULs, so a compressed PE — or an APK's DEX and - // `.so` members — look textual and paid for two to four extra FULL engine - // passes. A `Target:0` pattern already ran against the raw bytes, and - // `target_ok` confines Target:3/4/7 to text-ish types, so for a - // positively-typed binary those passes cannot match. They were pure cost. - if !db.engine.is_empty() && is_textual_type(ft) && normalize::is_textual(data) { - // One at a time. Each normalisation is a full-size copy of `data`, and - // building the set before scanning any of it held two — four for a - // script — alive at once, on top of the buffer they were made from. - // Nesting multiplies that: a container, its member and that member's - // member are each mid-scan while the walk is inside them. - for norm in normalizations(ft, data) { - db.engine - .scan_all_with_layout(&norm(), ft, None, container, &mut eng); - } - } - for (name, off, unofficial) in eng { - sink.hit( - report_name(&name, unofficial, unofficial_suffix), - off, - Method::Pattern, - ); - } - if !db.sections.is_empty() && ft == FileType::Pe { - let want_sha = db.sections.wants_sha(); - for (size, slice) in pe::section_slices(data) { - let d = hashes::section_digests(slice, want_sha); - if let Some((name, unofficial)) = db.sections.lookup(size, &d) { - sink.hit( - report_name(&name, unofficial, unofficial_suffix), - 0, - Method::Hash, - ); - } - } - } - if !db.hashes.is_empty() && hash_matchable(data.len() as u64) { - if let Some((name, unofficial)) = db.hashes.lookup(&digests_of(data), data.len() as u64) { - sink.hit( - report_name(&name, unofficial, unofficial_suffix), - 0, - Method::Hash, - ); - } - } - // YARA rules, in the same position the first-match core runs them. Omitting - // them here would make all-match see LESS than a normal scan — a file whose - // only detection is a YARA rule would come back with an empty list and a - // `Complete` outcome, which renders as OK. - // - // `None` for the filename matches `scan_bytes_member`: a member has its own - // identity rather than the outer file's, and exav does not track it, so - // filename externals stay undefined on this path in both cores. - if !db.yara.is_empty() { - let unofficial = db.yara.unofficial(); - for name in profile::timed("yara", data.len() as u64, || db.yara.scan(data, None)) { - sink.hit( - report_name(&name, unofficial, unofficial_suffix), - 0, - Method::Yara, - ); - } - } - // Bytecode names are reported verbatim, never `.UNOFFICIAL`-suffixed. - if !db.bytecode.is_empty() { - let (det, extracted) = db.bytecode.scan(data, ft, layout.as_ref()); - if let Some((name, _)) = det { - sink.hit(name, 0, Method::Bytecode); - } - return extracted; - } - Vec::new() + || contains_ci(b"unescape(") + || contains_ci(b"decodeuricomponent") + || contains_ci(b"eval(") + || contains_ci(b"function(") + || contains_ci(b"function (") + // Common malicious JScript/VBScript dropper markers (the latter rarely + // carry an `eval`/`function` trigger). Gating only decides whether the + // normalised script buffer is produced; it can never cause a match. + || contains_ci(b"activexobject") + || contains_ci(b"createobject") + || contains_ci(b"wscript") } -/// The matching core over one buffer, in whichever mode the sink asks for. -/// -/// The single place first-match and all-match differ. The traversal above never -/// branches on the mode; it calls this wherever a buffer needs scanning. -/// -/// `Some(outcome)` means the walk must stop and return it. -fn core_scan( +/// What the matching core leaves for the rest of an object's scan. +struct CoreRest { + /// The buffers a bytecode unpacker made of the object: content that exists + /// nowhere else, to scan as objects of their own. + unpacked: Vec>, + /// The type a `HandlerType:` signature gave the object, if one matched. + retype: Option, +} + +/// The matching core over an object's own bytes, `ft` being the type it is +/// matched as. Every detection goes to `sink`; `Err` means the walk must stop. +fn core( db: &Scanner, - data: &[u8], - ft_override: Option, - container: Option, + facts: &Facts, + ft: FileType, + obj: Obj<'_>, opts: &ScanOptions, sink: &mut Sink, -) -> Option { - if sink.wants_all() { - let ft = ft_override.unwrap_or_else(|| db.identify(data)); - // Buffers a bytecode unpacker produced are content that exists nowhere - // else. The first-match core recurses into them itself (see - // `scan_bytes_depth`), so the all-match core has to as well or a payload - // that only exists after unpacking is missed. - for buf in core_scan_all(db, data, ft, container, opts.unofficial_suffix, sink) { - let bft = db.identify(&buf); - core_scan_all(db, &buf, bft, container, opts.unofficial_suffix, sink); - } - return None; - } - let (sig, off, m, unofficial) = - scan_bytes_member(db, data, ft_override, container, opts.unofficial_suffix)?; - sink.hit( - report_name(&sig, unofficial, opts.unofficial_suffix), - off, - m, - ) +) -> Result { + let data = facts.data; + let unpacked = core_matches(db, facts, ft, obj, opts, sink); + // Taken whatever the outcome, so it cannot outlive this object and retype + // a later one at the same address. + let retype = engine::take_retype_source(data); + unpacked.map(|unpacked| CoreRest { unpacked, retype }) } -/// Max recursion into bytecode-extracted (unpacked) buffers, to bound -/// extraction bombs. -const MAX_BC_DEPTH: u32 = 4; -/// Cap on total bytes re-scanned from bytecode-extracted buffers per scan. -const MAX_BC_EXTRACT_TOTAL: u64 = 256 * 1024 * 1024; - -#[allow(clippy::too_many_arguments)] -fn scan_bytes_depth( +/// [`core`]'s matching: the signature engine over the object's bytes and their +/// normalised views, bytecode, PE section hashes, the whole-file hash, YARA. +/// +/// The single place a normal scan and `--all-matches` differ: the engine is +/// asked for the first match, or for all of them. The traversal above never +/// branches on the mode. +fn core_matches( db: &Scanner, - data: &[u8], - depth: u32, - extract_budget: &mut u64, - ft_override: Option, - container: Option, - filename: Option<&str>, - suffix: bool, -) -> Option { + facts: &Facts, + ft: FileType, + obj: Obj<'_>, + opts: &ScanOptions, + sink: &mut Sink, +) -> Result>, DeepOutcome> { + let data = facts.data; + let suffix = opts.unofficial_suffix; + let materialize = opts.deep_analysis_max as usize; + let all = sink.wants_all(); + // The engine passes over an ignored (`.ign`/`.ign2`) match to look for the + // next one; every other matcher's hits are ignored by the sink. let skip = |name: &str, unofficial: bool| is_ignored(db, name, unofficial, suffix); - let ft = if ft_override.is_some() { - ft_override - } else if !db.engine.is_empty() || !db.sections.is_empty() || !db.bytecode.is_empty() { - Some(db.identify(data)) - } else { - None + let mut hit = |name: &str, unofficial: bool, offset: u64, method: Method| match sink.hit( + report_name(name, unofficial, suffix), + offset, + method, + ) { + Some(stop) => Err(stop), + None => Ok(()), }; - if let Some(ft) = ft { - // PE layout lets the engine resolve EP/section-relative offsets. - let layout = if ft == FileType::Pe { - pe::layout(data) - } else { - None - }; - // PE-icon perceptual metrics for `IconGroup1/2`-constrained logical - // signatures: computed once per scan, only for a PE when `.idb` entries - // are loaded. They gate such sigs (the structural condition must also - // hold) — see [`icon`] and the engine's `IconCtx`. - let icon_metrics = if ft == FileType::Pe && !db.icons.is_empty() { - profile::timed("icon", data.len() as u64, || icon::pe_icon_metrics(data)) - } else { - Vec::new() - }; - let icon_ctx = engine::IconCtx::new(&db.icons, &icon_metrics); - if let Some((name, off, unofficial)) = profile::timed("engine", data.len() as u64, || { - db.engine - .scan_first(data, ft, layout.as_ref(), container, Some(&icon_ctx), &skip) - }) { - return Some((name, off, Method::Pattern, unofficial)); - } - // Normalised-content pass: HTML/text/mail (`Target:3/4/7`) signatures - // are written against canonicalised content, not raw bytes. Run the - // engine over normalised variants of textual input. - // Normalisation is only worth doing when a signature could match the - // result. `target_ok` already confines Target:3/4/7 (HTML/mail/text) - // signatures to text-ish types, and a Target:0 byte pattern has already - // been run against the raw bytes by the pass above — so for a - // positively-typed binary (PE, ELF, Mach-O, a container…) the two - // normalised passes are pure cost. They were running anyway, because - // `is_textual` counts 0x80..=0xff as text and most packed binaries carry - // few NULs, so a compressed PE looked textual and paid for two extra - // full passes over its bytes. - if !db.engine.is_empty() && is_textual_type(ft) && normalize::is_textual(data) { - // Built and dropped one at a time — see the note on the same loop - // in `scan_all_with_layout`'s caller. Holding the whole set was - // two to four full-size copies of `data` alive simultaneously. - for norm in normalizations(ft, data) { - let norm = profile::timed("normalize", data.len() as u64, norm); - if let Some((name, off, unofficial)) = - profile::timed("engine", norm.len() as u64, || { - db.engine - .scan_first(&norm, ft, None, container, None, &skip) - }) - { - return Some((name, off, Method::Pattern, unofficial)); - } + // A PE's structure (layout, icons, section table) is parsed from all of it. + let image = pe_image(ft, data, opts); + // PE layout lets the engine resolve EP/section-relative offsets. + let layout = image.as_deref().and_then(pe::layout); + // PE-icon perceptual metrics for `IconGroup1/2`-constrained logical + // signatures: computed only for a PE when `.idb` entries are loaded. They + // gate such sigs (the structural condition must also hold); see [`icon`]. + let icon_metrics = match &image { + Some(image) if !db.icons.is_empty() => { + profile::timed("icon", data.len() as u64, || icon::pe_icon_metrics(image)) + } + _ => Vec::new(), + }; + let icon_ctx = engine::IconCtx::new(&db.icons, &icon_metrics); + // The bytecode triggers that fire on the raw bytes, found in the same + // sweep as every other signature. + let mut fired = Vec::new(); + let mut engine_pass = |src: &dyn ByteSource, + layout: Option<&pe::PeLayout>, + icons: Option<&engine::IconCtx>, + fired: Option<&mut Vec>| + -> Result<(), DeepOutcome> { + let mut found = Vec::new(); + profile::timed("engine", src.len() as u64, || { + let c = obj.container; + if all { + db.engine.scan_all_source( + src, + ft, + layout, + c, + icons, + &mut found, + materialize, + fired, + ); + } else { + found.extend(db.engine.scan_first_source( + src, + ft, + layout, + c, + icons, + &skip, + materialize, + fired, + )); } + }); + for (name, off, unofficial) in found { + hit(&name, unofficial, off, Method::Pattern)?; + } + Ok(()) + }; + engine_pass(data, layout.as_ref(), Some(&icon_ctx), Some(&mut fired))?; + // Normalised-content passes: HTML/text/mail (`Target:3/4/7`) signatures + // are written against canonicalised content, not raw bytes. Only worth + // making when a signature could match the result: `target_ok` confines + // Target:3/4/7 to text-ish types, and a Target:0 pattern has already run + // against the raw bytes, so for a positively-typed binary they are pure + // cost. `is_textual` alone does not say so: it counts 0x80..=0xff as text, + // and most packed binaries carry few NULs. + if !db.engine.is_empty() + && is_textual_type(ft) + && normalize::is_textual(&data.window(0, normalize::SAMPLE)) + { + // One at a time: each is a full-size copy of `data`, and nesting + // stacks them (a container, its member and that member's member are + // each mid-scan while the walk is inside them). + for kind in normalizations(ft, data) { + let Some(view) = profile::timed("normalize", data.len() as u64, || { + make_view(kind, data, opts) + }) else { + continue; + }; + engine_pass(view.source(), None, None, None)?; } - // Bytecode programs whose trigger/hook fires (gated execution). Bytecode - // detection names are reported verbatim (not `.UNOFFICIAL`-suffixed). + } + // Bytecode programs whose trigger/hook fires (gated execution). Their + // names are reported verbatim, never `.UNOFFICIAL`-suffixed. + let mut unpacked = Vec::new(); + if !db.bytecode.is_empty() { let (det, extracted) = profile::timed("bytecode", data.len() as u64, || { - db.bytecode.scan(data, ft, layout.as_ref()) + db.bytecode.scan_source(data, ft, materialize, &fired) }); - if let Some((name, _idx)) = det.filter(|(name, _)| !skip(name, false)) { - return Some((name, 0, Method::Bytecode, false)); - } - // Unpacker programs surface embedded files; re-scan them recursively - // (bounded) so the payload inside a packed binary is caught. - if depth < MAX_BC_DEPTH { - for (bcix, buf) in extracted.into_iter().enumerate() { - let cost = buf.len() as u64; - if *extract_budget < cost { - break; // extraction-bomb guard: stop re-scanning further - } - *extract_budget -= cost; - // Record where this buffer came from. Every OTHER extraction site - // pushes a guard, and this one did not — so a detection inside a - // bytecode-unpacked buffer was reported with no location at all, - // i.e. as though it had matched the container's own bytes. That - // is how four correct RTF detections came to look like - // `Target:1` (PE) signatures firing on an RTF file, which reads - // exactly like a target-gating bug and is not one. The unpacker - // has no member name to offer, so the index is the identity. - let _mpg = MatchPathGuard::enter(&format!("bytecode-unpacked-{}", bcix + 1)); - if let Some(hit) = scan_bytes_depth( - db, - &buf, - depth + 1, - extract_budget, - None, - container, - None, - suffix, - ) { - match_loc_record(); - return Some(hit); - } - // A packer may surface an embedded archive (e.g. an unpacked - // payload that is itself a ZIP/gzip). Unpack it too, bounded by - // the same extraction budget, so the real payload is reached. - let bft = profile::timed("filetype", buf.len() as u64, || filetype::identify(&buf)); - // This deep bytecode-extracted-buffer rescan runs only behind a - // ClamAV `.cbc` unpacker firing; it isn't part of the common - // diff-tested path, so it always runs at full capability. - if let Some(fmt) = unpack_target(bft, &buf, false) { - let inner_container = container_cltype(fmt, &buf); - let _ag = engine::AncestryGuard::enter(inner_container); - let mut b = Budget::new(unpack::Limits::default()); - // Stream members; `Some(Some(hit))` stops with a detection, - // `Some(None)` stops on the extraction-bomb guard, `None` - // continues. Extraction errors are ignored (best effort). - let res = unpack::extract_each( - fmt, - &buf, - &mut b, - &mut |e: unpack::Entry, _b: &mut Budget| { - let ec = e.data.len() as u64; - if *extract_budget < ec { - return Some(None); - } - *extract_budget -= ec; - // Same reason as above: this member has a real name, - // so a hit here can be attributed precisely. - let _mpg = MatchPathGuard::enter(&e.name); - let hit = scan_bytes_depth( - db, - &e.data, - depth + 1, - extract_budget, - None, - inner_container, - None, - suffix, - ); - if hit.is_some() { - match_loc_record(); - } - hit.map(Some) - }, - ); - if let Ok(Some(Some(hit))) = res { - return Some(hit); - } - } - } + if let Some((name, _)) = det { + hit(&name, false, 0, Method::Bytecode)?; } + unpacked = extracted; } - if !db.sections.is_empty() && ft == Some(FileType::Pe) { - let want_sha = db.sections.wants_sha(); - if let Some((name, unofficial)) = profile::timed("sections", data.len() as u64, || { - for (size, slice) in pe::section_slices(data) { - let d = hashes::section_digests(slice, want_sha); - if let Some(hit) = db.sections.lookup(size, &d).filter(|(n, u)| !skip(n, *u)) { - return Some(hit); - } + if !db.sections.is_empty() { + if let Some(image) = &image { + let want_sha = db.sections.wants_sha(); + let found: Vec<_> = profile::timed("sections", data.len() as u64, || { + pe::section_slices(image) + .into_iter() + .filter_map(|(size, slice)| { + db.sections + .lookup(size, &hashes::section_digests(slice, want_sha)) + }) + .collect() + }); + for (name, unofficial) in found { + hit(&name, unofficial, 0, Method::Hash)?; } - None - }) { - return Some((name, 0, Method::Hash, unofficial)); } } if !db.hashes.is_empty() && hash_matchable(data.len() as u64) { if let Some((name, unofficial)) = profile::timed("hashes", data.len() as u64, || { - db.hashes.lookup(&digests_of(data), data.len() as u64) - }) - .filter(|(n, u)| !skip(n, *u)) - { - return Some((name, 0, Method::Hash, unofficial)); + db.hashes.lookup(facts.digests(db), data.len() as u64) + }) { + hit(&name, unofficial, 0, Method::Hash)?; } } if !db.yara.is_empty() { let unofficial = db.yara.unofficial(); - if let Some(name) = - profile::timed("yara", data.len() as u64, || db.yara.scan(data, filename)) - .into_iter() - .find(|n| !skip(n, unofficial)) - { - return Some((name, 0, Method::Yara, unofficial)); - } - } - None -} - -/// Streaming detection core (constant memory, any size): Aho-Corasick + -/// triple hasher in one forward pass. -/// -/// `suffix` is [`ScanOptions::unofficial_suffix`], which decides the name an -/// ignore-list entry has to match. -fn stream_core(db: &Scanner, reader: R, suffix: bool) -> io::Result> { - // The first match that is not on the ignore list. Running the iterator to - // its end consumes the whole stream, so the hashers below saw every byte. - let first_pattern = |reader: &mut dyn Read| -> io::Result> { - for mat in db.patterns.ac().stream_find_iter(reader) { - let mat = mat?; - let (name, unofficial) = db.patterns.name_prov(mat.pattern().as_usize()); - if is_ignored(db, name, unofficial, suffix) { - continue; - } - return Ok(Some(( - name.to_string(), - mat.start() as u64, - Method::Pattern, - unofficial, - ))); - } - Ok(None) - }; - // When no hash signatures are loaded, skip the (expensive) triple hash on - // every byte and just run the pattern matcher over the raw reader. - if db.hashes.is_empty() { - let mut reader = reader; - return first_pattern(&mut reader); - } - let mut tee = TeeHasher::new(reader); - if let Some(hit) = first_pattern(&mut tee)? { - return Ok(Some(hit)); - } - let size = tee.bytes_read(); - let digests = tee.finalize(); - if hash_matchable(size) { - if let Some((name, unofficial)) = db - .hashes - .lookup(&digests, size) - .filter(|(n, u)| !is_ignored(db, n, *u, suffix)) - { - return Ok(Some((name, 0, Method::Hash, unofficial))); + for name in profile::timed("yara", data.len() as u64, || { + db.yara.scan_source(data, obj.filename, materialize) + }) { + hit(&name, unofficial, 0, Method::Yara)?; } } - Ok(None) + Ok(unpacked) } #[cfg(test)] @@ -4343,26 +3852,262 @@ mod tests { use patterns::eicar; use std::io::Cursor; + /// A reader that counts the bytes read through it. + struct Counting( + Cursor>, + std::sync::Arc, + ); + + impl std::io::Read for Counting { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.0.read(buf)?; + self.1 + .fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed); + Ok(n) + } + } + + impl std::io::Seek for Counting { + fn seek(&mut self, p: std::io::SeekFrom) -> std::io::Result { + self.0.seek(p) + } + } + + /// How many times over a scan by `db` reads `data` from a source it does + /// not hold in memory. + fn passes_over(db: &Scanner, data: &[u8]) -> f64 { + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::Arc; + let read = Arc::new(AtomicU64::new(0)); + let reader = Counting(Cursor::new(data.to_vec()), read.clone()); + let opts = ScanOptions { + deep_analysis_max: 1 << 20, + ..ScanOptions::default() + }; + scan_seekable(db, reader, data.len() as u64, &opts).unwrap(); + read.load(Ordering::Relaxed) as f64 / data.len() as f64 + } + + /// Every full read of an object not held in memory is counted, per kind of + /// object and of signature set. Each is a read of the source, for an HTTP + /// one a download, so a change that adds one fails here: raising a bound + /// needs the maintainer's approval, not a new expectation. + /// + /// What each read is: one for detection's search for its markers (and + /// for an executable, a self-extractor's payload), carving, and the + /// digests when a hash signature has the object's size; one for the + /// signature sweep; and one at most for a signature with an unbounded + /// gap, whose literal after it is searched through the rest of the object + /// once. Each bound allows half a read more, for window seams and headers + /// read again. + #[test] + fn a_streamed_object_is_read_a_few_times() { + let size = 12 << 20; + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let mut noise: Vec = (0..size) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 24) as u8 + }) + .collect(); + // No magic a carver or a format check would stop at by chance. + for w in 0..noise.len() - 1 { + if matches!( + &noise[w..w + 2], + b"MZ" | b"PK" | b"7z" | b"BZ" | b"\x1f\x8b" | b"Ra" | b"MS" + ) { + noise[w] ^= 0x80; + } + } + let db = |hash_size: usize| { + let mut l = loader::Builder::new(); + l.add_named_bytes("a.ndb", b"Sig.Pat:0:*:0badc0de0badf00d0bad\n", true); + l.add_named_bytes( + "b.ldb", + b"Sig.Gap;Engine:51-255,Target:0;0&1;0badf00d;0badc0de*d00d\n", + true, + ); + let hdb = format!("0123456789abcdef0123456789abcdef:{hash_size}:Sig.Hash\n"); + l.add_named_bytes("c.hdb", hdb.as_bytes(), true); + let hsb = format!("{}:{hash_size}:Sig.Sha\n", "ab".repeat(32)); + l.add_named_bytes("d.hsb", hsb.as_bytes(), true); + let fp = format!("fedcba9876543210fedcba9876543210:{hash_size}:Some.Good\n"); + l.add_named_bytes("e.fp", fp.as_bytes(), true); + l.build().unwrap() + }; + let (other_size, this_size) = (db(12345), db(size)); + let mut pe = noise.clone(); + pe[..2].copy_from_slice(b"MZ"); + let mut gap = noise.clone(); + gap[1000..1004].copy_from_slice(&[0x0b, 0xad, 0xc0, 0xde]); + let cases: [(&str, &Scanner, &[u8], f64); 5] = [ + ("unknown", &other_size, &noise, 2.0), + ("PE", &other_size, &pe, 2.0), + ("unknown, hashed", &this_size, &noise, 2.0), + ("PE, hashed", &this_size, &pe, 2.0), + ("unknown, a gap to the end", &other_size, &gap, 3.0), + ]; + for (what, db, data, bound) in cases { + let passes = passes_over(db, data); + assert!( + passes <= bound + 0.5, + "{what}: read {passes:.2} times over, the bound is {bound}" + ); + } + } + + /// The one read of a streamed object finds what each step's own read of + /// it would, whichever step asks first. + #[test] + fn one_read_serves_every_step() { + use byte_source::CHUNK; + // Longer than the start detection reads, so it searches the rest. + let mut data = vec![b'.'; (5 << 20) + 77]; + let end = data.len(); + let mut pe = b"MZ".to_vec(); + pe.resize(0x3c, 0); + pe.extend_from_slice(&0x40u32.to_le_bytes()); + pe.extend_from_slice(b"PE\0\0\x4c\x01\x03\0"); + pe.resize(0x40 + 20, 0); + pe.extend_from_slice(&0xe0u16.to_le_bytes()); + pe.extend_from_slice(&[0, 0, 0x0b, 0x01]); + let mut elf = b"\x7fELF\x02\x01\x01".to_vec(); + elf.resize(16, 0); + elf.extend_from_slice(&2u16.to_le_bytes()); + let mut put = |at: usize, bytes: &[u8]| data[at..at + bytes.len()].copy_from_slice(bytes); + put(0, &pe); + put(CHUNK - 1, &pe); + put(2 * CHUNK - 3, &elf); + put(3 * CHUNK - 5, b"7z\xbc\xaf\x27\x1c"); + put(4 * CHUNK - 2, b"PK\x03\x04"); + put(end - 3 * CHUNK - 3, b"AU3!EA06"); + let size = data.len(); + let mut l = loader::Builder::new(); + let hdb = format!("0123456789abcdef0123456789abcdef:{size}:Sig.Hash\n"); + l.add_named_bytes("a.hdb", hdb.as_bytes(), true); + let hsb = format!("{}:{size}:Sig.Sha\n", "ab".repeat(32)); + l.add_named_bytes("b.hsb", hsb.as_bytes(), true); + let db = l.build().unwrap(); + + let slice: &[u8] = &data; + let want_fmt = unpack::detect(&slice); + let want_embedded = pe::embedded_in(&slice); + let want_digests = hashes::digests_wanted(&slice, digests_wanted_by(&db, size as u64)); + // Found by the search past detection's start, in a build that has it. + #[cfg(feature = "all-formats")] + assert_eq!(want_fmt, Some(unpack::Format::Autoit)); + assert!(!want_embedded.pe.is_empty() && !want_embedded.archives.is_empty()); + assert!(!want_digests.md5.is_empty() && !want_digests.sha256.is_empty()); + for first in 0..3 { + let cache = + byte_source::BlockCache::with_sizes(Cursor::new(data.clone()), 4093, 16 * 4093) + .unwrap(); + let facts = Facts::for_scan(&cache, &db, true); + let mut embedded = None; + for step in (0..3).map(|k| (first + k) % 3) { + match step { + 0 => assert_eq!(facts.format(), want_fmt, "first {first}"), + 1 => embedded = Some(facts.embedded()), + _ => { + let d = facts.digests(&db); + let w = &want_digests; + assert_eq!( + (&d.md5, &d.sha1, &d.sha256), + (&w.md5, &w.sha1, &w.sha256), + "first {first}" + ); + } + } + } + let got = embedded.unwrap(); + assert_eq!( + (got.pe, got.elf, got.macho, got.archives), + ( + want_embedded.pe.clone(), + want_embedded.elf.clone(), + want_embedded.macho.clone(), + want_embedded.archives.clone() + ), + "first {first}" + ); + } + } + + /// A step that needs no read of a streamed object does not start the one + /// read for the others: no digest wanted at its size, or an object + /// detection holds whole in the start it reads anyway. + #[test] + fn a_step_that_needs_no_read_makes_none() { + use std::sync::atomic::{AtomicU64, Ordering}; + // An allowlist entry at another size: the allowlist is asked, and + // wants no digest. + let mut l = loader::Builder::new(); + l.add_named_bytes( + "a.fp", + b"fedcba9876543210fedcba9876543210:12345:Some.Good\n", + true, + ); + let db = l.build().unwrap(); + for len in [1 << 20, 5 << 20] { + let read = std::sync::Arc::new(AtomicU64::new(0)); + let reader = Counting(Cursor::new(vec![b'.'; len]), read.clone()); + let cache = byte_source::BlockCache::with_sizes(reader, 4093, 16 * 4093).unwrap(); + let facts = Facts::for_scan(&cache, &db, false); + let before = read.load(Ordering::Relaxed); + assert!(!allowlisted(&db, &facts)); + assert_eq!( + read.load(Ordering::Relaxed), + before, + "{len}: the allowlist read it" + ); + facts.format(); + let passes = (read.load(Ordering::Relaxed) - before) as f64 / len as f64; + // Its start, and past that start a search through all of it. + let bound = if unpack::Prescan::needed(len) { + 2.0 + } else { + 1.0 + }; + assert!( + passes <= bound + 0.1, + "{len}: detection read it {passes:.2} times over" + ); + } + } + + /// [`super::unpack_target`] of an object held in memory. + fn unpack_target( + ft: FileType, + data: &[u8], + restrict: bool, + opts: &ScanOptions, + ) -> Option { + super::unpack_target(ft, &Facts::new(&data), restrict, opts) + } + #[test] fn restrict_extractors_gates_absent_formats() { // Of exav's extractors, `ar` and `lzip` are absent from stock ClamAV; // cpio/xar are supported by both, so the compat mask gates the absent - // formats alone — gating cpio/xar would make exav miss detections + // formats alone: gating cpio/xar would make exav miss detections // ClamAV makes. - // `ar` — restricted only when `restrict` is set. + // `ar`: restricted only when `restrict` is set. + let opts = ScanOptions::default(); assert_eq!( - unpack_target(FileType::Ar, b"", false), + unpack_target(FileType::Ar, b"", false, &opts), Some(unpack::Format::Ar) ); - assert_eq!(unpack_target(FileType::Ar, b"", true), None); - // cpio / xar — supported by ClamAV, so never gated. + assert_eq!(unpack_target(FileType::Ar, b"", true, &opts), None); + // cpio / xar: supported by ClamAV, so never gated. for (ft, fmt) in [ (FileType::Cpio, unpack::Format::Cpio), (FileType::Xar, unpack::Format::Xar), ] { - assert_eq!(unpack_target(ft, b"", false), Some(fmt)); + assert_eq!(unpack_target(ft, b"", false, &opts), Some(fmt)); assert_eq!( - unpack_target(ft, b"", true), + unpack_target(ft, b"", true, &opts), Some(fmt), "{ft:?} must not be gated" ); @@ -4381,27 +4126,31 @@ mod tests { b } - /// Compat restricts UPX unpacking to PE — ClamAV's UPX unpacker runs only + /// Compat restricts UPX unpacking to PE: ClamAV's UPX unpacker runs only /// from its PE path, so exav's ELF/Mach-O UPX reach (e.g. UPX-packed Mirai /// ELFs) must be gated off under `restrict` to stay apples-to-apples. #[cfg(feature = "upx")] #[test] fn restrict_scopes_upx_to_pe() { let buf = fake_upx(); + let opts = ScanOptions::default(); assert!(unpack::is_upx(&buf), "test buffer must look like UPX"); // Normal mode: UPX unpacked for every executable type. for ft in [FileType::Pe, FileType::Elf, FileType::MachO] { - assert_eq!(unpack_target(ft, &buf, false), Some(unpack::Format::Upx)); + assert_eq!( + unpack_target(ft, &buf, false, &opts), + Some(unpack::Format::Upx) + ); } // Compat: PE stays on, ELF/Mach-O are gated off. assert_eq!( - unpack_target(FileType::Pe, &buf, true), + unpack_target(FileType::Pe, &buf, true, &opts), Some(unpack::Format::Upx), "PE-UPX must stay on under compat (ClamAV does it too)" ); for ft in [FileType::Elf, FileType::MachO] { assert_eq!( - unpack_target(ft, &buf, true), + unpack_target(ft, &buf, true, &opts), None, "{ft:?}-UPX must be gated off under compat" ); @@ -4624,9 +4373,27 @@ mod tests { assert_eq!(infected_as(&on_disk), Some("Test.ZipName")); } + /// A detection in a member stored in an archive is the member's, at any + /// depth: the members are scanned before the container's own bytes, which + /// hold the same bytes stored. + #[test] + #[cfg(any(feature = "zip", feature = "all-formats"))] + fn a_stored_member_is_attributed_to_itself_at_any_depth() { + let stored = |name: &str, body: &[u8]| zip_bytes(&[(name, body, true)]); + let inner = stored("eicar.txt", eicar()); + let outer = stored("inner.zip", &inner); + let db = Scanner::builtin(); + let opts = ScanOptions::default(); + for (data, want) in [(&inner, "eicar.txt"), (&outer, "inner.zip/eicar.txt")] { + let (r, loc) = + scan_seekable_located(&db, Cursor::new(data), data.len() as u64, &opts).unwrap(); + assert!(matches!(r.verdict, Verdict::Infected { .. }), "{r:?}"); + assert_eq!(loc.as_deref(), Some(want)); + } + } + /// An ignored signature is passed over, not stopped at: a real detection in - /// a later member must still be found, through the buffered and the - /// streamed walk alike. + /// a later member must still be found, in memory and through a reader alike. #[test] #[cfg(any(feature = "zip", feature = "all-formats"))] fn an_ignored_signature_does_not_hide_a_later_member() { @@ -4676,11 +4443,12 @@ mod tests { } } - /// A file too large for the full engine was matched by literal patterns and - /// hashes alone. That is not a complete scan, so it is not clean, while a - /// hit the literal pass does find still wins. + /// A file too large to hold gets the full engine, read through a block + /// cache: a wildcard signature is found as well as a literal one. A clean + /// text file is clean only when its normalised views could be made, which + /// takes a spill. #[test] - fn a_file_over_the_deep_analysis_limit_is_not_called_clean() { + fn a_file_over_the_deep_analysis_limit_gets_the_full_engine() { let db = db_from(&[ ("w.ndb", "Test.Wild:0:*:6576696c??7061796c6f6164\n"), ("l.ndb", "Test.Literal:0:*:6d61726b65726d61726b6572\n"), @@ -4693,26 +4461,32 @@ mod tests { wild.resize(1024, b' '); let mut literal = b"xx markermarker xx".to_vec(); literal.resize(1024, b' '); + let mut clean = b"xx nothing to see xx".to_vec(); + clean.resize(1024, b' '); let dir = crate::tmpfile::TempDir::new().unwrap(); let path = dir.path().join("big.txt"); - std::fs::write(&path, &wild).unwrap(); - for v in [ - seekable(&db, &wild, &opts), - scan_path(&db, &path, &opts).unwrap().verdict, - ] { - assert!( - matches!(v, Verdict::LimitsExceeded { .. }), - "a literal-only scan is not a full one: {v:?}" - ); - } - std::fs::write(&path, &literal).unwrap(); - for v in [ - seekable(&db, &literal, &opts), - scan_path(&db, &path, &opts).unwrap().verdict, - ] { - assert_eq!(infected_as(&v), Some("Test.Literal")); + for (data, want) in [(&wild, "Test.Wild"), (&literal, "Test.Literal")] { + std::fs::write(&path, data).unwrap(); + for v in [ + seekable(&db, data, &opts), + scan_path(&db, &path, &opts).unwrap().verdict, + ] { + assert_eq!(infected_as(&v), Some(want), "{v:?}"); + } } + assert!( + matches!(seekable(&db, &clean, &opts), Verdict::LimitsExceeded { .. }), + "without a spill the text views were not scanned" + ); + let spilling = ScanOptions { + spill: Some(std::sync::Arc::new(spill::tests::MemSpill { + budget: usize::MAX, + made: Default::default(), + })), + ..opts.clone() + }; + assert_eq!(seekable(&db, &clean, &spilling), Verdict::Clean); } /// A size-limit reason names the flag an operator would raise. @@ -4778,7 +4552,7 @@ mod tests { #[test] fn detects_eicar_pattern() { let db = Scanner::builtin(); - let r = scan_stream(&db, Cursor::new(eicar().to_vec())).unwrap(); + let r = analyze(&db, eicar(), &ScanOptions::default()); assert!(matches!( r.verdict, Verdict::Infected { @@ -4815,11 +4589,28 @@ mod tests { } } + /// ClamAV's `.ftm` types anything starting `60 EA` as ARJ. An object that + /// only happens to start that way must not be opened as a damaged ARJ and + /// reported `UNSCANNABLE`; a real one still is ARJ. + #[test] + #[cfg(feature = "all-formats")] + fn ftm_arj_typing_needs_a_real_header() { + let mut db = Scanner::builtin(); + db.ftm + .extend_from_text("0:0:60ea:ARJ:CL_TYPE_ANY:CL_TYPE_ARJ\n"); + let fake = b"\x60\xea\x10\x00 not an ARJ header, just bytes"; + assert_eq!(db.identify(fake), FileType::Unknown); + let r = analyze(&db, fake, &ScanOptions::default()); + assert!(matches!(r.verdict, Verdict::Clean), "{:?}", r.verdict); + let real = include_bytes!("../../exav-unpack/tests/fixtures/sample.arj"); + assert_eq!(db.identify(real), FileType::Arj); + } + /// Every normalised view a textual file is matched against must still be /// produced, and each must be built only when asked for. /// /// The views are handed out as thunks so the scanner holds one full-size - /// copy at a time instead of all of them — measured at 270 MB saved on a + /// copy at a time instead of all of them, measured at 270 MB saved on a /// 97 MB script. The risk in that shape is silently losing a view: a /// dropped normalisation costs detections and nothing fails, because the /// remaining views still match everything they always did. @@ -4847,11 +4638,15 @@ mod tests { ); // And each one produces something to scan. - for (i, f) in normalizations(FileType::Text, script) + for (i, kind) in normalizations(FileType::Text, script) .into_iter() .enumerate() { - assert!(!f().is_empty(), "normalisation {i} produced nothing"); + let view = make_view(kind, script, &ScanOptions::default()).expect("a view"); + assert!( + !view.source().is_empty(), + "normalisation {i} produced nothing" + ); } } @@ -4920,7 +4715,7 @@ mod tests { db.hashes .extend_from_text(&format!("{}:*:Test.ByHash\n", d.sha256)); db.hashes.finalize(); - let r = scan_stream(&db, Cursor::new(b"some clean-looking content".to_vec())).unwrap(); + let r = analyze(&db, b"some clean-looking content", &ScanOptions::default()); match r.verdict { Verdict::Infected { signature, @@ -4956,12 +4751,6 @@ mod tests { "{n}-byte object: hash match should be {}", n >= 6 ); - // The streaming core has its own hash path and must agree. - let hit = matches!( - scan_stream(&db, Cursor::new(data.clone())).unwrap().verdict, - Verdict::Infected { .. } - ); - assert_eq!(hit, n >= 6, "{n}-byte object via stream_core"); } } @@ -4996,7 +4785,7 @@ mod tests { #[test] fn clean_is_clean() { let db = Scanner::builtin(); - let r = scan_stream(&db, Cursor::new(b"nothing to see".to_vec())).unwrap(); + let r = analyze(&db, b"nothing to see", &ScanOptions::default()); assert_eq!(r.verdict, Verdict::Clean); } @@ -5035,13 +4824,20 @@ mod tests { ); } + /// A match across the block cache's block boundaries is found, at its + /// offset in the object. #[test] fn detects_across_buffer_boundary() { let db = Scanner::builtin(); let mut data = vec![b'A'; 3_000_000]; data.extend_from_slice(eicar()); data.extend(std::iter::repeat_n(b'B', 3_000_000)); - let r = scan_stream(&db, Cursor::new(data)).unwrap(); + let opts = ScanOptions { + deep_analysis_max: 1 << 20, + ..ScanOptions::default() + }; + let size = data.len() as u64; + let r = scan_seekable(&db, Cursor::new(data), size, &opts).unwrap(); match r.verdict { Verdict::Infected { offset, .. } => assert_eq!(offset, 3_000_000), other => panic!("expected infection, got {other:?}"), @@ -5067,15 +4863,21 @@ mod tests { f } - // A container too large for structural analysis that does NOT stream (its - // decoder needs random access over a buffered slice — here OLE2) must not be - // cleared: its contents were never unpacked, so the verdict is - // LimitsExceeded, never Clean. This is the invariant for the still-buffered - // formats. (ZIP/tar/gzip now stream past the cap — see the tests below.) + /// `opts` with somewhere to spill what is too large to hold. + fn spilling(opts: ScanOptions) -> ScanOptions { + ScanOptions { + spill: Some(std::sync::Arc::new(spill::tests::MemSpill { + budget: usize::MAX, + made: Default::default(), + })), + ..opts + } + } + /// A seekable source that serves the object once and then fails. /// /// Models the case that actually bites: not a source that is broken from - /// the start — that fails the first read and is obviously an error — but one + /// the start (that fails the first read and is obviously an error) but one /// that works, gets re-read, and dies the second time. A flaky range server /// is exactly this, and a scan re-reads a container to run the checks that /// need its whole bytes. @@ -5115,8 +4917,8 @@ mod tests { /// A source that dies mid-scan must never come back `Clean`. /// - /// The bytes it failed to deliver still exist — this is not truncation, - /// where the content really is absent and a clean answer is honest — so + /// The bytes it failed to deliver still exist. This is not truncation, + /// where the content really is absent and a clean answer is honest, so /// the only truthful outcomes are a detection, a partial verdict, or an /// error to the caller. #[test] @@ -5141,12 +4943,15 @@ mod tests { } } + /// A container over the deep-analysis limit whose format is read whole + /// (here OLE2) has its members unscanned, so the verdict is LimitsExceeded, + /// never Clean. #[test] #[cfg_attr( target_family = "wasm", ignore = "host filesystem/tempdir unavailable under WASI" )] - fn oversize_nonstreaming_container_is_limits_not_clean() { + fn oversize_container_read_whole_is_limits_not_clean() { use std::io::Write; let mut buf = std::io::Cursor::new(Vec::new()); { @@ -5171,8 +4976,9 @@ mod tests { } // A gzip/tar whose size exceeds `deep_analysis_max` is walked - // member-by-member off disk, never buffered whole, and fully scanned — so - // the cap bounds memory without bounding what the scan can find. + // member-by-member off disk, never held whole, and its members, too large + // to hold as well, go through the spill: the cap bounds memory without + // bounding what the scan can find. #[test] #[cfg_attr( target_family = "wasm", @@ -5187,13 +4993,10 @@ mod tests { let blob = e.finish().unwrap(); let f = write_temp(&blob); let db = Scanner::builtin(); - // A cap of one byte is far below the content, and the streaming path - // still unpacks it and finds EICAR: the cap governs buffering, not - // reach. - let opts = ScanOptions { + let opts = spilling(ScanOptions { deep_analysis_max: 1, ..Default::default() - }; + }); let r = scan_path(&db, f.path(), &opts).unwrap(); assert!( matches!(r.verdict, Verdict::Infected { .. }), @@ -5209,7 +5012,7 @@ mod tests { )] fn streamed_tar_over_cap_finds_eicar() { let mut ar = tar::Builder::new(Vec::new()); - // A benign padding member first, then the payload — proves the walk + // A benign padding member first, then the payload: proves the walk // continues past clean members without buffering the whole archive. let pad = vec![b'Z'; 8192]; let mut h = tar::Header::new_gnu(); @@ -5223,10 +5026,10 @@ mod tests { let blob = ar.into_inner().unwrap(); let f = write_temp(&blob); let db = Scanner::builtin(); - let opts = ScanOptions { + let opts = spilling(ScanOptions { deep_analysis_max: 1, ..Default::default() - }; + }); let r = scan_path(&db, f.path(), &opts).unwrap(); assert!( matches!(r.verdict, Verdict::Infected { .. }), @@ -5235,10 +5038,7 @@ mod tests { ); } - // A streamed container's own whole-file hash must still be detected — the - // streaming path runs a constant-memory raw scan (hash + patterns) before - // walking members, matching the buffered `scan_bytes_core`. Regression guard - // for the streaming routing. + // A container's own whole-file hash is matched as well as its members. #[test] #[cfg_attr( target_family = "wasm", @@ -5264,11 +5064,10 @@ mod tests { } } - // Recursion streaming (c): a gzip NESTED inside a tar, whose decompressed - // content exceeds `max_buffer_bytes` with EICAR buried past that cap. The old - // buffered path decoded the nested gzip into a `max_buffer_bytes`-capped Vec - // and reported LIMITS before reaching EICAR; the streaming recursion scans the - // whole decompressed member (RAM bounded by deep_analysis_max) and finds it. + // A gzip NESTED inside a tar, whose decompressed content exceeds both + // `max_buffer_bytes` and `deep_analysis_max`, with EICAR buried past them: + // the member is decoded as it is read and spilled, not capped or truncated + // at either limit. With nowhere to spill it, it is reported, not cleared. #[test] #[cfg_attr( target_family = "wasm", @@ -5300,16 +5099,22 @@ mod tests { let mut opts = ScanOptions::default(); opts.limits.max_buffer_bytes = 256; opts.deep_analysis_max = 256; - let r = scan_path(&db, f.path(), &opts).unwrap(); + let r = scan_path(&db, f.path(), &spilling(opts.clone())).unwrap(); assert!( matches!(r.verdict, Verdict::Infected { .. }), "nested gzip past the entry cap must be streamed and detected, got {:?}", r.verdict ); + match scan_path(&db, f.path(), &opts).unwrap().verdict { + Verdict::LimitsExceeded { reason } => { + assert!(reason.contains("could not be spilled"), "{reason}") + } + other => panic!("expected a limit without a spill, got {other:?}"), + } } - // Flat content larger than the deep-analysis cap still goes through the - // streaming core, so a literal signature is found. + // Flat content larger than the deep-analysis cap gets the full engine + // through a block cache, so a signature in it is found. #[test] #[cfg_attr( target_family = "wasm", @@ -5380,7 +5185,7 @@ mod tests { /// This is the shape that turns a network fault into a clean verdict: the /// reader knows more bytes exist (`pos < len`) but has none to hand. If it /// answers `Ok(0)` it is claiming end-of-file, and every layer above treats - /// that as an ordinary truncated archive — content actually absent, which + /// that as an ordinary truncated archive (content actually absent), which /// exav is entitled to call clean. It must be an error instead. #[cfg(feature = "http")] #[test] @@ -5459,6 +5264,8 @@ mod tests { let addr = listener.local_addr().unwrap(); let served = Arc::new(AtomicU64::new(0)); let served_t = served.clone(); + let requests = Arc::new(AtomicU64::new(0)); + let requests_t = requests.clone(); let blob_t = blob.clone(); thread::spawn(move || { for stream in listener.incoming() { @@ -5466,6 +5273,7 @@ mod tests { Ok(s) => s, Err(_) => break, }; + requests_t.fetch_add(1, Ordering::SeqCst); let mut rdr = BufReader::new(stream.try_clone().unwrap()); let mut line = String::new(); if rdr.read_line(&mut line).unwrap_or(0) == 0 { @@ -5518,8 +5326,8 @@ mod tests { ); // What is bounded is MEMORY, not transfer. A range source may be read - // more than once — the container is read again to run the checks that - // need its whole bytes — so asserting a fraction of the object is + // more than once (the container is read again to run the checks that + // need its whole bytes), so asserting a fraction of the object is // fetched would pin a property exav does not promise. What it does // promise is that a scan does not grow without limit: the object is // ~1 MB and is walked a small, constant number of times. @@ -5529,5 +5337,203 @@ mod tests { "fetched {bytes} of {total}; a scan should re-read the source a \ small constant number of times, not unboundedly" ); + // Read straight through, each request continues the previous one and + // fetches twice as much: 64 KiB, 128 KiB, ... rather than 16 of 64 KiB. + let requests = requests.load(Ordering::SeqCst); + assert!( + requests <= 8, + "{requests} range requests for a {total}-byte object" + ); + } +} + +/// The scan of an object read through a block cache answers as the scan of +/// the same bytes held in memory: same verdict, same detections. +#[cfg(test)] +mod parity { + use super::*; + use std::io::Cursor; + + /// Every fixture of this crate and of the extractors, de-obfuscated. + fn fixtures() -> Vec<(String, Vec)> { + fn walk(dir: &std::path::Path, out: &mut Vec<(String, Vec)>) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let path = e.path(); + if path.is_dir() { + walk(&path, out); + } else if let Ok(data) = + // `read_fixture` unmasks `.xor` itself. + unpack::read_fixture( + path.to_string_lossy().trim_end_matches(".xor"), + ) + { + if data.len() < 8 << 20 { + out.push((path.display().to_string(), data)); + } + } + } + } + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut out = Vec::new(); + walk(&root.join("tests/fixtures"), &mut out); + walk(&root.join("../exav-unpack/tests/fixtures"), &mut out); + out + } + + fn database() -> Scanner { + let mut b = loader::Builder::new(); + b.add_named_bytes( + "t.ndb", + b"Test.Wild:0:*:4d5a90??03\nTest.Html:3:*:3c7363726970743e6576616c\n\ + Test.Pe:1:EP+0:??\n", + false, + ); + b.add_named_bytes( + "t.ldb", + b"Test.Ldb;Engine:51-255,Target:0;0&1;504b0304;2e786d6c\n", + false, + ); + #[cfg(feature = "yara")] + b.add_named_bytes( + "t.yar", + b"import \"pe\"\n\ + rule many_pk { strings: $a = \"PK\" condition: #a > 3 }\n\ + rule sections { condition: pe.number_of_sections > 2 }\n", + false, + ); + b.build().unwrap() + } + + fn spilling(opts: &ScanOptions) -> ScanOptions { + ScanOptions { + spill: Some(std::sync::Arc::new(spill::tests::MemSpill { + budget: usize::MAX, + made: Default::default(), + })), + ..opts.clone() + } + } + + /// How the streamed scan of `data` differs from the in-memory one, if it + /// does. Both have spill space, so a member too large to hold is scanned + /// the same way on each. + fn differs(db: &Scanner, data: &[u8], opts: &ScanOptions) -> Option { + let cache = + byte_source::BlockCache::with_sizes(Cursor::new(data.to_vec()), 4093, 16 * 4093) + .unwrap(); + let opts = &spilling(opts); + let want = analyze(db, data, opts).verdict; + let got = analyze_source(db, &cache, opts).verdict; + if format!("{want:?}") != format!("{got:?}") { + return Some(format!("verdict: memory {want:?}, streamed {got:?}")); + } + let size = data.len() as u64; + let got = scan_seekable(db, Cursor::new(data), size, opts).map(|r| r.verdict); + if format!("{:?}", Ok::<_, ()>(&want)) != format!("{:?}", got.as_ref().map_err(|_| ())) { + return Some(format!("verdict: memory {want:?}, seekable {got:?}")); + } + let want = analyze_all_with_outcome(db, data, opts); + let got = analyze_all_source(db, &cache, opts); + if format!("{want:?}") != format!("{got:?}") { + return Some(format!("all matches: memory {want:?}, streamed {got:?}")); + } + None + } + + fn check(db: &Scanner, inputs: &[(String, Vec)], opts: &ScanOptions) { + let mut wrong = Vec::new(); + for (name, data) in inputs { + if let Some(d) = differs(db, data, opts) { + wrong.push(format!("{name}: {d}")); + } + } + assert!( + wrong.is_empty(), + "{} of {} differ:\n{}", + wrong.len(), + inputs.len(), + wrong.join("\n") + ); + } + + #[test] + fn a_streamed_scan_answers_as_an_in_memory_one() { + let inputs = fixtures(); + assert!(inputs.len() > 100, "the fixtures are where they were"); + let db = database(); + let opts = ScanOptions { + heuristics: true, + alert_broken: true, + alert_broken_media: true, + alert_partition_intersection: true, + alert_phishing: true, + structured_cc_count: Some(1), + ..ScanOptions::default() + }; + check(&db, &inputs, &opts); + } + + /// The same, over a real database and a directory of samples: + /// `EXAV_PARITY_DB=<.exavdb> EXAV_PARITY_CORPUS= cargo test -p exav-core + /// --release --lib parity::corpus -- --ignored --nocapture`. + #[test] + #[ignore = "needs a database and a corpus"] + fn corpus() { + let (Ok(db), Ok(dir)) = ( + std::env::var("EXAV_PARITY_DB"), + std::env::var("EXAV_PARITY_CORPUS"), + ) else { + eprintln!("skip: EXAV_PARITY_DB / EXAV_PARITY_CORPUS unset"); + return; + }; + let db = database::load(std::path::Path::new(&db)).expect("load the database"); + let limit: usize = std::env::var("EXAV_PARITY_LIMIT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(usize::MAX); + let opts = ScanOptions::default(); + let (mut seen, mut wrong) = (0, Vec::new()); + // One at a time: a corpus does not fit in memory. + for e in walkdir_files(std::path::Path::new(&dir)) + .into_iter() + .take(limit) + { + let Ok(data) = std::fs::read(&e) else { + continue; + }; + if data.len() >= 64 << 20 { + continue; + } + seen += 1; + if let Some(d) = differs(&db, &data, &opts) { + eprintln!("DIFFERS {}: {d}", e.display()); + wrong.push(e.display().to_string()); + } + } + assert!(wrong.is_empty(), "{} of {seen} differ", wrong.len()); + eprintln!("{seen} samples, all the same"); + } + + fn walkdir_files(dir: &std::path::Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&d) else { + continue; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else { + out.push(p); + } + } + } + out.sort(); + out } } diff --git a/crates/exav-core/src/loader.rs b/crates/exav-core/src/loader.rs index 1ccf4f6b..bdf901de 100644 --- a/crates/exav-core/src/loader.rs +++ b/crates/exav-core/src/loader.rs @@ -22,14 +22,14 @@ pub const CONTAINER_EXTENSIONS: &[&str] = &["cvd", "cld"]; /// Loose signature-file extensions, loaded as unofficial. pub const SIGNATURE_EXTENSIONS: &[&str] = &[ "ndb", "ndu", "db", "hdb", "hdu", "hsb", "hsu", "fdb", "imp", "ldb", "ldu", "mdb", "mdu", - "msb", "cdb", "ftm", "pdb", "gdb", "wdb", "crb", "yar", "yara", "fp", "sfp", "ign", "ign2", - "cbc", "idb", "pwdb", + "msb", "msu", "cdb", "ftm", "pdb", "gdb", "wdb", "crb", "yar", "yara", "fp", "sfp", "ign", + "ign2", "cbc", "idb", "pwdb", ]; /// Whether a filename's extension is one the loader routes. /// /// A file in a signature directory with any other extension is skipped without -/// being read — the directory legitimately holds `.cdiff`, `.sign`, `.info` and +/// being read: the directory legitimately holds `.cdiff`, `.sign`, `.info` and /// updater state alongside the databases, and reading those as signatures would /// be worse than ignoring them. The cost is that a file which *should* have been /// a database is skipped just as quietly, so anything that puts a file there is @@ -61,8 +61,6 @@ pub enum LoadError { TooLarge { path: String }, #[error("parsing signature container: {0}")] Container(String), - #[error("building pattern set: {0}")] - Patterns(String), #[error("loading database {path}: {source}")] Database { path: String, @@ -94,14 +92,12 @@ fn read_capped(path: &Path) -> Result, LoadError> { use crate::fuzzy::FuzzyDb; use crate::hashes::{HashDb, SectionHashDb}; use crate::ml::HeuristicModel; -use crate::patterns::{eicar, Pattern, PatternSet}; +use crate::patterns::eicar; use crate::Scanner; /// Accumulates parsed signatures from multiple files before building. #[derive(Default)] pub struct Builder { - /// Literal patterns for the streaming path (literal `.ndb` + `.db`). - patterns: Vec, /// The full in-memory engine, fed signature text file-by-file so the raw /// text is dropped as it's compiled (rather than concatenated and held /// alive through the memory-heavy automaton build). @@ -116,14 +112,14 @@ pub struct Builder { /// Raw `.cbc` texts; parsed and gated into a runtime at [`Self::build`]. bytecode_sources: Vec, /// ClamAV `--detect-pua`: when false (default), the PUA-variant databases - /// (`.ndu`/`.ldu`/`.hdu`/`.hsu`/`.mdu`) are skipped entirely and `PUA.*` + /// (`.ndu`/`.ldu`/`.hdu`/`.hsu`/`.mdu`/`.msu`) are skipped entirely and `PUA.*` /// signatures elsewhere are dropped. detect_pua: bool, /// Retained for API/CLI compatibility (`--clamav-compat` calls /// [`Self::set_unofficial_suffix`]); it does not affect what is stored. /// - /// Provenance — whether a signature came from an unofficial (non-`.cvd`) - /// database — is now recorded *per signature*, always, regardless of this + /// Provenance (whether a signature came from an unofficial, non-`.cvd` + /// database) is now recorded *per signature*, always, regardless of this /// flag. The `.UNOFFICIAL` name suffix is applied at REPORT /// time (gated on `ScanOptions::unofficial_suffix`), so ONE database serves both compat and /// non-compat scans. Official `.cvd`/`.cld` sigs carry `unofficial=false` and @@ -150,6 +146,10 @@ pub struct Builder { /// phishing heuristic for brand scoping and false-positive suppression. #[cfg(feature = "phishing")] phishing: crate::phishing::PhishingDb, + /// A prebuilt database met in a directory, which loads only by its own + /// path. Kept so a directory holding nothing else is an error rather than + /// the built-in baseline. + prebuilt_skipped: Option, } impl Builder { @@ -183,7 +183,7 @@ impl Builder { /// Load a path that is either a single db file/container or a directory. /// Directories are scanned **recursively**: db files at any depth are loaded /// (entries sorted for deterministic order). Hidden (`.`-prefixed) - /// sub-directories are skipped — `.git`, caches, editor/backup dirs — so a + /// sub-directories are skipped (`.git`, caches, editor/backup dirs), so a /// nested feed layout loads fully without slurping unrelated files. Recursion /// follows symlinks but is depth-capped so a symlink cycle can't loop forever. pub fn add_path(&mut self, path: &Path) -> Result<(), LoadError> { @@ -259,8 +259,13 @@ impl Builder { Ok(()) } // Skip anything else in a database directory (.cdiff/.sign/.info, - // state files, etc.) without reading it. - _ => Ok(()), + // state files, etc.) without reading it past its magic. + _ => { + if self.prebuilt_skipped.is_none() && crate::database::is_database_file(path) { + self.prebuilt_skipped = Some(path.to_path_buf()); + } + Ok(()) + } } } @@ -275,51 +280,37 @@ impl Builder { fn add_named_text(&mut self, name: &str, text: &str, official: bool) { let ext = name.rsplit('.').next().unwrap_or("").to_ascii_lowercase(); // Provenance: a signature from any database that is NOT a signed - // `.cvd`/`.cld` container is "unofficial" — ClamAV suffixes such + // `.cvd`/`.cld` container is "unofficial"; ClamAV suffixes such // detections with `.UNOFFICIAL`. // We record this bit per signature, ALWAYS (independent of any compat // flag), and store the CLEAN name. The suffix is applied at report time // (see `crate::report_name`), gated on `ScanOptions::unofficial_suffix`, so a single // database serves both compat and non-compat scans. let unofficial = !official; - // The `.??u` databases (`.ndu`/`.ldu`/`.hdu`/`.hsu`/`.mdu`) are ClamAV's - // PUA signature sets, loaded only with `DetectPUA`. Skip them by default - // so exav doesn't flag PUA (including hash-based PUA, which has no `PUA.` - // name gate) on a default config. - if !self.detect_pua && matches!(ext.as_str(), "ndu" | "ldu" | "hdu" | "hsu" | "mdu") { + // The `.??u` databases (`.ndu`/`.ldu`/`.hdu`/`.hsu`/`.mdu`/`.msu`) are + // ClamAV's PUA signature sets, loaded only with `DetectPUA`. Skip them by + // default so exav doesn't flag PUA (including hash-based PUA, which has no + // `PUA.` name gate) on a default config. + if !self.detect_pua && matches!(ext.as_str(), "ndu" | "ldu" | "hdu" | "hsu" | "mdu" | "msu") + { return; } match ext.as_str() { // `.ndb`/`.ndu` (unpacked-content variant share the format). - "ndb" | "ndu" => { - // Literal subset for streaming; full text for the engine. - let (mut pats, _unsup) = PatternSet::parse_ndb_prov(text, unofficial); - // The engine drops `PUA.*` unless PUA detection is on; the - // streaming set has to agree or a large file reports a PUA - // the same content in a small file would not. - if !self.detect_pua { - pats.retain(|p| !p.name.starts_with("PUA.")); - } - self.patterns.append(&mut pats); - self.engine.add_ndb(text, unofficial); - } - // Literal `Name=HEX` signatures. The engine is what in-memory scans - // match with, so they go there too. In the streaming set alone they - // were only ever seen by a file too large to buffer. + "ndb" | "ndu" => self.engine.add_ndb(text, unofficial), + // Literal `Name=HEX` signatures. "db" => { - if let Ok(pats) = PatternSet::parse_simple(text) { - for mut p in pats { - if p.name.starts_with("PUA.") && !self.detect_pua { + if let Ok(pats) = crate::patterns::parse_simple(text) { + for (name, bytes) in pats { + if name.starts_with("PUA.") && !self.detect_pua { continue; } - p.unofficial = unofficial; - self.engine.add_literal_prov(&p.name, &p.bytes, unofficial); - self.patterns.push(p); + self.engine.add_literal_prov(&name, &bytes, unofficial); } } } "hdb" | "hsb" | "hdu" | "hsu" => self.hashes.extend_from_text_prov(text, unofficial), - "mdb" | "mdu" | "msb" => self.sections.extend_from_text_prov(text, unofficial), + "mdb" | "mdu" | "msb" | "msu" => self.sections.extend_from_text_prov(text, unofficial), "fdb" => self.fuzzy.extend_from_text(text), "imp" => self.fuzzy.extend_imp(text), "cdb" => self.cdb.extend_from_text_prov(text, unofficial), @@ -335,9 +326,9 @@ impl Builder { #[cfg(not(feature = "phishing"))] "pdb" | "gdb" | "wdb" => {} // Authenticode-cert database (`.crb`): block-list of signer certs. - // We match on the certificate identity (subject-hash + serial) — RSA + // We match on the certificate identity (subject-hash + serial). RSA // signature *verification* remains out of scope (would reintroduce the - // excluded `rsa` crate — see docs/DEPENDENCIES.md). + // excluded `rsa` crate, see docs/DEPENDENCIES.md). "crb" => self.crb.parse_into(text), "yar" | "yara" => self.yara.extend_from_text(text, unofficial), // Whole-file hash allowlist (clears a detection). @@ -377,12 +368,9 @@ impl Builder { } } - /// Build the final database. If no patterns were loaded, the built-in - /// EICAR pattern is retained so the engine always has at least a test - /// signature. + /// Build the final database. The EICAR test signature is always added, so + /// the engine has at least that one. pub fn build(mut self) -> Result { - // Fold EICAR into the engine so in-memory scans detect it without the - // streaming automaton; also keep it in the streaming literal set. self.engine.add_literal("Eicar-Test-Signature", eicar()); // Sort + intern the hash tables BEFORE building the (memory-heavy) @@ -398,14 +386,14 @@ impl Builder { // stored in the database and scans don't recompile on first use. self.yara.finalize(); + // Bytecode triggers join the engine, to be matched in the same sweep. + let bytecode = crate::bytecode::runtime::BytecodeRuntime::from_sources( + self.bytecode_sources, + &mut self.engine, + ); let engine = self.engine.build_with_budget(self.max_build_mem); - let mut pats = self.patterns; - pats.push(Pattern::new("Eicar-Test-Signature", eicar().to_vec())); - let patterns = PatternSet::build(&pats, 0).map_err(LoadError::Patterns)?; - Ok(Scanner { - patterns, engine, hashes, sections, @@ -414,9 +402,7 @@ impl Builder { yara: self.yara, allow, ignored: self.ignored, - bytecode: crate::bytecode::runtime::BytecodeRuntime::from_sources( - self.bytecode_sources, - ), + bytecode, model: Box::new(HeuristicModel), ml_threshold: 0.85, ftm: self.ftm, @@ -531,12 +517,33 @@ pub fn load_with_options_mem( source: e, }); } + // A mistyped path would otherwise load nothing and return the built-in + // baseline, a scanner that finds nothing but the test string. + if !path.exists() { + return Err(LoadError::Io { + path: path.display().to_string(), + source: std::io::Error::new(std::io::ErrorKind::NotFound, "no such file or directory"), + }); + } let mut builder = Builder::new(); builder.set_detect_pua(detect_pua); builder.set_unofficial_suffix(unofficial_suffix); builder.set_max_build_mem(max_build_mem); builder.add_path(path)?; - builder.build() + let prebuilt = builder.prebuilt_skipped.take(); + let scanner = builder.build()?; + match prebuilt { + Some(db) if scanner.signature_count() <= Scanner::builtin().signature_count() => { + Err(LoadError::Io { + path: db.display().to_string(), + source: std::io::Error::other( + "a prebuilt database loads by its own path, not from the directory \ + holding it", + ), + }) + } + _ => Ok(scanner), + } } #[cfg(test)] @@ -549,8 +556,8 @@ mod tests { /// /// They have to be, because a file this says yes to and the loader skips is /// invisible: it is fetched, reported as updated, and never becomes a - /// signature. The directory is also a working area — `.cdiff`, `.sign`, - /// `.info` and updater state live beside the databases — so "skip what I do + /// signature. The directory is also a working area (`.cdiff`, `.sign`, + /// `.info` and updater state live beside the databases), so "skip what I do /// not know" is right, and the answer to which is which can only be in one /// place. #[test] @@ -582,6 +589,43 @@ mod tests { } } + /// A path that loads nothing is an error, not the built-in baseline: a + /// typo, or a directory whose only database is a prebuilt one (which loads + /// by its own path). + #[test] + fn a_path_that_loads_nothing_is_an_error() { + let dir = std::env::temp_dir().join(format!("exav-loader-empty-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + assert!(load(&dir.join("no-such-dir")).is_err()); + + let mut b = Builder::new(); + b.add_named_bytes("x.ndb", b"Test.X:0:*:4142434445464748\n", false); + let db = dir.join("x.exavdb"); + crate::database::save(&b.build().unwrap(), &db).unwrap(); + let err = load(&dir).map(|_| ()).unwrap_err(); + assert!(err.to_string().contains("x.exavdb"), "{err}"); + // The file itself loads, and beside real signatures it is only skipped. + assert!(load(&db).is_ok()); + std::fs::write(dir.join("y.ndb"), "Test.Y:0:*:4142434445464749\n").unwrap(); + assert!(load(&dir).is_ok()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// `.msu` is ClamAV's PUA section-hash set, loaded like `.msb` and, like + /// every `.??u` set, only when PUA detection is on. + #[test] + fn msu_loads_as_pua_section_hashes() { + let line = b"4096:d41d8cd98f00b204e9800998ecf8427e:PUA.Test.Section\n"; + let count = |pua: bool| { + let mut b = Builder::new(); + b.set_detect_pua(pua); + b.add_named_bytes("x.msu", line, false); + b.build().unwrap().signature_count() + }; + assert!(loads_by_extension("x.msu")); + assert_eq!(count(true), count(false) + 1); + } + #[test] fn parses_pwdb_cleartext_and_hex() { // "secret" = 736563726574 in hex. @@ -632,7 +676,7 @@ Sig.C;td;0;cleartext-pw\n"; // duplicate password de-duped writeln!(f, "imphash:abcd:Sig.F").unwrap(); let db = load(dir.path()).unwrap(); - assert!(!db.patterns.is_empty()); + assert!(!db.engine.is_empty()); // "de*ad" has no >=2-byte literal anchor -> counted unsupported. assert_eq!(db.unsupported_count(), 1); assert!(!db.hashes.is_empty()); @@ -729,7 +773,7 @@ Sig.C;td;0;cleartext-pw\n"; // duplicate password de-duped } /// (c) an official `.cvd` signature is NEVER suffixed, even in compat, while a - /// loose sig loaded alongside it IS — proving per-signature provenance for a + /// loose sig loaded alongside it IS. This proves per-signature provenance for a /// mixed source. #[test] fn official_cvd_never_suffixed_mixed_source() { @@ -787,15 +831,14 @@ Sig.C;td;0;cleartext-pw\n"; // duplicate password de-duped // In compat the detection name is suffixed and matches the ign entry. assert_eq!(analyze(&db, data, &compat).verdict, Verdict::Clean); // In non-compat the detection is the clean `Demo.Loose`, which the - // suffixed ign entry does NOT name — so it still fires. + // suffixed ign entry does NOT name, so it still fires. assert!(matches!( analyze(&db, data, &ScanOptions::default()).verdict, Verdict::Infected { .. } )); } - /// A `.db` signature is matched by an ordinary in-memory scan, not only by - /// the streaming pass a file too large to buffer gets. + /// A `.db` signature is matched by an ordinary scan. #[test] #[cfg_attr( target_family = "wasm", @@ -812,15 +855,15 @@ Sig.C;td;0;cleartext-pw\n"; // duplicate password de-duped } } - /// `PUA.*` stays off unless asked for on every path, the streaming literal - /// pass included. + /// `PUA.*` stays off unless asked for. #[test] #[cfg_attr( target_family = "wasm", ignore = "host filesystem/tempdir unavailable under WASI" )] - fn pua_literals_stay_off_the_streaming_path_by_default() { - use crate::{scan_stream, Verdict}; + fn pua_literals_stay_off_by_default() { + use crate::{analyze, ScanOptions, Verdict}; + let scan = |db: &Scanner, data: &[u8]| analyze(db, data, &ScanOptions::default()); let dir = crate::tmpfile::TempDir::new().unwrap(); std::fs::write( dir.path().join("p.ndb"), @@ -830,20 +873,16 @@ Sig.C;td;0;cleartext-pw\n"; // duplicate password de-duped std::fs::write(dir.path().join("a.ndb"), "Demo.X:0:*:6e6f7065\n").unwrap(); let data = b"xx puapuapuap xx"; let db = load(dir.path()).unwrap(); - assert_eq!(scan_stream(&db, &data[..]).unwrap().verdict, Verdict::Clean); + assert_eq!(scan(&db, data).verdict, Verdict::Clean); let mut b = Builder::new(); b.set_detect_pua(true); b.add_path(dir.path()).unwrap(); let pua = b.build().unwrap(); - assert!(matches!( - scan_stream(&pua, &data[..]).unwrap().verdict, - Verdict::Infected { .. } - )); + assert!(matches!(scan(&pua, data).verdict, Verdict::Infected { .. })); } - /// A literal `.ndb` signature is held twice, by the engine and by the - /// streaming set, and must be counted once. + /// A literal `.ndb` signature is counted once. #[test] #[cfg_attr( target_family = "wasm", diff --git a/crates/exav-core/src/normalize.rs b/crates/exav-core/src/normalize.rs index f73052a3..bba16a4c 100644 --- a/crates/exav-core/src/normalize.rs +++ b/crates/exav-core/src/normalize.rs @@ -14,10 +14,13 @@ //! None of these interpret attacker-controlled sizes; they walk the input once //! and can't panic on any byte sequence. +use crate::byte_source::{Bytes, Indexed}; + /// Fraction (percent) of sampled bytes that must be "text-like" for a buffer to /// be treated as textual, and the NUL-byte ceiling above which it's binary. const TEXT_PERCENT: usize = 90; -const SAMPLE: usize = 8192; +/// Bytes of an object's start [`is_textual`] looks at. +pub(crate) const SAMPLE: usize = 8192; /// Heuristic: does this buffer look like text worth running normalised /// signatures over? Empty input is not textual; a buffer with more than ~1% NUL @@ -51,9 +54,26 @@ fn lower(b: u8) -> u8 { b.to_ascii_lowercase() } +/// Where a normaliser writes its view: memory, or a spill for an object too +/// large to hold. +pub(crate) trait Out { + fn push(&mut self, b: u8); + fn last(&self) -> Option; +} + +impl Out for Vec { + #[inline] + fn push(&mut self, b: u8) { + Vec::push(self, b); + } + fn last(&self) -> Option { + self.as_slice().last().copied() + } +} + /// Append `b`, collapsing any run of ASCII whitespace to a single space. #[inline] -fn push_collapsed(out: &mut Vec, b: u8, prev_ws: &mut bool) { +fn push_collapsed(out: &mut O, b: u8, prev_ws: &mut bool) { if b.is_ascii_whitespace() { if !*prev_ws { out.push(b' '); @@ -68,28 +88,42 @@ fn push_collapsed(out: &mut Vec, b: u8, prev_ws: &mut bool) { /// Normalise HTML: decode entities, lowercase, collapse whitespace. Tags are /// preserved (so `Target:3` signatures written against markup still match). pub fn html(data: &[u8]) -> Vec { - let decoded = decode_entities(data); - let mut out = Vec::with_capacity(decoded.len()); + let mut out = Vec::with_capacity(data.len()); + html_into(&mut Indexed(data), &mut out); + out +} + +/// [`html`] from any object into any [`Out`]. +pub(crate) fn html_into(data: &mut B, out: &mut O) { let mut prev_ws = false; - for &b in &decoded { - push_collapsed(&mut out, lower(b), &mut prev_ws); + let mut i = 0usize; + let n = data.len(); + while i < n { + let (b, next) = decode_entity_at(data, i); + push_collapsed(out, lower(b), &mut prev_ws); + i = next; } - out } /// Normalise plain text: lowercase, drop control bytes, collapse whitespace. pub fn text(data: &[u8]) -> Vec { let mut out = Vec::with_capacity(data.len()); + text_into(&mut Indexed(data), &mut out); + out +} + +/// [`text`] from any object into any [`Out`]. +pub(crate) fn text_into(data: &mut B, out: &mut O) { let mut prev_ws = false; - for &b in data { + for i in 0..data.len() { + let b = data.at(i); if b.is_ascii_whitespace() { - push_collapsed(&mut out, b, &mut prev_ws); + push_collapsed(out, b, &mut prev_ws); } else if b >= 0x20 { out.push(lower(b)); prev_ws = false; } } - out } /// A JavaScript identifier byte (letters, digits, `_`, `$`). @@ -104,59 +138,52 @@ fn is_word(b: u8) -> bool { /// comment-and-case obfuscation of `eval(unescape(...))`-style loaders (a comment /// between `eval(` and `unescape(` must not leave a separating space). pub fn javascript(data: &[u8]) -> Vec { - let stripped = strip_comments(data); - let mut out: Vec = Vec::with_capacity(stripped.len()); + let mut out = Vec::with_capacity(data.len()); + javascript_into(&mut Indexed(data), &mut out); + out +} + +/// [`javascript`] from any object into any [`Out`]. +pub(crate) fn javascript_into(data: &mut B, out: &mut O) { let mut pending_ws = false; - for &b in &stripped { + strip_comments(data, &mut |b| { if b.is_ascii_whitespace() { pending_ws = true; - continue; + return; } let c = lower(b); if pending_ws { pending_ws = false; - if let Some(&last) = out.last() { + if let Some(last) = out.last() { if is_word(last) && is_word(c) { out.push(b' '); } } } out.push(c); - } - out + }); } -/// Decode HTML character references: numeric decimal (`i`), numeric hex -/// (`i`/`i`) and the handful of named entities. Only well-formed, -/// terminated references decode; anything else is copied verbatim. -fn decode_entities(data: &[u8]) -> Vec { - let mut out = Vec::with_capacity(data.len()); - let mut i = 0usize; - let n = data.len(); - while i < n { - if data[i] != b'&' { - out.push(data[i]); - i += 1; - continue; - } - // Find the terminating ';' within a short window (entities are short). - let limit = (i + 12).min(n); - let semi = data[i + 1..limit].iter().position(|&b| b == b';').map(|p| i + 1 + p); - let Some(semi) = semi else { - out.push(b'&'); - i += 1; - continue; - }; - let body = &data[i + 1..semi]; - if let Some(decoded) = decode_reference(body) { - out.push(decoded); - i = semi + 1; - } else { - out.push(b'&'); - i += 1; - } +/// The byte an HTML character reference at `i` decodes to, and where the text +/// after it starts: numeric decimal (`i`), numeric hex (`i`/`i`) +/// and the handful of named entities. Only well-formed, terminated references +/// decode; anything else is the byte at `i` itself. +#[inline] +fn decode_entity_at(data: &mut B, i: usize) -> (u8, usize) { + let b = data.at(i); + if b != b'&' { + return (b, i + 1); + } + // Find the terminating ';' within a short window (entities are short). + let limit = (i + 12).min(data.len()); + let window = data.range(i + 1, limit); + let Some(p) = window.iter().position(|&b| b == b';') else { + return (b'&', i + 1); + }; + match decode_reference(&window[..p]) { + Some(decoded) => (decoded, i + 1 + p + 1), + None => (b'&', i + 1), } - out } /// Decode the inside of one `&…;` reference to a single byte, if recognised. @@ -196,17 +223,17 @@ fn decode_reference(body: &[u8]) -> Option { /// Remove `/* … */` and `// …` comments, treating them as inert inside string /// literals (`'…'`, `"…"`, `` `…` ``) so a comment marker in a string survives. -fn strip_comments(data: &[u8]) -> Vec { - let mut out = Vec::with_capacity(data.len()); +/// Each byte kept goes to `out`. +fn strip_comments(data: &mut B, out: &mut F) { let n = data.len(); let mut i = 0usize; let mut quote: Option = None; while i < n { - let b = data[i]; + let b = data.at(i); if let Some(q) = quote { - out.push(b); + out(b); if b == b'\\' && i + 1 < n { - out.push(data[i + 1]); // keep the escaped char intact + out(data.at(i + 1)); // keep the escaped char intact i += 2; continue; } @@ -219,30 +246,29 @@ fn strip_comments(data: &[u8]) -> Vec { match b { b'\'' | b'"' | b'`' => { quote = Some(b); - out.push(b); + out(b); i += 1; } - b'/' if i + 1 < n && data[i + 1] == b'*' => { + b'/' if i + 1 < n && data.at(i + 1) == b'*' => { i += 2; - while i + 1 < n && !(data[i] == b'*' && data[i + 1] == b'/') { + while i + 1 < n && !(data.at(i) == b'*' && data.at(i + 1) == b'/') { i += 1; } i = (i + 2).min(n); - out.push(b' '); // comment becomes a token separator + out(b' '); // comment becomes a token separator } - b'/' if i + 1 < n && data[i + 1] == b'/' => { + b'/' if i + 1 < n && data.at(i + 1) == b'/' => { i += 2; - while i < n && data[i] != b'\n' { + while i < n && data.at(i) != b'\n' { i += 1; } } _ => { - out.push(b); + out(b); i += 1; } } } - out } #[cfg(test)] @@ -308,4 +334,178 @@ mod tests { let out = text(b"Hello\x00\x01World\t\tGoodbye"); assert_eq!(out, b"helloworld goodbye"); } + + /// The normalisers as they were, over a slice in two passes. + mod before { + use super::super::{decode_reference, is_word, lower}; + + fn collapse(out: &mut Vec, b: u8, prev_ws: &mut bool) { + if b.is_ascii_whitespace() { + if !*prev_ws { + out.push(b' '); + *prev_ws = true; + } + } else { + out.push(b); + *prev_ws = false; + } + } + + pub fn html(data: &[u8]) -> Vec { + let mut decoded = Vec::new(); + let mut i = 0usize; + let n = data.len(); + while i < n { + if data[i] != b'&' { + decoded.push(data[i]); + i += 1; + continue; + } + let limit = (i + 12).min(n); + let semi = data[i + 1..limit].iter().position(|&b| b == b';').map(|p| i + 1 + p); + match semi.and_then(|s| decode_reference(&data[i + 1..s]).map(|d| (d, s))) { + Some((d, s)) => { + decoded.push(d); + i = s + 1; + } + None => { + decoded.push(b'&'); + i += 1; + } + } + } + let mut out = Vec::new(); + let mut prev_ws = false; + for &b in &decoded { + collapse(&mut out, lower(b), &mut prev_ws); + } + out + } + + pub fn text(data: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut prev_ws = false; + for &b in data { + if b.is_ascii_whitespace() { + collapse(&mut out, b, &mut prev_ws); + } else if b >= 0x20 { + out.push(lower(b)); + prev_ws = false; + } + } + out + } + + pub fn javascript(data: &[u8]) -> Vec { + let mut stripped = Vec::new(); + let n = data.len(); + let mut i = 0usize; + let mut quote: Option = None; + while i < n { + let b = data[i]; + if let Some(q) = quote { + stripped.push(b); + if b == b'\\' && i + 1 < n { + stripped.push(data[i + 1]); + i += 2; + continue; + } + if b == q { + quote = None; + } + i += 1; + continue; + } + match b { + b'\'' | b'"' | b'`' => { + quote = Some(b); + stripped.push(b); + i += 1; + } + b'/' if i + 1 < n && data[i + 1] == b'*' => { + i += 2; + while i + 1 < n && !(data[i] == b'*' && data[i + 1] == b'/') { + i += 1; + } + i = (i + 2).min(n); + stripped.push(b' '); + } + b'/' if i + 1 < n && data[i + 1] == b'/' => { + i += 2; + while i < n && data[i] != b'\n' { + i += 1; + } + } + _ => { + stripped.push(b); + i += 1; + } + } + } + let mut out: Vec = Vec::new(); + let mut pending_ws = false; + for &b in &stripped { + if b.is_ascii_whitespace() { + pending_ws = true; + continue; + } + let c = lower(b); + if pending_ws { + pending_ws = false; + if let Some(&last) = out.last() { + if is_word(last) && is_word(c) { + out.push(b' '); + } + } + } + out.push(c); + } + out + } + } + + #[test] + fn references_at_the_edge_of_the_window_decode_as_they_did() { + for s in [ + &b"aAb"[..], + b"aAb", + b"A", + b"A", + b"&&&#;&#x;&", + ] { + assert_eq!(html(s), before::html(s), "{:?}", String::from_utf8_lossy(s)); + } + assert_eq!(html(b"aAb"), b"aab"); + } + + #[test] + fn the_views_are_what_they_were_from_memory_or_read_in_blocks() { + use crate::byte_source::{BlockCache, Stepper, CHUNK}; + let alphabet = b"&#;x6aA9lgt amp;\t\n/*'\"`\\\x01Zq_$"; + let mut state = 0x853C_49E6_748F_EA9Bu64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for round in 0..400 { + let len = if round < 390 { (next() % 64) as usize } else { 3 * CHUNK + (next() % 100) as usize }; + let data: Vec = (0..len).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect(); + let what = String::from_utf8_lossy(&data[..data.len().min(64)]).into_owned(); + assert_eq!(html(&data), before::html(&data), "html {what:?}"); + assert_eq!(text(&data), before::text(&data), "text {what:?}"); + assert_eq!(javascript(&data), before::javascript(&data), "javascript {what:?}"); + let cache = BlockCache::with_sizes(std::io::Cursor::new(data.clone()), 7, 56).unwrap(); + let mut got = Vec::new(); + html_into(&mut Stepper::new(&cache), &mut got); + assert_eq!(got, html(&data), "html read in blocks, {what:?}"); + got.clear(); + text_into(&mut Stepper::new(&cache), &mut got); + assert_eq!(got, text(&data), "text read in blocks, {what:?}"); + got.clear(); + javascript_into(&mut Stepper::new(&cache), &mut got); + assert_eq!(got, javascript(&data), "javascript read in blocks, {what:?}"); + } + } } diff --git a/crates/exav-core/src/patterns.rs b/crates/exav-core/src/patterns.rs index 3fee3512..e4d248bc 100644 --- a/crates/exav-core/src/patterns.rs +++ b/crates/exav-core/src/patterns.rs @@ -1,71 +1,4 @@ -//! Literal byte-pattern matching via a single streaming Aho-Corasick -//! automaton (all patterns matched in one pass, any file size). - -use std::sync::OnceLock; - -use aho_corasick::{AhoCorasick, MatchKind}; -use serde::{Deserialize, Serialize}; - -use crate::hexsig::{parse_ndb_body, NdbPattern}; - -/// A named literal byte pattern. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Pattern { - pub name: String, - pub bytes: Vec, - /// Whether this pattern came from an unofficial (non-`.cvd`) database. The - /// clean name is stored; the `.UNOFFICIAL` suffix is applied at report time - /// (compat mode). Defaults to `false` so older databases load as official. - #[serde(default)] - pub unofficial: bool, -} - -impl Pattern { - /// A new official (non-suffixed) pattern. - pub fn new(name: impl Into, bytes: impl Into>) -> Self { - Self { - name: name.into(), - bytes: bytes.into(), - unofficial: false, - } - } - - /// A new pattern carrying explicit `unofficial` provenance. - pub fn with_prov(name: impl Into, bytes: impl Into>, unofficial: bool) -> Self { - Self { - name: name.into(), - bytes: bytes.into(), - unofficial, - } - } -} - -/// Whether an `.ndb` signature's `Target`/`Offset` columns make it safe to add -/// to the streaming literal set — i.e. it is valid at any offset in any file -/// type. Only `Target:0` (any type) combined with `Offset:*` (any offset) -/// qualifies; anything else carries a constraint the constraint-free streaming -/// matcher cannot honor. Blank columns (malformed line) are treated as -/// constrained (excluded) so uncertainty never widens the match. -fn stream_safe(target: &str, offset: &str) -> bool { - target.trim() == "0" && offset.trim() == "*" -} - -/// A set of literal patterns for the streaming path. The Aho-Corasick -/// automaton is built lazily on first use: in-memory (file) scans go through -/// the full engine, which already covers these literals, so only stdin/pipe -/// scans pay the (one-time) construction — keeping cold start cheap. -pub struct PatternSet { - ac: OnceLock, - /// The source patterns; the automaton is (re)built from these on demand. - pub(crate) src: Vec, - /// Number of source signatures skipped, e.g. wildcard `.ndb` bodies. - /// How many `.ndb` lines the STREAMING set could not carry. Most are - /// wildcard bodies, which the full buffered engine loads and matches - /// perfectly well — so this is streaming-set bookkeeping, **not** a - /// coverage gap, and must not be reported to users as skipped signatures. - /// See `Db::unsupported_count`. - pub unsupported: usize, -} +//! exav's own literal signature format, `Name=HEX` (`.db`). /// The EICAR anti-virus test string, assembled at runtime. /// @@ -74,138 +7,21 @@ pub struct PatternSet { /// literal. pub use exav_unpack::eicar; -impl PatternSet { - /// Build from literal patterns. `unsupported` records how many source - /// signatures could not be represented (e.g. wildcard `.ndb` sigs). The - /// automaton itself is constructed lazily (see `PatternSet::ac`). - pub fn build(patterns: &[Pattern], unsupported: usize) -> Result { - Ok(Self { - ac: OnceLock::new(), - src: patterns.to_vec(), - unsupported, - }) - } - - /// The streaming automaton, built on first use (Standard match kind is - /// required for streaming search; detection only needs presence). - pub(crate) fn ac(&self) -> &AhoCorasick { - self.ac.get_or_init(|| { - let raw: Vec<&[u8]> = self.src.iter().map(|p| p.bytes.as_slice()).collect(); - AhoCorasick::builder() - .match_kind(MatchKind::Standard) - .build(&raw) - .expect("streaming pattern automaton builds") - }) - } - - /// `(clean_name, unofficial)` for a pattern index — the report layer applies - /// the `.UNOFFICIAL` suffix in compat mode. - pub(crate) fn name_prov(&self, i: usize) -> (&str, bool) { - (&self.src[i].name, self.src[i].unofficial) - } - - /// A set containing only the EICAR test signature. - pub fn builtin() -> Self { - Self::build(&[Pattern::new("Eicar-Test-Signature", eicar().to_vec())], 0) - .expect("builtin pattern set builds") - } - - /// Parse `.ndb` lines (`Name:Type:Offset:HexSig[:MinFL[:MaxFL]]`) into - /// literal patterns, returning the patterns and the number of bodies - /// skipped because they use wildcards. - pub fn parse_ndb(text: &str) -> (Vec, usize) { - Self::parse_ndb_prov(text, false) - } - - /// As [`PatternSet::parse_ndb`], tagging each parsed literal with `unofficial` - /// provenance (set for `.ndb` files from a non-`.cvd` database). - pub fn parse_ndb_prov(text: &str, unofficial: bool) -> (Vec, usize) { - let mut patterns = Vec::new(); - let mut unsupported = 0usize; - for line in text.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let mut parts = line.splitn(4, ':'); - let name = parts.next().unwrap_or(""); - let target = parts.next().unwrap_or(""); - let offset = parts.next().unwrap_or(""); - let body = match parts.next() { - Some(b) => b, - None => { - unsupported += 1; - continue; - } - }; - // The streaming literal set matches over raw, unbuffered bytes where - // no PE layout and no per-buffer file type exist, so it may only - // carry signatures that are valid *anywhere, in any file type*: - // `Target:0` (any) with `Offset:*` (any). A type- or offset- - // constrained signature — e.g. a short pattern pinned to a PE entry - // point (`1:EP+0,64:5746c3`) — would false-positive if matched - // unanchored here, since the constraint can't be checked. Such sigs - // are still loaded and correctly enforced by the full (buffered) - // engine, so they are simply excluded from the streaming set rather - // than counted unsupported. - if !stream_safe(target, offset) { - continue; - } - match parse_ndb_body(body) { - NdbPattern::Literal(bytes) if !bytes.is_empty() => { - patterns.push(Pattern::with_prov(name, bytes, unofficial)); - } - _ => unsupported += 1, - } - } - (patterns, unsupported) - } - - /// Parse a minimal `Name=HEX` database (exav's own simple format). - pub fn parse_simple(text: &str) -> Result, String> { - let mut patterns = Vec::new(); - for (i, line) in text.lines().enumerate() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let (name, hex) = line - .split_once('=') - .ok_or_else(|| format!("line {}: expected Name=HEX", i + 1))?; - let bytes = - crate::hexsig::decode_hex(hex).map_err(|e| format!("line {}: {e}", i + 1))?; - patterns.push(Pattern::new(name.trim(), bytes)); - } - Ok(patterns) - } - - /// Convenience: build a set directly from `.ndb` text. - pub fn from_ndb(text: &str) -> Result { - let (patterns, unsupported) = Self::parse_ndb(text); - if patterns.is_empty() { - return Err(format!( - "no literal .ndb patterns (skipped {unsupported} unsupported)" - )); - } - Self::build(&patterns, unsupported) - } - - /// Convenience: build a set directly from `Name=HEX` text. - pub fn from_simple_db(text: &str) -> Result { - let patterns = Self::parse_simple(text)?; - if patterns.is_empty() { - return Err("no patterns".into()); +/// Parse a `Name=HEX` database into `(name, bytes)` pairs. +pub fn parse_simple(text: &str) -> Result)>, String> { + let mut patterns = Vec::new(); + for (i, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; } - Self::build(&patterns, 0) - } - - pub fn len(&self) -> usize { - self.src.len() - } - - pub fn is_empty(&self) -> bool { - self.src.is_empty() + let (name, hex) = line + .split_once('=') + .ok_or_else(|| format!("line {}: expected Name=HEX", i + 1))?; + let bytes = crate::hexsig::decode_hex(hex).map_err(|e| format!("line {}: {e}", i + 1))?; + patterns.push((name.trim().to_string(), bytes)); } + Ok(patterns) } #[cfg(test)] @@ -213,39 +29,15 @@ mod tests { use super::*; #[test] - fn ndb_loads_literals_and_counts_wildcards() { - let db = "Sig.A:0:*:deadbeef\nSig.B:0:*:dead*beef\nSig.C:0:*:cafe\n"; - let set = PatternSet::from_ndb(db).unwrap(); - assert_eq!(set.len(), 2); // A and C literal; B wildcard skipped - assert_eq!(set.unsupported, 1); - } - - #[test] - fn stream_set_excludes_constrained_sigs() { - // Regression: the streaming literal set matches over raw bytes with no - // file-type/offset context, so a `Target`- or `Offset`-constrained sig - // (e.g. the real `Win.Spyware.Zbot-1290:1:EP+0,64:5746c3` — a 3-byte - // pattern pinned to a PE entry point) must NOT enter it, or it - // false-positives unanchored on any file that happens to contain those - // bytes. Only `Target:0` + `Offset:*` (valid anywhere) qualifies. - let db = concat!( - "Generic.Any:0:*:deadbeef\n", // kept: any type, any offset - "Pe.Ep.Short:1:EP+0,64:5746c3\n", // dropped: PE-only, entry-point - "Any.AtOffset:0:512:cafebabe\n", // dropped: absolute offset - "Elf.Anywhere:6:*:0badf00d\n", // dropped: ELF target only + fn simple_db_lines_parse() { + let got = parse_simple("# comment\nSig.A =4142\n\nSig.B=ff\n").unwrap(); + assert_eq!( + got, + vec![ + ("Sig.A".to_string(), b"AB".to_vec()), + ("Sig.B".to_string(), vec![0xff]) + ] ); - let (patterns, _unsupported) = PatternSet::parse_ndb(db); - let names: Vec<&str> = patterns.iter().map(|p| p.name.as_str()).collect(); - assert_eq!(names, ["Generic.Any"], "only the unconstrained sig is stream-safe"); - } - - #[test] - fn stream_safe_predicate() { - assert!(stream_safe("0", "*")); - assert!(!stream_safe("1", "*")); // PE target - assert!(!stream_safe("0", "EP+0")); // entry-point anchored - assert!(!stream_safe("0", "0")); // absolute offset 0 - assert!(!stream_safe("", "")); // malformed → treated as constrained + assert!(parse_simple("no separator").is_err()); } } - diff --git a/crates/exav-core/src/pe.rs b/crates/exav-core/src/pe.rs index 5280fb4c..8aa236e6 100644 --- a/crates/exav-core/src/pe.rs +++ b/crates/exav-core/src/pe.rs @@ -6,6 +6,7 @@ use goblin::pe::PE; use md5::{Digest, Md5}; +use crate::byte_source::ByteSource; use crate::hexsig::encode_hex; /// Structural facts extracted from a PE file. @@ -91,7 +92,7 @@ pub fn analyze(data: &[u8]) -> Option { .unwrap_or(imp.dll) .to_ascii_lowercase(); // Imports by ordinal contribute "dll.ord" (the imphash/Mandiant - // convention), not the function name — goblin renders an ordinal + // convention), not the function name; goblin renders an ordinal // import's name as "ORDINAL ". Dropping these (or hashing the // rendered string) would change the imphash and miss `.imp` sigs. if imp.name.starts_with("ORDINAL ") { @@ -132,7 +133,15 @@ pub fn shannon_entropy(data: &[u8]) -> f64 { for &b in data { counts[b as usize] += 1; } - let len = data.len() as f64; + entropy_of_counts(&counts, data.len()) +} + +/// [`shannon_entropy`] of `len` bytes, from how many times each value occurs. +pub(crate) fn entropy_of_counts(counts: &[u64; 256], len: usize) -> f64 { + if len == 0 { + return 0.0; + } + let len = len as f64; let mut h = 0.0f64; for &c in counts.iter() { if c > 0 { @@ -157,72 +166,327 @@ pub struct PeLayout { pub entry: Option, /// File offset (PointerToRawData) of each section, in order. pub section_rawptrs: Vec, - /// SizeOfRawData of each section, in the same order — the section's extent + /// SizeOfRawData of each section, in the same order: the section's extent /// in *file* space, which is what an `SEn:` offset is measured against. pub section_rawsizes: Vec, - /// File offsets of the `VS_VERSION_INFO` string keys — the anchor set a + /// File offsets of the `VS_VERSION_INFO` string keys, the anchor set a /// `VI:` offset matches against. Empty for a PE with no version resource. pub version_info: Vec, } -/// Extract [`PeLayout`] from a PE. `None` if `data` is not a parseable PE. -/// Whether a file that *claims* to be an executable fails to parse as one — -/// ClamAV's `Heuristics.Broken.Executable`, reported under `--alert-broken`. +/// Whether a file that *claims* to be an executable describes a layout no +/// loader could map: ClamAV's `Heuristics.Broken.Executable`, reported under +/// `--alert-broken`. +/// +/// The signal is the contradiction: something typed as a PE, ELF or Mach-O by +/// its magic, whose headers then do not hold together. Only the headers a +/// loader reads to map the image are judged. What they point at (imports, +/// resources, debug data, signatures, section names) is malformed routinely in +/// software that runs, and a full parse that fails on any of it made this the +/// largest false-positive class exav had: DLLs with a non-UTF-8 export name, +/// resource directories pointing nowhere, Android libraries with junk section +/// headers. /// -/// The signal is the contradiction, not the failure: something typed as a PE, -/// ELF or Mach-O by its magic, whose headers then do not hold together. Ordinary -/// software does not ship malformed executables; malformed ones are produced by -/// truncation, by corruption, and by droppers that lean on a loader being more -/// forgiving than a parser. +/// Each condition was measured against clamscan 1.5.4 on headers mutated one +/// field at a time; see `tests/suites/broken_executable.rs`. /// -/// Deliberately narrow — it only answers for input that already carries an +/// Deliberately narrow: it only answers for input that already carries an /// executable magic, so an arbitrary file can never be "a broken executable". pub fn looks_broken(data: &[u8]) -> bool { - if data.starts_with(b"MZ") { - // A bare `MZ` with no reachable PE header is a DOS executable, which is - // not broken — just old. Only claim breakage when the file points at a - // PE header and that header does not parse. - let Some(e) = data - .get(0x3c..0x40) - .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize) - else { - return false; + looks_broken_in(&data) +} + +/// [`looks_broken`] over an object that need not be held in memory. +pub(crate) fn looks_broken_in(src: &dyn ByteSource) -> bool { + let head = src.window(0, 4); + if head.starts_with(b"MZ") { + pe_broken(src) + } else if head.starts_with(b"\x7fELF") { + elf_broken(src) + } else { + macho_broken(src) + } +} + +/// Header fields read through one cached window, so a walk over many small +/// records does not cost a read of the object per field. +struct Fields<'a> { + src: &'a dyn ByteSource, + at: usize, + buf: std::borrow::Cow<'a, [u8]>, + be: bool, +} + +impl<'a> Fields<'a> { + fn new(src: &'a dyn ByteSource) -> Self { + Fields { + src, + at: 0, + buf: std::borrow::Cow::Borrowed(&[]), + be: false, + } + } + + /// `len` bytes at `off`, or `None` when the object ends first. + fn get(&mut self, off: u64, len: usize) -> Option<&[u8]> { + let off = usize::try_from(off).ok()?; + let end = off.checked_add(len)?; + if end > self.src.len() { + return None; + } + if off < self.at || end > self.at + self.buf.len() { + self.at = off; + self.buf = self.src.window(off, len.max(64 * 1024)); + } + self.buf.get(off - self.at..end - self.at) + } + + fn u16(&mut self, off: u64) -> Option { + let be = self.be; + self.get(off, 2).map(|b| read_uint(b, be) as u16) + } + + fn u32(&mut self, off: u64) -> Option { + let be = self.be; + self.get(off, 4).map(|b| read_uint(b, be) as u32) + } + + /// A 4- or 8-byte field, by the file's class. + fn word(&mut self, off: u64, wide: bool) -> Option { + let be = self.be; + self.get(off, if wide { 8 } else { 4 }).map(|b| read_uint(b, be)) + } +} + +/// An unsigned integer of up to 8 bytes in the given byte order. +fn read_uint(b: &[u8], be: bool) -> u64 { + let fold = |v: u64, &x: &u8| (v << 8) | u64::from(x); + if be { + b.iter().fold(0, fold) + } else { + b.iter().rev().fold(0, fold) + } +} + +/// `x` rounded up to a multiple of `a`, or `x` itself when `a` is 0. +fn round_up(x: u64, a: u64) -> u64 { + if a == 0 { + x + } else { + x.div_ceil(a) * a + } +} + +fn pe_broken(src: &dyn ByteSource) -> bool { + let mut f = Fields::new(src); + let n = src.len() as u64; + // No PE header is a DOS program, and a COFF header cut short is too little + // to judge: neither describes a layout. + let Some(e) = f.u32(0x3c).map(u64::from) else { + return false; + }; + let Some(coff) = f.get(e, 24) else { + return false; + }; + if coff[..4] != *b"PE\0\0" { + return false; + } + let nsec = u64::from(u16::from_le_bytes([coff[6], coff[7]])); + let optsz = u64::from(u16::from_le_bytes([coff[20], coff[21]])); + let opt = e + 24; + // Any magic but PE32+ is read as PE32. + let fixed = if f.u16(opt) == Some(0x20b) { 112 } else { 96 }; + if optsz < fixed { + return true; + } + let Some(h) = f.get(opt, fixed as usize) else { + return true; + }; + let le32 = |at: usize| u64::from(u32::from_le_bytes([h[at], h[at + 1], h[at + 2], h[at + 3]])); + let (entry, salign, falign, headers) = (le32(16), le32(32), le32(36), le32(60)); + let native = u16::from_le_bytes([h[68], h[69]]) == 1; + let dirs = le32(fixed as usize - 4).min(16) * 8; + if optsz < fixed + dirs { + return true; + } + // The data directories cut short: incomplete rather than wrong. + if opt + fixed + dirs > n { + return false; + } + if nsec == 0 { + return true; + } + // Drivers (the native subsystem) may use any alignment. + let aligned = falign != 0 && falign % 0x200 == 0 && salign != 0 && salign % 0x1000 == 0; + if salign == 0 || !(native || aligned) { + return true; + } + let Some(table) = f.get(opt + optsz, (nsec * 40) as usize) else { + return true; + }; + // (VirtualSize, VirtualAddress, SizeOfRawData, PointerToRawData). A + // section whose raw data starts at or past the end of the file is left out + // of the layout, as though the table did not list it. + let sections = || { + table + .as_chunks::<40>() + .0 + .iter() + .map(|s| { + let at = |i: usize| u64::from(u32::from_le_bytes([s[i], s[i + 1], s[i + 2], s[i + 3]])); + (at(8), at(12), at(16), at(20)) + }) + .filter(|&(_, _, raw_size, raw_ptr)| raw_size == 0 || raw_ptr < n) + }; + let Some((_, first, _, _)) = sections().next() else { + return true; + }; + // Laid out in memory right after the headers, each section where the one + // before it ends. + if first != round_up(headers, salign) { + return true; + } + let mut end = first; + for (vsize, va, raw_size, _) in sections() { + if va != end { + return true; + } + end = va + round_up(if vsize != 0 { vsize } else { raw_size }, salign); + } + // The entry point has to land on bytes the file holds. + if entry < first { + return entry >= n; + } + let Some((_, va, _, raw_ptr)) = sections().find(|&(_, va, raw_size, _)| { + entry >= va && entry - va < round_up(raw_size, falign) + }) else { + return true; + }; + let raw_start = raw_ptr.checked_div(falign).map_or(raw_ptr, |q| q * falign); + raw_start + (entry - va) >= n +} + +fn elf_broken(src: &dyn ByteSource) -> bool { + let mut f = Fields::new(src); + let Some(ident) = f.get(0, 16) else { + return false; + }; + let wide = match ident[4] { + 1 => false, + 2 => true, + _ => return true, + }; + // Anything but little-endian is read as big-endian. + f.be = ident[5] != 1; + let (header, ph_size, sh_size) = if wide { (64, 56, 64) } else { (52, 32, 40) }; + // A header cut short is too little to judge. + if f.get(0, header).is_none() { + return false; + } + let w = if wide { 8 } else { 4 }; + let (Some(entry), Some(phoff), Some(shoff)) = + (f.word(24, wide), f.word(24 + w, wide), f.word(24 + 2 * w, wide)) + else { + return false; + }; + let fields = 28 + 3 * w; // e_ehsize, then the table geometry + let geometry = [2, 4, 6, 8].map(|i| f.u16(fields + i).map(u64::from)); + let [Some(ph_entsize), Some(phnum), Some(sh_entsize), Some(shnum)] = geometry else { + return false; + }; + if phnum > 0 { + if phnum > 128 || ph_entsize != ph_size { + return true; + } + if f.get(phoff, (phnum * ph_size) as usize).is_none() { + return true; + } + // Inside some segment's memory image, whatever its type. 0 is no entry. + let (vaddr_at, memsz_at) = if wide { (16, 40) } else { (8, 20) }; + let mapped = (0..phnum).any(|i| { + let p = phoff + i * ph_size; + let (Some(vaddr), Some(memsz)) = (f.word(p + vaddr_at, wide), f.word(p + memsz_at, wide)) else { + return false; + }; + entry >= vaddr && entry - vaddr < memsz + }); + if entry != 0 && !mapped { + return true; + } + } + // A zero entry size is a stripped section table, which is reported under + // its own name (see `elf_section_headers_stripped`). + if sh_entsize == 0 { + return false; + } + sh_entsize != sh_size || (shnum > 0 && f.get(shoff, (shnum * sh_size) as usize).is_none()) +} + +fn macho_broken(src: &dyn ByteSource) -> bool { + let mut f = Fields::new(src); + let n = src.len() as u64; + // Thin images only: the universal ("fat") magic is a container, and a Java + // class shares its bytes. + let Some(magic) = f.get(0, 4) else { + return false; + }; + let (wide, be) = match magic { + [0xCE, 0xFA, 0xED, 0xFE] => (false, false), + [0xCF, 0xFA, 0xED, 0xFE] => (true, false), + [0xFE, 0xED, 0xFA, 0xCE] => (false, true), + [0xFE, 0xED, 0xFA, 0xCF] => (true, true), + _ => return false, + }; + f.be = be; + let header: u64 = if wide { 32 } else { 28 }; + if n < header { + return false; + } + let (Some(cpu), Some(ncmds)) = (f.u32(4), f.u32(16).map(u64::from)) else { + return false; + }; + // Every command moves the walk at least 8 bytes on, so more than the file + // can hold is a read past its end. This also bounds the walk. + if ncmds == 0 || ncmds * 8 > n - header { + return true; + } + let (segment, segment_size, section_size) = if wide { (0x19, 72, 80) } else { (0x1, 56, 68) }; + // Thread state is read in full on these CPUs (i386, PowerPC, PowerPC 64), + // and skipped on the others. + let thread_state = match cpu { + 7 => Some(64), + 18 => Some(160), + 0x0100_0012 => Some(312), + _ => None, + }; + let mut at = header; + for _ in 0..ncmds { + let (Some(cmd), Some(size)) = (f.u32(at), f.u32(at + 4)) else { + return true; }; - if data.get(e..e + 4) != Some(b"PE\0\0") { - return false; - } - // Parse WITHOUT the attribute-certificate table. That table is an - // Authenticode signature living in the overlay: it is not mapped into the - // image and the loader never reads it, so a truncated or garbage - // signature blob does not stop the binary running — and it is routine in - // malware, and in anything signed and then tampered with. - // - // Including it made this the largest false-positive class exav had. - // Measured on the flagged files: 47 of 53 had entirely sound headers — - // entry point inside a section, section extents within the file, no - // overlaps, sane NumberOfSections and SizeOfImage — and were reported - // broken purely because an appended signature would not parse. - let mut opts = goblin::pe::options::ParseOptions::default(); - opts.parse_attribute_certificates = false; - return PE::parse_with_opts(data, &opts).is_err(); - } - if data.starts_with(b"\x7fELF") { - return goblin::elf::Elf::parse(data).is_err(); - } - // Thin Mach-O magics only; the universal/"fat" magic is a container, and a - // Java class shares its byte pattern. - const MACHO: [[u8; 4]; 4] = [ - [0xCF, 0xFA, 0xED, 0xFE], - [0xFE, 0xED, 0xFA, 0xCF], - [0xCE, 0xFA, 0xED, 0xFE], - [0xFE, 0xED, 0xFA, 0xCE], - ]; - if MACHO.iter().any(|m| data.starts_with(m)) { - return goblin::mach::MachO::parse(data, 0).is_err(); + if cmd == segment { + // A segment is read by its structure, whatever its cmdsize says. + let Some(nsects) = f.u32(at + segment_size - 8) else { + return true; + }; + at += segment_size + u64::from(nsects) * section_size; + if at > n { + return true; + } + continue; + } + if matches!(cmd, 0x4 | 0x5) { + // LC_THREAD, LC_UNIXTHREAD: flavor, count, then the registers. + if thread_state.is_some_and(|len| at + 16 + len > n) { + return true; + } + } + at += u64::from(size).max(8); } false } +/// Extract [`PeLayout`] from a PE. `None` if `data` is not a parseable PE. pub fn layout(data: &[u8]) -> Option { let pe = PE::parse(data).ok()?; let mut section_rawptrs = Vec::with_capacity(pe.sections.len()); @@ -268,7 +532,7 @@ pub struct BcPeSection { #[derive(Debug, Clone, Default)] pub struct BcPe { pub sections: Vec, - /// `SizeOfHeaders` — RVAs below this map directly to file offsets. + /// `SizeOfHeaders`: RVAs below this map directly to file offsets. pub hdr_size: u32, /// Byte image of `cli_pe_hook_data` (the `__clambc_pedata` global). pub pedata: Vec, @@ -396,12 +660,14 @@ fn build_pedata(pe: &PE, bc: &BcPe, file_size: usize) -> Vec { b } +/// The raw array, not `dirs()`: that skips absent entries, so enumerating it +/// shifts the slots, and it panics on the reserved 16th entry. fn put_dirs(b: &mut [u8], base: usize, oh: &goblin::pe::optional_header::OptionalHeader) { - for (i, dd) in oh.data_directories.dirs().enumerate().take(16) { + for (i, dd) in oh.data_directories.data_directories.iter().enumerate().take(16) { + let Some((_, dd)) = dd else { continue }; let off = base + i * 8; - // `dd` is (index, &DataDirectory); write VirtualAddress + Size. - b[off..off + 4].copy_from_slice(&dd.1.virtual_address.to_le_bytes()); - b[off + 4..off + 8].copy_from_slice(&dd.1.size.to_le_bytes()); + b[off..off + 4].copy_from_slice(&dd.virtual_address.to_le_bytes()); + b[off + 4..off + 8].copy_from_slice(&dd.size.to_le_bytes()); } } @@ -409,65 +675,254 @@ fn put_dirs(b: &mut [u8], base: usize, oh: &goblin::pe::optional_header::Optiona /// crafted with many `MZ` markers). const MAX_EMBEDDED_PE: usize = 16; -/// Offsets of PE images embedded at a **non-zero** offset (an `MZ` whose -/// `e_lfanew` points to a `PE\0\0` header within bounds). File-infectors and -/// droppers append/embed executables this way; scanning them is needed to match -/// signatures (incl. section hashes) the same way ClamAV does. -/// The image at offset 0, if any, is handled by the normal scan and skipped. -pub fn embedded_pe_offsets(data: &[u8]) -> Vec { - let mut out = Vec::new(); - // Need at least an MZ + e_lfanew header; also keeps `&data[from..]` (from=1) - // from panicking on an empty/tiny buffer. - if data.len() < 0x40 { - return out; - } - let mut from = 1usize; // skip a top-level image at offset 0 - while out.len() < MAX_EMBEDDED_PE { - let Some(rel) = memchr::memmem::find(&data[from..], b"MZ") else { +/// Most archive candidates carved from one buffer. +const MAX_EMBEDDED_ARCHIVES: usize = 32; + +/// What carving finds embedded at a non-zero offset in an object: validated +/// PE, ELF and Mach-O images, and archive candidates found by their magic, each +/// in ascending order and within its cap. +#[derive(Default)] +pub(crate) struct Embedded { + pub(crate) pe: Vec, + pub(crate) elf: Vec, + pub(crate) macho: Vec, + pub(crate) archives: Vec, +} + +/// The magics carving looks for, and what each marks. +#[derive(Clone, Copy)] +enum Magic { + Pe, + Elf, + /// A thin Mach-O, big-endian or not. + MachoThin(bool), + MachoFat, + Archive, +} + +const CARVE_MAGICS: [(&[u8], Magic); 14] = [ + (b"MZ", Magic::Pe), + (b"\x7fELF", Magic::Elf), + (&[0xCE, 0xFA, 0xED, 0xFE], Magic::MachoThin(false)), // MH_MAGIC (32-bit), LE host + (&[0xCF, 0xFA, 0xED, 0xFE], Magic::MachoThin(false)), // MH_MAGIC_64 (64-bit), LE host + (&[0xFE, 0xED, 0xFA, 0xCE], Magic::MachoThin(true)), // MH_CIGAM (32-bit), BE host + (&[0xFE, 0xED, 0xFA, 0xCF], Magic::MachoThin(true)), // MH_CIGAM_64 (64-bit), BE host + (b"\xCA\xFE\xBA\xBE", Magic::MachoFat), + (b"PK\x03\x04", Magic::Archive), // ZIP / OOXML / APK / JAR + (b"\x1f\x8b\x08", Magic::Archive), // GZIP + (b"BZh", Magic::Archive), // BZIP2 + (b"\xfd7zXZ\x00", Magic::Archive), // XZ + (b"7z\xbc\xaf\x27\x1c", Magic::Archive), // 7-Zip + (b"Rar!\x1a\x07", Magic::Archive), // RAR + (b"MSCF", Magic::Archive), // CAB +]; + +/// Everything carving looks for in `data`, found in one read of it, a window +/// at a time: the image at offset 0 is the object's own, and each candidate +/// is validated as it is met. A kind stops being looked for at its cap. +pub(crate) fn embedded_in(data: &dyn ByteSource) -> Embedded { + let mut carving = Carving::new(); + let len = data.len(); + let mut at = 0; + while at < len && !carving.full() { + let w = data.window(at, (len - at).min(crate::byte_source::CHUNK)); + if w.is_empty() { break; - }; - let off = from + rel; - from = off + 1; - if off + 0x40 > data.len() { + } + let last = at + w.len() >= len; + let owned = if last { w.len() } else { w.len().saturating_sub(Carving::OVERLAP).max(1) }; + carving.feed(data, at, &w, owned); + if last { break; } - let e_lfanew = u32::from_le_bytes([ - data[off + 0x3c], - data[off + 0x3d], - data[off + 0x3e], - data[off + 0x3f], - ]) as usize; - // e_lfanew is relative to the MZ; the PE header must be in-bounds. - let Some(pe) = off.checked_add(e_lfanew) else { - continue; - }; - if e_lfanew < 0x40 || pe + 4 > data.len() { - continue; + at += owned; + } + carving.found +} + +/// The search [`embedded_in`] makes, fed a window at a time by a read of the +/// object that serves other searches too. +pub(crate) struct Carving { + finders: Vec>, + hits: Vec<(usize, Magic)>, + pub(crate) found: Embedded, +} + +impl Carving { + /// Bytes each window must run into the next, so a magic across the seam + /// is seen whole. + pub(crate) const OVERLAP: usize = { + let mut longest = 0; + let mut i = 0; + while i < CARVE_MAGICS.len() { + if CARVE_MAGICS[i].0.len() > longest { + longest = CARVE_MAGICS[i].0.len(); + } + i += 1; } - if &data[pe..pe + 4] == b"PE\0\0" && valid_pe_coff(data, pe) { - out.push(off); + longest - 1 + }; + + pub(crate) fn new() -> Self { + Carving { + finders: CARVE_MAGICS.iter().map(|(m, _)| memchr::memmem::Finder::new(m)).collect(), + hits: Vec::new(), + found: Embedded::default(), } } - out + + /// Whether every kind is at its cap, so the rest of the object has + /// nothing more to add. + pub(crate) fn full(&self) -> bool { + let f = &self.found; + f.pe.len() >= MAX_EMBEDDED_PE + && f.elf.len() >= MAX_EMBEDDED_PE + && f.macho.len() >= MAX_EMBEDDED_PE + && f.archives.len() >= MAX_EMBEDDED_ARCHIVES + } + + /// Search the window `w` at `at` in `data` for the magics that start in + /// its first `owned` bytes; those after start in the next window, which + /// sees them whole. Candidates are validated against `data`. + pub(crate) fn feed(&mut self, data: &dyn ByteSource, at: usize, w: &[u8], owned: usize) { + let found = &mut self.found; + let hits = &mut self.hits; + hits.clear(); + for (f, (_, kind)) in self.finders.iter().zip(CARVE_MAGICS) { + hits.extend(f.find_iter(w).take_while(|&p| p < owned).map(|p| (at + p, kind))); + } + hits.sort_unstable_by_key(|&(off, _)| off); + for &(off, kind) in hits.iter() { + if off == 0 { + continue; // the object's own image or archive + } + let (list, cap, ok) = match kind { + Magic::Pe => (&mut found.pe, MAX_EMBEDDED_PE, pe_at(data, off)), + Magic::Elf => (&mut found.elf, MAX_EMBEDDED_PE, elf_at(data, off)), + Magic::MachoThin(be) => (&mut found.macho, MAX_EMBEDDED_PE, macho_thin_at(data, off, be)), + Magic::MachoFat => (&mut found.macho, MAX_EMBEDDED_PE, macho_fat_at(data, off)), + Magic::Archive => (&mut found.archives, MAX_EMBEDDED_ARCHIVES, true), + }; + if ok && list.len() < cap { + list.push(off); + } + } + } +} + +/// An `MZ` at `off` whose `e_lfanew` points to a sane `PE\0\0` header. +fn pe_at(data: &dyn ByteSource, off: usize) -> bool { + if off + 0x40 > data.len() { + return false; + } + let Some(e_lfanew) = le_u32_at(data, off + 0x3c) else { + return false; + }; + let e_lfanew = e_lfanew as usize; + // e_lfanew is relative to the MZ; the PE header must be in-bounds. + let Some(pe) = off.checked_add(e_lfanew) else { + return false; + }; + if e_lfanew < 0x40 || pe + 4 > data.len() { + return false; + } + data.window(pe, 4)[..] == *b"PE\0\0" && valid_pe_coff(data, pe) +} + +/// A validated ELF identification header at `off`: class, data and version, +/// and a plausible `e_type`. +fn elf_at(data: &dyn ByteSource, off: usize) -> bool { + let h = data.window(off, 18); + if h.len() != 18 { + return false; + } + let class = h[4]; // EI_CLASS: 1=32-bit, 2=64-bit + let endian = h[5]; // EI_DATA: 1=LE, 2=BE + let version = h[6]; // EI_VERSION: 1 + if !matches!(class, 1 | 2) || !matches!(endian, 1 | 2) || version != 1 { + return false; + } + // e_type at offset 16 (2 bytes, endianness per EI_DATA): REL/EXEC/DYN/CORE. + let etype = if endian == 1 { + u16::from_le_bytes([h[16], h[17]]) + } else { + u16::from_be_bytes([h[16], h[17]]) + }; + matches!(etype, 1..=4) +} + +fn rd_u32(h: &[u8], o: usize, be: bool) -> u32 { + let b = [h[o], h[o + 1], h[o + 2], h[o + 3]]; + if be { + u32::from_be_bytes(b) + } else { + u32::from_le_bytes(b) + } +} + +/// A thin Mach-O header at `off` with a sane `filetype` and `ncmds`. +fn macho_thin_at(data: &dyn ByteSource, off: usize, be: bool) -> bool { + let h = data.window(off, 28); + if h.len() != 28 { + return false; + } + let filetype = rd_u32(&h, 12, be); // MH_OBJECT..MH_KEXT_BUNDLE + let ncmds = rd_u32(&h, 16, be); + (1..=11).contains(&filetype) && (1..=10_000).contains(&ncmds) +} + +/// A fat Mach-O header at `off`: a sane arch count and a real `cputype` +/// first, which a Java `.class` (the same magic) does not have. +fn macho_fat_at(data: &dyn ByteSource, off: usize) -> bool { + let h = data.window(off, 28); + if h.len() != 28 { + return false; + } + let nfat = rd_u32(&h, 4, true); + let cputype0 = rd_u32(&h, 8, true); + (1..=64).contains(&nfat) && is_macho_cputype(cputype0) +} + +/// Offsets of PE images embedded at a **non-zero** offset (an `MZ` whose +/// `e_lfanew` points to a `PE\0\0` header within bounds). File-infectors and +/// droppers append/embed executables this way; scanning them is needed to match +/// signatures (incl. section hashes) the same way ClamAV does. +/// The image at offset 0, if any, is handled by the normal scan and skipped. +pub fn embedded_pe_offsets(data: &[u8]) -> Vec { + embedded_pe_offsets_in(&data) +} + +/// [`embedded_pe_offsets`] over an object that need not be held in memory. +pub(crate) fn embedded_pe_offsets_in(data: &dyn ByteSource) -> Vec { + embedded_in(data).pe +} + +/// The little-endian `u32` at `at`, if the object holds it. +fn le_u32_at(data: &dyn ByteSource, at: usize) -> Option { + Some(u32::from_le_bytes(data.window(at, 4)[..].try_into().ok()?)) } /// A carved `MZ…PE\0\0` region is treated as a scannable embedded PE only when /// its COFF + optional header pass basic structural sanity. Without this, any /// bytes in a compressed/encrypted stream that merely contain the `MZ` and -/// `PE\0\0` markers get carved and scanned as a bogus PE — where a short +/// `PE\0\0` markers get carved and scanned as a bogus PE, where a short /// entry-point-anchored signature can false-positive on the garbage, since /// [`layout`] still computes an "entry" from the junk section table. ClamAV /// gates embedded-PE extraction the same way. `pe` is the offset of `PE\0\0`. -fn valid_pe_coff(data: &[u8], pe: usize) -> bool { +fn valid_pe_coff(data: &dyn ByteSource, pe: usize) -> bool { // 20-byte COFF header after `PE\0\0`, then the optional-header magic. if pe + 26 > data.len() { return false; } - let rd16 = |o: usize| u16::from_le_bytes([data[o], data[o + 1]]); - let machine = rd16(pe + 4); - let n_sections = rd16(pe + 6); - let opt_size = rd16(pe + 20); - let magic = rd16(pe + 24); + let h = data.window(pe, 26); + if h.len() != 26 { + return false; + } + let rd16 = |o: usize| u16::from_le_bytes([h[o], h[o + 1]]); + let machine = rd16(4); + let n_sections = rd16(6); + let opt_size = rd16(20); + let magic = rd16(24); // Machine must be set, section count within the PE-spec maximum (96), an // optional header must be present, and its magic must be PE32 (0x10b) or // PE32+ (0x20b). Random data clears all four only vanishingly rarely. @@ -482,37 +937,12 @@ fn valid_pe_coff(data: &[u8], pe: usize) -> bool { /// `e_type`) so a coincidental `\x7fELF` byte run is not carved. The image at /// offset 0, if any, is handled by the normal scan and skipped. pub fn embedded_elf_offsets(data: &[u8]) -> Vec { - let mut out = Vec::new(); - if data.len() < 0x14 { - return out; - } - let mut from = 1usize; // skip a top-level image at offset 0 - while out.len() < MAX_EMBEDDED_PE { - let Some(rel) = memchr::memmem::find(&data[from..], b"\x7fELF") else { - break; - }; - let off = from + rel; - from = off + 1; - if off + 18 > data.len() { - break; - } - let class = data[off + 4]; // EI_CLASS: 1=32-bit, 2=64-bit - let endian = data[off + 5]; // EI_DATA: 1=LE, 2=BE - let version = data[off + 6]; // EI_VERSION: 1 - if !matches!(class, 1 | 2) || !matches!(endian, 1 | 2) || version != 1 { - continue; - } - // e_type at offset 16 (2 bytes, endianness per EI_DATA): REL/EXEC/DYN/CORE. - let etype = if endian == 1 { - u16::from_le_bytes([data[off + 16], data[off + 17]]) - } else { - u16::from_be_bytes([data[off + 16], data[off + 17]]) - }; - if matches!(etype, 1..=4) { - out.push(off); - } - } - out + embedded_elf_offsets_in(&data) +} + +/// [`embedded_elf_offsets`] over an object that need not be held in memory. +pub(crate) fn embedded_elf_offsets_in(data: &dyn ByteSource) -> Vec { + embedded_in(data).elf } /// A Mach-O `cputype` we accept when validating a fat/universal header (used to @@ -530,71 +960,19 @@ fn is_macho_cputype(ct: u32) -> bool { /// engine re-type the payload so Mach-O signatures match at the embedded offset. /// Handles thin images (32/64-bit, both byte orders) and fat/universal images, /// each validated (thin: a sane `filetype` + `ncmds`; fat: `nfat_arch` range and -/// a real `cputype` for the first arch) so a coincidental magic byte run — in -/// particular a Java `.class`, which also begins `CA FE BA BE` — is not carved. +/// a real `cputype` for the first arch) so a coincidental magic byte run (in +/// particular a Java `.class`, which also begins `CA FE BA BE`) is not carved. /// The image at offset 0, if any, is handled by the normal scan and skipped. pub fn embedded_macho_offsets(data: &[u8]) -> Vec { - let mut out = Vec::new(); - if data.len() < 0x20 { - return out; - } - // (magic bytes as stored on disk, is_big_endian) - const THIN: [([u8; 4], bool); 4] = [ - ([0xCE, 0xFA, 0xED, 0xFE], false), // MH_MAGIC (32-bit), LE host - ([0xCF, 0xFA, 0xED, 0xFE], false), // MH_MAGIC_64 (64-bit), LE host - ([0xFE, 0xED, 0xFA, 0xCE], true), // MH_CIGAM (32-bit), BE host - ([0xFE, 0xED, 0xFA, 0xCF], true), // MH_CIGAM_64 (64-bit), BE host - ]; - let rd_u32 = |o: usize, be: bool| -> u32 { - let b = [data[o], data[o + 1], data[o + 2], data[o + 3]]; - if be { - u32::from_be_bytes(b) - } else { - u32::from_le_bytes(b) - } - }; - for (magic, be) in THIN { - let mut from = 1usize; // skip a top-level image at offset 0 - while out.len() < MAX_EMBEDDED_PE { - let Some(rel) = memchr::memmem::find(&data[from..], &magic) else { - break; - }; - let off = from + rel; - from = off + 1; - if off + 28 > data.len() { - break; - } - let filetype = rd_u32(off + 12, be); // MH_OBJECT..MH_KEXT_BUNDLE - let ncmds = rd_u32(off + 16, be); - if (1..=11).contains(&filetype) && (1..=10_000).contains(&ncmds) { - out.push(off); - } - } - } - // Fat/universal: magic is always big-endian on disk (`CA FE BA BE`). - let mut from = 1usize; - while out.len() < MAX_EMBEDDED_PE { - let Some(rel) = memchr::memmem::find(&data[from..], b"\xCA\xFE\xBA\xBE") else { - break; - }; - let off = from + rel; - from = off + 1; - if off + 28 > data.len() { - break; - } - let nfat = u32::from_be_bytes([data[off + 4], data[off + 5], data[off + 6], data[off + 7]]); - let cputype0 = - u32::from_be_bytes([data[off + 8], data[off + 9], data[off + 10], data[off + 11]]); - if (1..=64).contains(&nfat) && is_macho_cputype(cputype0) { - out.push(off); - } - } - out.sort_unstable(); - out.truncate(MAX_EMBEDDED_PE); - out + embedded_macho_offsets_in(&data) +} + +/// [`embedded_macho_offsets`] over an object that need not be held in memory. +pub(crate) fn embedded_macho_offsets_in(data: &dyn ByteSource) -> Vec { + embedded_in(data).macho } -/// Offsets (> 0) where a supported container's magic appears — an archive +/// Offsets (> 0) where a supported container's magic appears: an archive /// appended to or embedded in another file: SFX stubs, PE overlays (data after /// the last section), and droppers that staple a ZIP/CAB/7z/RAR/GZIP/XZ onto a /// carrier. The normal scan only types the buffer at offset 0, so these embedded @@ -603,34 +981,12 @@ pub fn embedded_macho_offsets(data: &[u8]) -> Vec { /// magic byte-run isn't treated as a real archive. Bounded to keep a buffer full /// of magic-like bytes from blowing up the work. pub fn embedded_archive_offsets(data: &[u8]) -> Vec { - const MAGICS: &[&[u8]] = &[ - b"PK\x03\x04", // ZIP / OOXML / APK / JAR - b"\x1f\x8b\x08", // GZIP - b"BZh", // BZIP2 - b"\xfd7zXZ\x00", // XZ - b"7z\xbc\xaf\x27\x1c", // 7-Zip - b"Rar!\x1a\x07", // RAR - b"MSCF", // CAB - ]; - const MAX_OFFS: usize = 32; - let mut out = Vec::new(); - if data.is_empty() { - return out; // `data[from..]` below would panic on an empty buffer - } - for m in MAGICS { - let mut from = 1usize; // offset > 0 only (offset 0 is the normal scan) - while out.len() < MAX_OFFS { - let Some(rel) = memchr::memmem::find(&data[from..], m) else { - break; - }; - out.push(from + rel); - from += rel + 1; - } - } - out.sort_unstable(); - out.dedup(); - out.truncate(MAX_OFFS); - out + embedded_archive_offsets_in(&data) +} + +/// [`embedded_archive_offsets`] over an object that need not be held in memory. +pub(crate) fn embedded_archive_offsets_in(data: &dyn ByteSource) -> Vec { + embedded_in(data).archives } /// `(raw_size, section_bytes)` for each PE section, for `.mdb`/`.msb` @@ -659,55 +1015,126 @@ pub fn section_slices(data: &[u8]) -> Vec<(u64, &[u8])> { out } -/// Whether an ELF's section-header table has been stripped — `e_shentsize` is +/// Whether an ELF's section-header table has been stripped: `e_shentsize` is /// zero, so the table cannot be walked at all. /// /// This is NOT breakage. Section headers are optional for execution: the program /// headers are what the loader reads, and these binaries run. It is, however, -/// deliberate — no toolchain emits a zero entry size, and it is a standard +/// deliberate: no toolchain emits a zero entry size, and it is a standard /// anti-analysis step (86 of 87 such files in the corpus were UPX-packed with the /// section table blanked afterwards). So it is worth reporting, under a name that /// says what was actually found. /// /// Deliberately narrow, and measured: over 1,120 corpus ELFs this separates -/// cleanly from the neighbouring conditions — 87 files have a zeroed entry size, +/// cleanly from the neighbouring conditions: 87 files have a zeroed entry size, /// 173 have `e_shnum == 0` with a canonical entry size (an ordinary way to say /// "no sections", not flagged), and 21 have a section table running past EOF -/// (genuine breakage, already caught by the parse check above). +/// (genuine breakage, caught by [`looks_broken`], as is a nonzero entry size +/// of the wrong length). pub fn elf_section_headers_stripped(data: &[u8]) -> bool { - if !data.starts_with(b"\x7fELF") { + elf_section_headers_stripped_in(&data) +} + +/// [`elf_section_headers_stripped`] over an object that need not be held in +/// memory. +pub(crate) fn elf_section_headers_stripped_in(src: &dyn ByteSource) -> bool { + let h = src.window(0, 64); + if !h.starts_with(b"\x7fELF") { return false; } - // e_ident[4] = class (1 = 32-bit, 2 = 64-bit), e_ident[5] = data encoding - // (1 = little-endian, 2 = big-endian). - let (Some(&class), Some(&enc)) = (data.get(4), data.get(5)) else { - return false; - }; - // `e_shentsize` sits at a different offset per class, and its canonical value - // is the size of one section header for that class. - let (off, canonical) = match class { - 1 => (46usize, 40u16), - 2 => (58usize, 64u16), + // `e_shentsize`, placed by the class. Zero reads the same in either byte + // order. + let off = match h.get(4) { + Some(1) => 46, + Some(2) => 58, _ => return false, }; - let Some(b) = data.get(off..off + 2) else { - return false; - }; - let shentsize = match enc { - 2 => u16::from_be_bytes([b[0], b[1]]), - _ => u16::from_le_bytes([b[0], b[1]]), - }; - // Only the zeroed case. A merely unusual-but-nonzero size is left alone: it - // is rare, and claiming it without evidence would trade one mislabel for - // another. - let _ = canonical; - shentsize == 0 + h.get(off..off + 2) == Some(&[0, 0][..]) } #[cfg(test)] mod tests { use super::*; + /// One read finds what looking for each magic at every offset finds: + /// every validated image and archive candidate, in order and within its + /// cap, whatever window seam it straddles, in memory and read through a + /// cache of small blocks. + #[test] + fn one_read_finds_every_embedded_candidate() { + use crate::byte_source::{BlockCache, CHUNK}; + let mut state = 0x2545_f491_4f6c_dd1du64; + let mut data: Vec = (0..7 * CHUNK + 77) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + b"abcdefgh"[(state % 8) as usize] + }) + .collect(); + let mut put = |at: usize, bytes: &[u8]| data[at..at + bytes.len()].copy_from_slice(bytes); + let mut elf = b"\x7fELF\x02\x01\x01".to_vec(); + elf.resize(16, 0); + elf.extend_from_slice(&2u16.to_le_bytes()); + let mut pe = b"MZ".to_vec(); + pe.resize(0x3c, 0); + pe.extend_from_slice(&0x40u32.to_le_bytes()); + pe.extend_from_slice(b"PE\0\0\x4c\x01\x03\0"); + pe.resize(0x40 + 20, 0); + pe.extend_from_slice(&0xe0u16.to_le_bytes()); + pe.extend_from_slice(&[0, 0, 0x0b, 0x01]); + let mut thin = vec![0xCF, 0xFA, 0xED, 0xFE]; + thin.resize(12, 0); + thin.extend_from_slice(&2u32.to_le_bytes()); + thin.extend_from_slice(&5u32.to_le_bytes()); + thin.resize(28, 0); + let mut fat = b"\xCA\xFE\xBA\xBE".to_vec(); + fat.extend_from_slice(&2u32.to_be_bytes()); + fat.extend_from_slice(&7u32.to_be_bytes()); + fat.resize(28, 0); + put(0, &pe); + for seam in 1..7 { + let at = seam * CHUNK; + put(at - 3, [&elf, &pe, &thin, &fat, &elf, &pe][seam - 1]); + put(at - 700, b"Rar!\x1a\x07"); + put(at + 900, b"\xfd7zXZ\x00"); + } + for k in 0..40 { + put(100 + k * 90, b"PK\x03\x04"); + } + // Every candidate, one offset at a time, and the cap of its kind. + let mut want = Embedded::default(); + for off in 1..data.len() { + for (magic, kind) in CARVE_MAGICS { + if !data[off..].starts_with(magic) { + continue; + } + let d: &dyn ByteSource = &&data[..]; + let (list, cap, ok) = match kind { + Magic::Pe => (&mut want.pe, MAX_EMBEDDED_PE, pe_at(d, off)), + Magic::Elf => (&mut want.elf, MAX_EMBEDDED_PE, elf_at(d, off)), + Magic::MachoThin(be) => (&mut want.macho, MAX_EMBEDDED_PE, macho_thin_at(d, off, be)), + Magic::MachoFat => (&mut want.macho, MAX_EMBEDDED_PE, macho_fat_at(d, off)), + Magic::Archive => (&mut want.archives, MAX_EMBEDDED_ARCHIVES, true), + }; + if ok && list.len() < cap { + list.push(off); + } + } + } + assert_eq!((want.pe.len(), want.elf.len(), want.macho.len()), (2, 2, 2)); + assert_eq!(want.archives.len(), MAX_EMBEDDED_ARCHIVES); + let src = BlockCache::with_sizes(std::io::Cursor::new(data.clone()), 512, 4096).unwrap(); + let slice: &[u8] = &data; + for hay in [&slice as &dyn ByteSource, &src] { + let got = embedded_in(hay); + assert_eq!(got.pe, want.pe); + assert_eq!(got.elf, want.elf); + assert_eq!(got.macho, want.macho); + assert_eq!(got.archives, want.archives); + } + } + #[test] fn embedded_macho_carved_and_java_rejected() { // A 64-bit LE Mach-O executable header (MH_MAGIC_64 = CF FA ED FE) at a @@ -769,7 +1196,7 @@ mod tests { // Regression (the Zbot FP family): `MZ` … `PE\0\0` followed by a junk // COFF header (zero machine, no optional header, bogus magic) must NOT be - // carved — otherwise compressed/encrypted bytes that merely contain those + // carved, otherwise compressed/encrypted bytes that merely contain those // markers get scanned as a bogus PE, where a short entry-point-anchored // signature false-positives on the garbage. let mut junk = vec![0u8; 0x80]; @@ -887,4 +1314,91 @@ mod tests { assert_eq!(rd32(280), 0x1000); // opt64 AddressOfEntryPoint assert_eq!(rd32(644), 0x200); // hdr_size (SizeOfHeaders) } + + /// Each data directory lands in its own slot, whichever are absent, and + /// the reserved 16th one (which goblin's `dirs()` panics on) is copied too. + #[test] + fn pedata_data_directories_keep_their_index() { + let set_dir = |pe: &mut Vec, i: usize, rva: u32, size: u32| { + let at = 0x58 + 112 + i * 8; + pe[at..at + 4].copy_from_slice(&rva.to_le_bytes()); + pe[at + 4..at + 8].copy_from_slice(&size.to_le_bytes()); + }; + let mut pe = minimal_pe(b"abcdefgh"); + set_dir(&mut pe, 8, 0x1004, 4); // GlobalPtr, not parsed by goblin + set_dir(&mut pe, 15, 0x1008, 8); // reserved + let bc = bytecode_pe(&pe).expect("parses"); + let pd = &bc.pedata; + let rd32 = |o: usize| u32::from_le_bytes([pd[o], pd[o + 1], pd[o + 2], pd[o + 3]]); + for base in [376, 504] { + assert_eq!(rd32(base), 0, "slot 0 stays empty"); + assert_eq!((rd32(base + 64), rd32(base + 68)), (0x1004, 4)); + assert_eq!((rd32(base + 120), rd32(base + 124)), (0x1008, 8)); + } + } + + /// Noise with embedded images and archive magics planted at `spots`, some + /// valid and some a byte off. + fn carrier(len: usize, spots: &[usize]) -> Vec { + let mut state = 0x2545_F491_4F6C_DD1Du64; + let mut d: Vec = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + b"MZPE\x7fELF\xca\xfe\xba\xbe\x00\x01\x02 x"[(state % 16) as usize] + }) + .collect(); + for (k, &at) in spots.iter().enumerate() { + let mut img = vec![0u8; 0x80]; + match k % 5 { + 0 => { + img[..2].copy_from_slice(b"MZ"); + img[0x3c..0x40].copy_from_slice(&0x40u32.to_le_bytes()); + img[0x40..0x44].copy_from_slice(b"PE\0\0"); + img[0x44..0x46].copy_from_slice(&0x14cu16.to_le_bytes()); + img[0x46..0x48].copy_from_slice(&(1 + k as u16 % 3).to_le_bytes()); + img[0x54..0x56].copy_from_slice(&0xe0u16.to_le_bytes()); + img[0x58..0x5a].copy_from_slice(&0x10bu16.to_le_bytes()); + } + 1 => img[..18].copy_from_slice(b"\x7fELF\x02\x01\x01\0\0\0\0\0\0\0\0\0\x02\0"), + 2 => img[..20].copy_from_slice(b"\xcf\xfa\xed\xfe\x07\0\0\x01\x03\0\0\0\x02\0\0\0\x05\0\0\0"), + 3 => img[..12].copy_from_slice(b"\xca\xfe\xba\xbe\0\0\0\x02\0\0\0\x07"), + _ => img[..4].copy_from_slice(if k % 2 == 0 { b"PK\x03\x04" } else { b"MSCF" }), + } + let at = at.min(len - img.len()); + d[at..at + img.len()].copy_from_slice(&img); + } + d + } + + #[test] + fn carving_offsets_are_the_same_read_in_blocks() { + use crate::byte_source::{BlockCache, CHUNK}; + // Each kind straddling a chunk seam at several splits, and some + // images whose headers run past the end. + let mut spots = Vec::new(); + for k in 0..60 { + spots.push((k + 1) * CHUNK - (k * 7) % 0x60); + } + let d = carrier(62 * CHUNK, &spots); + let cache = BlockCache::with_sizes(std::io::Cursor::new(d.clone()), 509, 8 * 509).unwrap(); + for (name, f) in [ + ("pe", embedded_pe_offsets_in as fn(&dyn ByteSource) -> Vec), + ("elf", embedded_elf_offsets_in), + ("macho", embedded_macho_offsets_in), + ("archive", embedded_archive_offsets_in), + ] { + let want = f(&d); + assert!(!want.is_empty(), "{name}: nothing planted was found"); + assert_eq!(f(&cache), want, "{name}"); + } + // The tail of the object cut through a header. + for cut in [d.len() - 20, d.len() - 1] { + let short = &d[..cut]; + let cache = BlockCache::with_sizes(std::io::Cursor::new(short.to_vec()), 509, 8 * 509).unwrap(); + assert_eq!(embedded_pe_offsets_in(&cache), embedded_pe_offsets(short)); + assert_eq!(embedded_elf_offsets_in(&cache), embedded_elf_offsets(short)); + } + } } diff --git a/crates/exav-core/src/phishing.rs b/crates/exav-core/src/phishing.rs index 74b6626e..6a012958 100644 --- a/crates/exav-core/src/phishing.rs +++ b/crates/exav-core/src/phishing.rs @@ -194,16 +194,25 @@ pub fn scan(data: &[u8], db: &PhishingDb) -> Option { /// [`scan`], also saying whether every link was examined: `false` when the /// allow-list budget ran out before the end of `data`. pub fn scan_complete(data: &[u8], db: &PhishingDb) -> (Option, bool) { + scan_complete_source(&data, db) +} + +/// [`scan_complete`] over an object that need not be held in memory. +pub(crate) fn scan_complete_source( + data: &dyn crate::byte_source::ByteSource, + db: &PhishingDb, +) -> (Option, bool) { let mut allow_checks = 0usize; - let mut next = find_anchor(data, 0); + let mut bytes = crate::byte_source::Stepper::new(data); + let mut next = find_anchor(data, &mut bytes, 0); while let Some(start) = next { - next = find_anchor(data, start + 3); + next = find_anchor(data, &mut bytes, start + 3); // An anchor is read up to the next one at most, so no byte is read for // two anchors and the whole scan stays linear in `data`. let end = next .unwrap_or(data.len()) .min(start.saturating_add(MAX_ANCHOR_SPAN)); - let region = latin1(&data[start..end]); + let region = latin1(&data.window(start, end - start)); let region_lc = region.to_ascii_lowercase(); let Some(href) = extract_href(®ion, ®ion_lc) else { continue; @@ -226,12 +235,28 @@ pub fn scan_complete(data: &[u8], db: &PhishingDb) -> (Option, bool) { (None, true) } -/// Offset of the next ` Option { - let hay = data.get(from..)?; - memchr::memchr_iter(b'<', hay) - .find(|&i| matches!(hay.get(i + 1..i + 3), Some([b'a' | b'A', b' ']))) - .map(|i| from + i) +/// Offset of the next ` Option { + use crate::byte_source::Bytes; + if let Some(data) = data.as_slice() { + let hay = data.get(from..)?; + return memchr::memchr_iter(b'<', hay) + .find(|&i| matches!(hay.get(i + 1..i + 3), Some([b'a' | b'A', b' ']))) + .map(|i| from + i); + } + let n = data.len(); + (from..n).find(|&i| { + bytes.at(i) == b'<' + && i + 2 < n + && matches!(bytes.at(i + 1), b'a' | b'A') + && bytes.at(i + 2) == b' ' + }) } /// True if the `.wdb` allow-list covers this (href, display) pair. @@ -434,6 +459,32 @@ mod tests { scan(data, &PhishingDb::default()) } + #[test] + fn a_page_read_in_blocks_is_judged_as_in_memory() { + use crate::byte_source::{BlockCache, CHUNK}; + let benign: &[u8] = br#"example.com "#; + let spoof: &[u8] = + br#"https://www.paypal.com/signin"#; + for at in [0, CHUNK - 3, CHUNK - 40, 3 * CHUNK + 1] { + let mut page = Vec::new(); + while page.len() < at { + page.extend_from_slice(benign); + } + page.truncate(at); + page.extend_from_slice(spoof); + page.extend_from_slice(&benign.repeat(50)); + let cache = + BlockCache::with_sizes(std::io::Cursor::new(page.clone()), 61, 4 * 61).unwrap(); + let want = scan_complete(&page, &PhishingDb::default()); + assert_eq!(want.0, Some(Phish::SpoofedDomain), "at {at}"); + assert_eq!( + scan_complete_source(&cache, &PhishingDb::default()), + want, + "at {at}" + ); + } + } + #[test] fn spoofed_domain() { assert_eq!( diff --git a/crates/exav-core/src/source.rs b/crates/exav-core/src/source.rs index 1c0659a9..7a991d65 100644 --- a/crates/exav-core/src/source.rs +++ b/crates/exav-core/src/source.rs @@ -1,24 +1,32 @@ //! Seekable input backends. //! -//! The engine scans two ways: a sequential `Read` (stdin, a pipe, an S3 -//! streaming GET) drives the constant-memory pattern+hash core, while a -//! `Read + Seek` source additionally lets [`crate::scan_seekable`] read a -//! ZIP's central directory and only the entries it extracts. +//! [`crate::scan_seekable`] reads a `Read + Seek` source by offset, so a +//! ZIP's central directory and its members are read where they are. A +//! sequential `Read` (stdin, a pipe) is buffered by the caller first. //! //! `HttpRangeReader` (enabled with the `http` feature) is a `Read + Seek` //! backend over HTTP(S) range requests, so an object on S3 (via a public or -//! presigned URL) can be scanned without downloading it whole — only the +//! presigned URL) can be scanned without downloading it whole: only the //! ranges the scanner touches are fetched. #[cfg(feature = "http")] mod http { use std::io::{self, Read, Seek, SeekFrom}; - /// Bytes fetched per range request (read-ahead block size). + /// Bytes fetched by a range request that does not continue the previous + /// one. const BLOCK: u64 = 64 * 1024; + /// Most bytes one range request fetches. + const MAX_BLOCK: u64 = 8 * 1024 * 1024; + /// A `Read + Seek` view over an HTTP(S) resource, served by range - /// requests with a one-block read-ahead cache. + /// requests, the last one's bytes kept. + /// + /// A request that continues the previous one fetches twice as much, up to + /// [`MAX_BLOCK`]: a pass through the object costs a request per 8 MiB + /// rather than per 64 KiB, while reads scattered through it (a ZIP's + /// directory, then its members) stay small. pub struct HttpRangeReader { agent: ureq::Agent, url: String, @@ -79,12 +87,18 @@ mod http { } fn fetch_block(&mut self, start: u64) -> io::Result<()> { + let continues = + !self.block.is_empty() && start == self.block_start + self.block.len() as u64; + let size = match continues { + true => (self.block.len() as u64 * 2).min(MAX_BLOCK), + false => BLOCK, + }; self.block.clear(); self.block_start = start; if start >= self.len { return Ok(()); } - let last = (start + BLOCK).min(self.len) - 1; + let last = (start + size).min(self.len) - 1; let resp = self .agent .get(&self.url) @@ -92,7 +106,7 @@ mod http { .call() .map_err(|e| io::Error::other(e.to_string()))?; let mut buf = Vec::new(); - resp.into_reader().take(BLOCK).read_to_end(&mut buf)?; + resp.into_reader().take(size).read_to_end(&mut buf)?; self.bytes_fetched += buf.len() as u64; self.requests += 1; self.block = buf; @@ -106,13 +120,19 @@ mod http { return Ok(0); } if !self.cached(self.pos) { - let aligned = (self.pos / BLOCK) * BLOCK; - self.fetch_block(aligned)?; + let next = self.block_start + self.block.len() as u64; + // Read on from the previous request, or from the aligned block + // holding a position elsewhere. + let start = match self.pos == next { + true => next, + false => (self.pos / BLOCK) * BLOCK, + }; + self.fetch_block(start)?; } let off = (self.pos - self.block_start) as usize; if off >= self.block.len() { // `self.pos < self.len` was established above, so the object is - // NOT finished — the server just did not give us the bytes it + // NOT finished; the server just did not give us the bytes it // said existed (a short 206, a range it declined to honour, a // truncated body). Returning `Ok(0)` here would report EOF, and // every reader upstream would treat the object as merely diff --git a/crates/exav-core/src/spill.rs b/crates/exav-core/src/spill.rs new file mode 100644 index 00000000..7e16e720 --- /dev/null +++ b/crates/exav-core/src/spill.rs @@ -0,0 +1,179 @@ +//! Room outside memory for bytes a scan makes, when the host provides it. +//! +//! The library never writes to disk itself. A scan of an object too large to +//! hold makes views of it (its normalised text) and meets members too large to +//! hold, and those have to go somewhere they can be read back at any offset. A +//! host that allows that hands a [`Spill`] in [`crate::ScanOptions::spill`]. +//! Without one, what would have needed it is reported as not fully scanned. + +use std::io::{Read, Seek}; + +use crate::byte_source::{BlockCache, CHUNK}; + +/// Makes spill files. +pub trait Spill: Send + Sync { + /// A new, empty spill file, or why there cannot be one. + fn create(&self) -> Result, String>; +} + +/// A spill file being written. +pub trait SpillWriter { + /// Append `bytes`, or say why they cannot be kept. + fn write(&mut self, bytes: &[u8]) -> Result<(), String>; + + /// Stop writing and read back what was written. The file lasts as long as + /// the reader. + fn finish(self: Box) -> Result, String>; +} + +/// A spill file read back. +pub trait SpillReader: Read + Seek {} + +impl SpillReader for T {} + +/// A spill file read back through a block cache. +pub(crate) type Spilled = BlockCache>; + +/// Collects bytes for a spill file a batch at a time. +pub(crate) struct SpillOut { + file: Box, + batch: Vec, + last: Option, + /// Why a write failed. Everything after it is dropped. + failed: Option, +} + +impl SpillOut { + pub(crate) fn new(file: Box) -> Self { + SpillOut { + file, + batch: Vec::with_capacity(CHUNK), + last: None, + failed: None, + } + } + + pub(crate) fn write(&mut self, bytes: &[u8]) { + self.batch.extend_from_slice(bytes); + if self.batch.len() >= CHUNK { + self.flush(); + } + } + + fn flush(&mut self) { + if self.failed.is_none() && !self.batch.is_empty() { + if let Err(e) = self.file.write(&self.batch) { + self.failed = Some(e); + } + } + self.batch.clear(); + } + + /// What was written, to read back. + pub(crate) fn finish(mut self) -> Result { + self.flush(); + if let Some(e) = self.failed { + return Err(e); + } + let reader = self.file.finish()?; + BlockCache::new(reader).map_err(|e| format!("spill file unreadable: {e}")) + } +} + +impl crate::normalize::Out for SpillOut { + #[inline] + fn push(&mut self, b: u8) { + self.batch.push(b); + self.last = Some(b); + if self.batch.len() >= CHUNK { + self.flush(); + } + } + + fn last(&self) -> Option { + self.last + } +} + +/// Why a stream could not be spilled. +#[derive(Debug)] +pub(crate) enum SpillStreamError { + /// Reading the stream failed. + Read(std::io::Error), + /// The spill file could not be made, written or read back. + Spill(String), +} + +/// Write all of `reader` to a new spill file, and read it back. +pub(crate) fn spill_stream( + spill: &dyn Spill, + reader: &mut dyn Read, +) -> Result { + let mut out = SpillOut::new(spill.create().map_err(SpillStreamError::Spill)?); + let mut buf = vec![0u8; CHUNK]; + loop { + match reader.read(&mut buf) { + Ok(0) => break, + Ok(n) => out.write(&buf[..n]), + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return Err(SpillStreamError::Read(e)), + } + if let Some(e) = out.failed.take() { + return Err(SpillStreamError::Spill(e)); + } + } + out.finish().map_err(SpillStreamError::Spill) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::byte_source::ByteSource; + use std::io::Cursor; + use std::sync::{Arc, Mutex}; + + /// A spill held in memory, with a byte budget, for tests. + pub(crate) struct MemSpill { + pub budget: usize, + pub made: Arc>, + } + + struct MemWriter(Vec, usize); + + impl Spill for MemSpill { + fn create(&self) -> Result, String> { + *self.made.lock().unwrap() += 1; + Ok(Box::new(MemWriter(Vec::new(), self.budget))) + } + } + + impl SpillWriter for MemWriter { + fn write(&mut self, bytes: &[u8]) -> Result<(), String> { + if self.0.len() + bytes.len() > self.1 { + return Err("spill budget".into()); + } + self.0.extend_from_slice(bytes); + Ok(()) + } + + fn finish(self: Box) -> Result, String> { + Ok(Box::new(Cursor::new(self.0))) + } + } + + #[test] + fn a_stream_spilled_reads_back_whole_or_says_why_not() { + let data: Vec = (0..3 * CHUNK + 5).map(|i| i as u8).collect(); + let spill = MemSpill { + budget: usize::MAX, + made: Arc::default(), + }; + let back = spill_stream(&spill, &mut Cursor::new(&data)).unwrap(); + assert_eq!(back.window(0, back.len()), &data[..]); + let small = MemSpill { + budget: CHUNK, + made: Arc::default(), + }; + assert!(spill_stream(&small, &mut Cursor::new(&data)).is_err()); + } +} diff --git a/crates/exav-core/src/stream_regex.rs b/crates/exav-core/src/stream_regex.rs new file mode 100644 index 00000000..373371df --- /dev/null +++ b/crates/exav-core/src/stream_regex.rs @@ -0,0 +1,363 @@ +//! Regular expressions over a [`ByteSource`], answering what a `regex` +//! `meta::Regex` answers over the same bytes held in memory. +//! +//! `meta::Regex` finds a leftmost-first match with a forward lazy DFA, which +//! reports where the match ends, then an anchored reverse lazy DFA run back from +//! that end, which reports where it starts. Both are driven here one byte at a +//! time through the source, so nothing is held but the DFAs' caches. The +//! start states, the one-byte delay of match states and the end-of-input +//! transitions are handled as `regex-automata`'s own searches handle them. +//! +//! A lazy DFA cannot follow everything a `meta::Regex` can: a Unicode word +//! boundary against a non-ASCII byte makes it quit. That is reported as +//! [`GaveUp`], never as an answer. + +use regex_automata::hybrid::dfa::{Cache, DFA}; +use regex_automata::nfa::thompson::{self, WhichCaptures}; +use regex_automata::util::start; +use regex_automata::util::syntax; +use regex_automata::{Anchored, MatchKind}; +use regex_syntax::hir::Hir; + +use crate::byte_source::ByteSource; + +/// Bytes read from the source at a time. +const STEP: usize = 64 * 1024; + +/// The DFAs met a construct they cannot follow over these bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct GaveUp; + +/// The lazy DFAs' state tables, grown as a search needs them. +pub(crate) struct Caches { + fwd: Cache, + rev: Cache, +} + +pub(crate) struct StreamRegex { + fwd: DFA, + rev: DFA, + /// The pattern can only match at the start of a search. + anchored: bool, +} + +impl StreamRegex { + /// The regex `regex::bytes::RegexBuilder::new(pattern)` builds, with + /// `size_limit(nfa_limit)`. `None` where that would fail, or where the lazy + /// DFAs cannot be built. + pub(crate) fn bytes_regex(pattern: &str, nfa_limit: usize) -> Option { + let hir = syntax::parse_with(pattern, &syntax::Config::new().utf8(false)).ok()?; + Self::from_hir(&hir, Some(nfa_limit)) + } + + /// A regex over a parsed pattern, matching as a `meta::Regex` built from + /// the same HIR does. + pub(crate) fn from_hir(hir: &Hir, nfa_limit: Option) -> Option { + let nfa_config = thompson::Config::new() + .utf8(false) + .nfa_size_limit(nfa_limit) + .which_captures(WhichCaptures::None); + let nfa = thompson::Compiler::new() + .configure(nfa_config.clone()) + .build_from_hir(hir) + .ok()?; + let nfarev = thompson::Compiler::new() + .configure(nfa_config.reverse(true)) + .build_from_hir(hir) + .ok()?; + let anchored = nfa.is_always_start_anchored(); + let config = DFA::config() + .unicode_word_boundary(true) + // Clearing the cache is slower, never wrong; giving up would be. + .minimum_cache_clear_count(None) + .cache_capacity(16 << 20); + let fwd = DFA::builder() + .configure(config.clone().match_kind(MatchKind::LeftmostFirst)) + .build_from_nfa(nfa) + .ok()?; + let rev = DFA::builder() + .configure(config.match_kind(MatchKind::All)) + .build_from_nfa(nfarev) + .ok()?; + Some(StreamRegex { fwd, rev, anchored }) + } + + /// Scratch space for searches with this regex, kept across searches. + pub(crate) fn caches(&self) -> Caches { + Caches { + fwd: self.fwd.create_cache(), + rev: self.rev.create_cache(), + } + } + + /// Whether the pattern matches anywhere in `src`. + pub(crate) fn is_match(&self, src: &dyn ByteSource) -> Result { + let mut caches = self.caches(); + Ok(self.forward(&mut caches.fwd, src, 0, true)?.is_some()) + } + + /// The leftmost-first match starting at or after `start`, as `(start, + /// end)`: what `meta::Regex::find` returns for `Input::new(bytes).span(start..)`. + pub(crate) fn find( + &self, + caches: &mut Caches, + src: &dyn ByteSource, + start: usize, + ) -> Result, GaveUp> { + let Some(end) = self.forward(&mut caches.fwd, src, start, false)? else { + return Ok(None); + }; + if end == start || self.anchored { + return Ok(Some((start, end))); + } + let from = self + .reverse(&mut caches.rev, src, start, end)? + .ok_or(GaveUp)?; + Ok(Some((from, end))) + } + + /// The matches `meta::Regex::find_iter` yields over `src`, until `f` + /// returns `false`. + pub(crate) fn for_each( + &self, + src: &dyn ByteSource, + f: &mut dyn FnMut(usize, usize) -> bool, + ) -> Result<(), GaveUp> { + let mut caches = self.caches(); + let len = src.len(); + let mut start = 0; + let mut last_end: Option = None; + while start <= len { + let Some((mut s, mut e)) = self.find(&mut caches, src, start)? else { + return Ok(()); + }; + // An empty match where the last one ended is skipped by searching + // again one byte on. + if s == e && Some(e) == last_end { + start += 1; + if start > len { + return Ok(()); + } + match self.find(&mut caches, src, start)? { + Some(m) => (s, e) = m, + None => return Ok(()), + } + } + if !f(s, e) { + return Ok(()); + } + start = e; + last_end = Some(e); + } + Ok(()) + } + + /// End of the leftmost-first match starting at or after `start` (the first + /// match end at all when `earliest`). + fn forward( + &self, + cache: &mut Cache, + src: &dyn ByteSource, + start: usize, + earliest: bool, + ) -> Result, GaveUp> { + let len = src.len(); + if start > len { + return Ok(None); + } + let behind = (start > 0).then(|| byte(src, start - 1)); + let config = start::Config::new() + .anchored(Anchored::No) + .look_behind(behind); + let mut sid = self.fwd.start_state(cache, &config).map_err(|_| GaveUp)?; + let mut end = None; + let mut at = start; + while at < len { + let w = src.window(at, (len - at).min(STEP)); + if w.is_empty() { + return Err(GaveUp); + } + for &b in w.iter() { + sid = next(&self.fwd, cache, sid, b)?; + if sid.is_tagged() { + // Matches are delayed by one byte: this state says a match + // ended just before `at`. + if sid.is_match() { + end = Some(at); + if earliest { + return Ok(end); + } + } else if sid.is_dead() { + return Ok(end); + } else if sid.is_quit() { + return Err(GaveUp); + } + } + at += 1; + } + } + sid = self.fwd.next_eoi_state(cache, sid).map_err(|_| GaveUp)?; + if sid.is_match() { + end = Some(len); + } + Ok(end) + } + + /// Start of the match ending at `end`, found by the anchored reverse DFA + /// run back towards `start`. + fn reverse( + &self, + cache: &mut Cache, + src: &dyn ByteSource, + start: usize, + end: usize, + ) -> Result, GaveUp> { + let behind = (end < src.len()).then(|| byte(src, end)); + let config = start::Config::new() + .anchored(Anchored::Yes) + .look_behind(behind); + let mut sid = self.rev.start_state(cache, &config).map_err(|_| GaveUp)?; + let mut from = None; + let mut hi = end; + while hi > start { + let lo = hi.saturating_sub(STEP).max(start); + let w = src.window(lo, hi - lo); + if w.len() != hi - lo { + return Err(GaveUp); + } + for (k, &b) in w.iter().enumerate().rev() { + let at = lo + k; + sid = next(&self.rev, cache, sid, b)?; + if sid.is_tagged() { + if sid.is_match() { + from = Some(at + 1); + } else if sid.is_dead() { + return Ok(from); + } else if sid.is_quit() { + return Err(GaveUp); + } + } + } + hi = lo; + } + if start > 0 { + sid = next(&self.rev, cache, sid, byte(src, start - 1))?; + if sid.is_match() { + from = Some(start); + } else if sid.is_quit() { + return Err(GaveUp); + } + } else { + sid = self.rev.next_eoi_state(cache, sid).map_err(|_| GaveUp)?; + if sid.is_match() { + from = Some(0); + } + } + Ok(from) + } +} + +fn next( + dfa: &DFA, + cache: &mut Cache, + sid: regex_automata::hybrid::LazyStateID, + b: u8, +) -> Result { + dfa.next_state(cache, sid, b).map_err(|_| GaveUp) +} + +fn byte(src: &dyn ByteSource, at: usize) -> u8 { + src.window(at, 1).first().copied().unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::byte_source::BlockCache; + use std::io::Cursor; + + const PATTERNS: &[&str] = &[ + "abc", + "a+b", + "a*", + "(a|ab)(c|bcd)", + "^ab", + "ab$", + "(?m)^ab$", + r"\bword\b", + r"\Bor", + "[0-9]{3,5}", + "x?", + "(?s).+z", + ".+z", + "(?i)HeLLo", + r"\x00\xff+", + "a{2,}?b", + "(ab|a)+c", + r"(?-u)\bfoo", + "", + "q|", + "b+$", + r"\Aa", + r"z\z", + "[^a]{2}", + ]; + + fn haystacks() -> Vec> { + let mut out = vec![ + b"".to_vec(), + b"abc".to_vec(), + b"xxabcdxxaab ab\nab abab".to_vec(), + b"a word, words wordy word".to_vec(), + b"12 1234 123456 99999z".to_vec(), + b"hello HELLO hElLo\x00\xff\xff\xffaab aaab".to_vec(), + b"caf\xc3\xa9 word \xe9word foo\xe9foo".to_vec(), + ]; + let mut state = 7u64; + for _ in 0..20 { + let mut h = Vec::new(); + for _ in 0..(state % 300) as usize { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + let alphabet = b"abcdqxz \n09\xe9\x00word"; + h.push(alphabet[(state % alphabet.len() as u64) as usize]); + } + out.push(h); + } + out + } + + #[test] + fn matches_what_the_in_memory_regex_matches() { + for p in PATTERNS { + let re = regex::bytes::Regex::new(p).unwrap(); + let sr = StreamRegex::bytes_regex(p, 16 << 20).unwrap(); + for h in haystacks() { + // Blocks of 3 bytes: every read crosses a seam somewhere. + let src = BlockCache::with_sizes(Cursor::new(h.clone()), 3, 12).unwrap(); + let want: Vec<(usize, usize)> = + re.find_iter(&h).map(|m| (m.start(), m.end())).collect(); + let mut got = Vec::new(); + match sr.for_each(&src, &mut |s, e| { + got.push((s, e)); + true + }) { + Ok(()) => assert_eq!(got, want, "/{p}/ on {:?}", String::from_utf8_lossy(&h)), + // A Unicode word boundary meeting non-ASCII input. + Err(GaveUp) => assert!(p.contains(r"\b") || p.contains(r"\B")), + } + if let Ok(m) = sr.is_match(&src) { + assert_eq!(m, re.is_match(&h), "/{p}/"); + } + let mut caches = sr.caches(); + for start in 0..=h.len() { + let want = re.find_at(&h, start).map(|m| (m.start(), m.end())); + if let Ok(m) = sr.find(&mut caches, &src, start) { + assert_eq!(m, want, "/{p}/ at {start}"); + } + } + } + } + } +} diff --git a/crates/exav-core/src/yara/atoms.rs b/crates/exav-core/src/yara/atoms.rs index 4e46e0ab..37394b64 100644 --- a/crates/exav-core/src/yara/atoms.rs +++ b/crates/exav-core/src/yara/atoms.rs @@ -3,10 +3,10 @@ //! Scanning every compiled pattern over the whole buffer (one `find_all` per //! pattern) is O(patterns × bytes) and dominates scan time on many-rule sets. //! This module builds a *required-literal gate*: for each pattern we extract one -//! or more **atoms** — literal byte substrings that MUST appear in EVERY match of +//! or more **atoms**: literal byte substrings that MUST appear in EVERY match of //! the pattern. A single shared Aho-Corasick automaton (daachorse) is run over //! the buffer ONCE; a pattern's `find_all` is then executed ONLY if one of its -//! atoms was found (or if the pattern is *ungated* — has no provable required +//! atoms was found (or if the pattern is *ungated*, i.e. has no provable required //! literal, e.g. `xor`, `/a+/`, alternations with an atomless branch). //! //! ## Why this is false-negative-safe @@ -16,12 +16,12 @@ //! zero matches, and returning an empty match list for it is IDENTICAL to running //! `find_all`. When we cannot prove such a literal exists, the pattern is left //! **ungated** and always scanned. We only ever *skip* a pattern whose required -//! literal is provably absent — never a pattern that might match. +//! literal is provably absent, never a pattern that might match. //! //! Required literals for regexp / hex patterns are extracted with //! `regex_syntax`'s literal `Extractor` (the same analysis the `regex` crate uses //! for its own prefilters): a finite prefix (or suffix) literal `Seq` with no -//! empty member is a sound set of required literals — every match begins (ends) +//! empty member is a sound set of required literals: every match begins (ends) //! with one of them. When the extractor gives up it returns an infinite `Seq`, //! which we treat as "no required literal" → ungated. @@ -31,6 +31,7 @@ use daachorse::{DoubleArrayAhoCorasick, DoubleArrayAhoCorasickBuilder}; use regex_syntax::hir::literal::{ExtractKind, Extractor}; use serde::{Deserialize, Serialize}; +use crate::byte_source::ByteSource; use crate::yara::matcher::{parse_hir, widen_hir, Base64Sub, Needle}; /// Maximum atom window length taken from a literal / base64 needle. A longer @@ -40,7 +41,7 @@ const MAX_ATOM: usize = 16; /// The gate classification of one compiled pattern. pub(crate) enum Gate { - /// No provable required literal — the pattern is always scanned. + /// No provable required literal, so the pattern is always scanned. Ungated, /// The pattern matches only if at least ONE of these atoms is present /// (a disjunction). `nocase` means the atoms are ASCII-lowercased and are @@ -203,7 +204,7 @@ pub(crate) fn regex_gate_ex( /// for a `wide ascii` regexp: the plain-form gate OR the wide-form gate). The /// atom sets are unioned (a required literal of either branch is a candidate); /// if either branch is ungated, or the two branches disagree on case-folding, the -/// combined gate is [`Gate::Ungated`] — never skip a pattern that might match. +/// combined gate is [`Gate::Ungated`]: never skip a pattern that might match. pub(crate) fn combine_gates(a: Gate, b: Gate) -> Gate { match (a, b) { (Gate::Ungated, _) | (_, Gate::Ungated) => Gate::Ungated, @@ -303,45 +304,63 @@ impl Pool { } /// Builds the automaton. On a (very unlikely) build failure the pool's - /// patterns are force-ungated so they are still scanned — never silently + /// patterns are force-ungated so they are still scanned, never silently /// dropped. - fn finish( - self, - ungated: &mut [bool], - ) -> (Option>, Vec>) { + fn finish(self, ungated: &mut [bool]) -> Lane { if self.atoms.is_empty() { - return (None, Vec::new()); + return Lane::default(); } match DoubleArrayAhoCorasickBuilder::new() .match_kind(daachorse::MatchKind::Standard) .build_with_values(self.atoms.iter().zip(0u32..)) { - Ok(ac) => (Some(ac), self.groups), + Ok(ac) => Lane { + ac: Some(ac), + groups: self.groups, + runs: crate::engine::longest_runs(&self.atoms), + }, Err(_) => { for g in &self.groups { for &pid in g { ungated[pid] = true; } } - (None, Vec::new()) + Lane::default() } } } } +/// One automaton of the gate: over its atoms, the patterns behind each, and +/// the longest run of each byte value in them (see [`crate::engine::sweep`]). +struct Lane { + ac: Option>, + groups: Vec>, + runs: [u32; 256], +} + +impl Default for Lane { + fn default() -> Self { + Lane { + ac: None, + groups: Vec::new(), + runs: [0; 256], + } + } +} + /// The prefilter gate for a whole compiled rule set. Case-sensitive atoms are -/// searched in the raw buffer; `nocase` atoms in an ASCII-lowercased copy. +/// searched in the raw buffer; `nocase` atoms in its bytes lowercased, both in +/// the one read of it. pub(crate) struct PatternGate { - ac_cs: Option>, - groups_cs: Vec>, - ac_ci: Option>, - groups_ci: Vec>, + cs: Lane, + ci: Lane, /// Per pattern id: `true` = always scan (ungated). ungated: Vec, /// Number of patterns with a required-literal gate (diagnostics). gated: usize, /// Kill switch: when set (via `EXAV_YARA_NO_GATE`), the prefilter is bypassed - /// and every pattern is scanned. A safety valve — results are identical to a + /// and every pattern is scanned. A safety valve: results are identical to a /// gated scan by construction, but this lets an operator rule the prefilter /// out if a gate bug is ever suspected. Read once at build time (never on the /// hot path). @@ -366,16 +385,14 @@ impl PatternGate { } } } - let (ac_cs, groups_cs) = cs.finish(&mut ungated); - let (ac_ci, groups_ci) = ci.finish(&mut ungated); + let cs = cs.finish(&mut ungated); + let ci = ci.finish(&mut ungated); // Whatever remains not-ungated after the (possibly failing) builds is // in fact gated. let gated = ungated.iter().filter(|&&u| !u).count(); PatternGate { - ac_cs, - groups_cs, - ac_ci, - groups_ci, + cs, + ci, ungated, gated, bypass: std::env::var_os("EXAV_YARA_NO_GATE").is_some(), @@ -391,7 +408,7 @@ impl PatternGate { /// Returns, per pattern id, whether `find_all` must be run: `true` for every /// ungated pattern, plus every gated pattern one of whose atoms occurs in /// `data`. A gated pattern whose atoms are all absent provably has no match. - pub(crate) fn select(&self, data: &[u8]) -> Vec { + pub(crate) fn select(&self, src: &dyn ByteSource) -> Vec { if self.bypass { return vec![true; self.ungated.len()]; } @@ -399,43 +416,38 @@ impl PatternGate { // Number of gated patterns still to resolve; lets us stop scanning once // every pattern is already slated to run (dense buffers). let mut pending = run.iter().filter(|&&r| !r).count(); - - if pending > 0 { - if let Some(ac) = &self.ac_cs { - for m in ac.find_overlapping_iter(data) { - for &pid in &self.groups_cs[m.value() as usize] { - if !run[pid] { - run[pid] = true; - pending -= 1; - } - } - if pending == 0 { - break; - } - } - } + if pending == 0 { + return run; } - - if pending > 0 { - if let Some(ac) = &self.ac_ci { - let lower = data.to_ascii_lowercase(); - for m in ac.find_overlapping_iter(&lower) { - for &pid in &self.groups_ci[m.value() as usize] { - if !run[pid] { - run[pid] = true; - pending -= 1; - } - } - if pending == 0 { - break; - } - } + let (mut lanes, mut groups) = (Vec::new(), Vec::new()); + for (lane, fold) in [(&self.cs, false), (&self.ci, true)] { + if let Some(ac) = &lane.ac { + lanes.push(crate::engine::Lane { + ac, + fold, + runs: &lane.runs, + }); + groups.push(&lane.groups); } } + crate::engine::sweep(&lanes, src, &mut |k, value, _, _| { + mark(groups[k], value, &mut run, &mut pending) + }); run } } +/// Slates the patterns of atom `value` to run. `false` once none is left. +fn mark(groups: &[Vec], value: u32, run: &mut [bool], pending: &mut usize) -> bool { + for &pid in &groups[value as usize] { + if !run[pid] { + run[pid] = true; + *pending -= 1; + } + } + *pending > 0 +} + // --------------------------------------------------------------------------- // Serialization of the compiled gate // --------------------------------------------------------------------------- @@ -445,7 +457,7 @@ impl PatternGate { // daachorse automaton travels as its own byte serialization (`serialize`), and // the per-atom pattern-id groups + the ungated bitmap travel as plain data. On // load the automaton is reconstructed via daachorse's checked `deserialize` (no -// rebuild). `bypass` is NOT serialized — it is an operator kill switch read +// rebuild). `bypass` is NOT serialized: it is an operator kill switch read // afresh from the environment on load, exactly as at build time. /// True when the prefilter kill switch is set. Read once (build or load time), @@ -456,13 +468,16 @@ fn bypass_env() -> bool { impl Serialize for PatternGate { fn serialize(&self, s: S) -> Result { - let ac_cs = self.ac_cs.as_ref().map(|ac| ac.serialize()); - let ac_ci = self.ac_ci.as_ref().map(|ac| ac.serialize()); + use crate::database::Blob; + let ac_cs = self.cs.ac.as_ref().map(|ac| ac.serialize()); + let ac_ci = self.ci.ac.as_ref().map(|ac| ac.serialize()); ( - ac_cs, - &self.groups_cs, - ac_ci, - &self.groups_ci, + ac_cs.as_deref().map(Blob), + &self.cs.groups, + &self.cs.runs[..], + ac_ci.as_deref().map(Blob), + &self.ci.groups, + &self.ci.runs[..], &self.ungated, self.gated, ) @@ -472,43 +487,115 @@ impl Serialize for PatternGate { impl<'de> Deserialize<'de> for PatternGate { fn deserialize>(d: D) -> Result { + use crate::database::BlobBuf; type Wire = ( - Option>, + Option, Vec>, - Option>, + Vec, + Option, Vec>, + Vec, Vec, usize, ); - let (ac_cs_bytes, groups_cs, ac_ci_bytes, groups_ci, ungated, gated): Wire = + let (ac_cs, groups_cs, runs_cs, ac_ci, groups_ci, runs_ci, ungated, gated): Wire = Deserialize::deserialize(d)?; fn rebuild( - bytes: Option>, - ) -> Result>, E> { - match bytes { - None => Ok(None), + bytes: Option, + groups: Vec>, + runs: Vec, + ) -> Result { + let runs = runs + .try_into() + .map_err(|_| serde::de::Error::custom("bad atom run table"))?; + let ac = match bytes { + None => None, Some(b) => { // Checked deserialize: rejects data that would cause // out-of-bounds access (the database is a trusted artifact, // but a corrupt/mismatched blob must fail loudly, not misread). let (ac, _rest) = - DoubleArrayAhoCorasick::::deserialize(&b).map_err(|e| { + DoubleArrayAhoCorasick::::deserialize(&b.0).map_err(|e| { serde::de::Error::custom(format!("bad atom automaton: {e}")) })?; - Ok(Some(ac)) + Some(ac) } - } + }; + Ok(Lane { ac, groups, runs }) } Ok(PatternGate { - ac_cs: rebuild(ac_cs_bytes)?, - groups_cs, - ac_ci: rebuild(ac_ci_bytes)?, - groups_ci, + cs: rebuild(ac_cs, groups_cs, runs_cs)?, + ci: rebuild(ac_ci, groups_ci, runs_ci)?, ungated, gated, bypass: bypass_env(), }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::byte_source::BlockCache; + + /// The gate slates a gated pattern to run exactly when one of its atoms + /// occurs in the object (lowercased for a `nocase` one), in memory and read + /// through a cache, runs of one byte included. + #[test] + fn the_gate_selects_the_patterns_whose_atoms_occur() { + let atoms = |a: &[&[u8]], nocase| Gate::Atoms { + atoms: a.iter().map(|x| x.to_vec()).collect(), + nocase, + }; + let gates = vec![ + atoms(&[b"abcd"], false), + atoms(&[b"xyz", &[0; 12]], false), + atoms(&[b"hello"], true), + Gate::Ungated, + atoms(&[b"zzzz", b"q\0\0\0\0\0\0q"], true), + ]; + let gate = PatternGate::from_gates(&gates); + let mut state = 0x5eed_u64; + let mut next = |n: usize| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state % n as u64) as usize + }; + for _ in 0..300 { + let mut hay = Vec::new(); + for _ in 0..1 + next(8) { + match next(4) { + 0 => hay.extend(std::iter::repeat_n( + b"\0zZ"[next(3)], + [3, 11, 12, 70, 3000][next(5)], + )), + 1 => hay.extend_from_slice( + [&b"abcd"[..], b"HeLLo", b"xyz", b"Q\0\0\0\0\0\0Q"][next(4)], + ), + _ => hay.extend((0..next(40)).map(|_| b"abcdxyzhelo\0HELOQ"[next(17)])), + } + } + let lower = hay.to_ascii_lowercase(); + let want: Vec = gates + .iter() + .map(|g| match g { + Gate::Ungated => true, + Gate::Atoms { atoms, nocase } => { + let h = if *nocase { &lower } else { &hay }; + atoms + .iter() + .any(|a| h.windows(a.len()).any(|w| w == &a[..])) + } + }) + .collect(); + let slice: &[u8] = &hay; + let cache = + BlockCache::with_sizes(std::io::Cursor::new(hay.clone()), 64, 1024).unwrap(); + assert_eq!(gate.select(&slice), want, "{hay:?}"); + assert_eq!(gate.select(&cache), want, "{hay:?}"); + } + } +} diff --git a/crates/exav-core/src/yara/compiler.rs b/crates/exav-core/src/yara/compiler.rs index a0284b87..91323b5a 100644 --- a/crates/exav-core/src/yara/compiler.rs +++ b/crates/exav-core/src/yara/compiler.rs @@ -2,6 +2,7 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; +use std::sync::OnceLock; use yara_x_parser::ast::AST; use yara_x_parser::ast::{ @@ -238,6 +239,7 @@ impl Compiler { let gate = PatternGate::from_gates(&self.atom_gates); Rules { patterns: self.patterns, + stream_regexes: self.defs.iter().map(|_| OnceLock::new()).collect(), defs: self.defs, pattern_names: self.pattern_names, rules: self.rules, diff --git a/crates/exav-core/src/yara/ir.rs b/crates/exav-core/src/yara/ir.rs index a4220b9f..4ab2ebaa 100644 --- a/crates/exav-core/src/yara/ir.rs +++ b/crates/exav-core/src/yara/ir.rs @@ -9,6 +9,7 @@ use std::rc::Rc; use regex_automata::{meta::Regex, Input}; use serde::{Deserialize, Serialize}; +use crate::byte_source::ByteSource; use crate::yara::matcher::{build_regex, Match}; use crate::yara::modules::{self, FuncId, ModuleCtx, ModuleKind}; @@ -365,7 +366,7 @@ pub(crate) enum Binding { /// Everything the evaluator needs at scan time. pub(crate) struct EvalCtx<'a> { - pub data: &'a [u8], + pub data: &'a dyn ByteSource, /// Per global-pattern-id, the sorted match list. pub matches: &'a [Vec], /// Results of rules that were evaluated earlier (by rule index). @@ -637,17 +638,17 @@ fn count_in_range(id: usize, range: &Option<(Box, Box)>, ctx: &EvalC /// Reads an integer of the given kind at `offset` (little- or big-endian). /// Out-of-bounds or negative offsets yield undefined (`None`), matching yara-x. -fn read_int(kind: IntKind, data: &[u8], offset: i64) -> Option { +fn read_int(kind: IntKind, data: &dyn ByteSource, offset: i64) -> Option { let off: usize = offset.try_into().ok()?; macro_rules! rd { ($n:literal, $conv:expr) => {{ - let bytes: [u8; $n] = data.get(off..off + $n)?.try_into().ok()?; + let bytes: [u8; $n] = data.window(off, $n).as_ref().try_into().ok()?; $conv(bytes) }}; } let v: i64 = match kind { - IntKind::U8 => *data.get(off)? as i64, - IntKind::I8 => *data.get(off)? as i8 as i64, + IntKind::U8 => rd!(1, |b: [u8; 1]| b[0] as i64), + IntKind::I8 => rd!(1, |b: [u8; 1]| b[0] as i8 as i64), IntKind::U16 => rd!(2, |b| u16::from_le_bytes(b) as i64), IntKind::I16 => rd!(2, |b| i16::from_le_bytes(b) as i64), IntKind::U32 => rd!(4, |b| u32::from_le_bytes(b) as i64), diff --git a/crates/exav-core/src/yara/matcher.rs b/crates/exav-core/src/yara/matcher.rs index 1408d8f2..f0e01b29 100644 --- a/crates/exav-core/src/yara/matcher.rs +++ b/crates/exav-core/src/yara/matcher.rs @@ -3,19 +3,23 @@ //! Each YARA string/pattern is lowered to a [`PatternMatcher`] that, given a //! buffer, produces the list of match offsets+lengths. Three kinds: //! -//! * `Literal` — text patterns (with `nocase`/`ascii`/`wide`/`xor`/`base64`/ +//! * `Literal`: text patterns (with `nocase`/`ascii`/`wide`/`xor`/`base64`/ //! `base64wide`/`fullword` handled by expanding into concrete needles). -//! * `Regex` — regexp patterns, run anchored at every start offset so that +//! * `Regex`: regexp patterns, run anchored at every start offset so that //! overlapping matches are reported the way YARA does. -//! * `Hex` — hex patterns, lowered to an equivalent byte regexp and run like +//! * `Hex`: hex patterns, lowered to an equivalent byte regexp and run like //! `Regex` (YARA hex jumps behave like lazy `.{a,b}?` over *any* byte). +use std::ops::Range; + use base64::Engine; use regex_automata::{meta, meta::Regex, util::syntax, Input}; use regex_syntax::hir::{self, Hir, HirKind}; use regex_syntax::ParserBuilder; use serde::{Deserialize, Serialize}; +use crate::byte_source::{ByteSource, CHUNK}; +use crate::stream_regex::{GaveUp, StreamRegex}; use crate::yara::error::{Error, Result}; /// A single match: byte offset and length within the scanned buffer. @@ -84,8 +88,8 @@ pub(crate) struct Base64Sub { // A [`PatternMatcher`] holds a compiled `regex_automata::meta::Regex` (and a // `base64` engine), neither of which is serde-serializable and neither of which // can hand back the source it was built from. So the on-disk database does NOT -// store the compiled matcher; it stores a [`PatternDef`] — the pattern's -// *definition* (regex source + flags, or the literal/base64 needle bytes) — from +// store the compiled matcher; it stores a [`PatternDef`], the pattern's +// *definition* (regex source + flags, or the literal/base64 needle bytes), from // which [`PatternDef::compile`] rebuilds an identical `PatternMatcher` on load. // Rebuilding a single pattern's regex is cheap; the aggregate cost that the // serialized database avoids is the daachorse atom-automaton build (see @@ -221,24 +225,44 @@ fn is_word(b: u8) -> bool { /// `verify_full_word` (fullword modifier: neighbours must be non-alphanumeric; /// underscore is NOT a word byte here, unlike regex `\b`). fn verify_full_word(data: &[u8], start: usize, end: usize, wide: bool, xor_key: u8) -> bool { + let before = &data[start.saturating_sub(2)..start]; + let after = &data[end..(end + 2).min(data.len())]; + full_word_between(before, after, wide, xor_key) +} + +/// [`verify_full_word`] given the (up to) two bytes before the match and the +/// (up to) two bytes after it. +fn full_word_between(before: &[u8], after: &[u8], wide: bool, xor_key: u8) -> bool { if wide { - if start >= 2 && (data[start - 1] ^ xor_key) == 0 && is_word(data[start - 2] ^ xor_key) { - return false; + if let [a, b] = before { + if (b ^ xor_key) == 0 && is_word(a ^ xor_key) { + return false; + } } - if end + 1 < data.len() && (data[end + 1] ^ xor_key) == 0 && is_word(data[end] ^ xor_key) { - return false; + if let [a, b] = after { + if (b ^ xor_key) == 0 && is_word(a ^ xor_key) { + return false; + } } } else { - if start >= 1 && is_word(data[start - 1] ^ xor_key) { + if before.last().is_some_and(|b| is_word(b ^ xor_key)) { return false; } - if end < data.len() && is_word(data[end] ^ xor_key) { + if after.first().is_some_and(|b| is_word(b ^ xor_key)) { return false; } } true } +/// [`verify_full_word`] over a source, reading only the neighbours. +fn full_word_in(src: &dyn ByteSource, start: usize, end: usize, wide: bool) -> bool { + let from = start.saturating_sub(2); + let before = src.window(from, start - from); + let after = src.window(end, 2); + full_word_between(&before, &after, wide, 0) +} + #[inline] fn needle_hit(data: &[u8], pos: usize, n: &Needle, key: u8) -> bool { let bytes = &n.bytes; @@ -262,11 +286,160 @@ fn needle_hit(data: &[u8], pos: usize, n: &Needle, key: u8) -> bool { true } +/// Matches one pattern keeps. Past it the rest are dropped and the pattern's +/// search is incomplete. yara-x stops at the same count. +pub(crate) const MAX_MATCHES: usize = 1_000_000; + +/// The matches found for one pattern. +pub(crate) struct Found { + /// Sorted by offset then length, without duplicates. + pub matches: Vec, + /// Every match was found: the pattern stayed under [`MAX_MATCHES`] and + /// was searched through the whole object. + pub complete: bool, +} + +/// Matches as they are found, up to [`MAX_MATCHES`]. +struct Collector { + out: Vec, + full: bool, +} + +impl Collector { + fn new() -> Self { + Collector { + out: Vec::new(), + full: false, + } + } + + fn push(&mut self, offset: usize, len: usize) { + if self.out.len() < MAX_MATCHES { + self.out.push(Match { offset, len }); + } else { + self.full = true; + } + } +} + +/// A pattern's regexes as a search over a source runs them, each with +/// whether it is the `wide` form. +pub(crate) type StreamRegexes = Vec<(StreamRegex, bool)>; + +/// Hands out a pattern's [`StreamRegexes`], built on first use. +pub(crate) type RegexSource<'s> = &'s dyn Fn() -> Option<&'s StreamRegexes>; + +/// Cap on the NFA one regexp compiles to. +const NFA_SIZE_LIMIT: usize = 64 << 20; +impl PatternDef { + /// The regexes of a regexp or hex pattern for a search over a source, + /// matching what [`PatternDef::compile`]'s `meta` regexes match. `None` + /// for other patterns, or where one cannot be built. + pub(crate) fn stream_regexes(&self) -> Option { + let build = |src: &str, ci: bool, dotall: bool, wide: bool| { + let hir = parse_hir(src, ci, dotall).ok()?; + let hir = if wide { widen_hir(hir) } else { hir }; + Some((StreamRegex::from_hir(&hir, Some(NFA_SIZE_LIMIT))?, wide)) + }; + match self { + PatternDef::Regex { + src, + ci, + dotall, + wide, + .. + } => Some(vec![build(src, *ci, *dotall, *wide)?]), + PatternDef::RegexMulti { + src, ci, dotall, .. + } => Some(vec![ + build(src, *ci, *dotall, false)?, + build(src, *ci, *dotall, true)?, + ]), + PatternDef::Hex { re_src } => Some(vec![build(re_src, false, false, false)?]), + PatternDef::Literal { .. } | PatternDef::Base64 { .. } => None, + } + } +} + +/// [`PatternMatcher::find_all`] of several [windowed](PatternMatcher::windowed) +/// patterns over a source not held in memory, in one read of it rather than a +/// read each: every window holds what any of their matches needs around it. +pub(crate) fn find_all_windowed(pats: &[&PatternMatcher], src: &dyn ByteSource) -> Vec { + let (before, after) = pats + .iter() + .map(|p| p.reach()) + .fold((0, 0), |(b, a), (pb, pa)| (b.max(pb), a.max(pa))); + let len = src.len(); + let mut outs: Vec = pats.iter().map(|_| Collector::new()).collect(); + let mut complete = true; + let mut base = 0; + while base < len && outs.iter().any(|o| !o.full) { + let end = (base + CHUNK).min(len); + let from = base.saturating_sub(before); + let to = end.saturating_add(after).min(len); + let w = src.window(from, to - from); + if w.len() != to - from { + complete = false; + break; + } + for (p, out) in pats.iter().zip(&mut outs) { + if !out.full { + p.search(&w, base - from..end - from, from, out); + } + } + base = end; + } + outs.into_iter() + .map(|out| { + let mut matches = out.out; + matches.sort_unstable_by(|a, b| a.offset.cmp(&b.offset).then(a.len.cmp(&b.len))); + matches.dedup(); + Found { + matches, + complete: complete && !out.full, + } + }) + .collect() +} + impl PatternMatcher { - /// Returns all matches, sorted by offset then length, de-duplicated by - /// (offset, length). - pub(crate) fn find_all(&self, data: &[u8]) -> Vec { - let mut out = Vec::new(); + /// Whether [`find_all_windowed`] searches this pattern: a literal or a + /// base64 one, whose matches need a bounded context around them. + pub(crate) fn windowed(&self) -> bool { + matches!( + self, + PatternMatcher::Literal { .. } | PatternMatcher::Base64 { .. } + ) + } + + /// Every match in `src`. `stream` supplies the pattern's regexes for a + /// source not held in memory; `materialize` is the largest such source + /// read whole when they cannot follow it. + pub(crate) fn find_all( + &self, + src: &dyn ByteSource, + stream: RegexSource<'_>, + materialize: usize, + ) -> Found { + let mut out = Collector::new(); + let mut complete = true; + match src.as_slice() { + Some(data) => self.search(data, 0..data.len(), 0, &mut out), + None => complete = self.search_source(src, stream, materialize, &mut out), + } + let mut matches = out.out; + matches.sort_unstable_by(|a, b| a.offset.cmp(&b.offset).then(a.len.cmp(&b.len))); + matches.dedup(); + Found { + matches, + complete: complete && !out.full, + } + } + + /// The matches starting at a position in `keep` of `data`, reported at + /// `origin` plus their position. `data` holds whatever each match needs + /// around it (see [`Self::reach`]) or ends where the object ends. + fn search(&self, data: &[u8], keep: Range, origin: usize, out: &mut Collector) { match self { PatternMatcher::Literal { needles, @@ -281,16 +454,13 @@ impl PatternMatcher { if n.bytes.is_empty() || n.bytes.len() > data.len() { continue; } + let last = data.len() - n.bytes.len(); if n.nocase { - let last = data.len() - n.bytes.len(); - for pos in 0..=last { + for pos in keep.start..keep.end.min(last + 1) { if needle_hit(data, pos, n, 0) { let end = pos + n.bytes.len(); if !*fullword || verify_full_word(data, pos, end, n.wide, 0) { - out.push(Match { - offset: pos, - len: n.bytes.len(), - }); + out.push(origin + pos, n.bytes.len()); } } } @@ -299,15 +469,15 @@ impl PatternMatcher { // start offset (e.g. `"aaaa"` in `"aaaaaa"` → 3), so // we advance the search window by one byte, not by the // needle length (which `memmem::find_iter` would do). - let mut base = 0; + let mut base = keep.start; while let Some(off) = memchr::memmem::find(&data[base..], &n.bytes) { let pos = base + off; + if pos >= keep.end { + break; + } let end = pos + n.bytes.len(); if !*fullword || verify_full_word(data, pos, end, n.wide, 0) { - out.push(Match { - offset: pos, - len: n.bytes.len(), - }); + out.push(origin + pos, n.bytes.len()); } base = pos + 1; } @@ -322,15 +492,12 @@ impl PatternMatcher { continue; } let last = data.len() - n.bytes.len(); - for pos in 0..=last { + for pos in keep.start..keep.end.min(last + 1) { for &key in &keys { if needle_hit(data, pos, n, key) { let end = pos + n.bytes.len(); if !*fullword || verify_full_word(data, pos, end, n.wide, key) { - out.push(Match { - offset: pos, - len: n.bytes.len(), - }); + out.push(origin + pos, n.bytes.len()); } // A given position can only be XOR-decoded by // one key at a time for a fixed plaintext, but @@ -343,19 +510,20 @@ impl PatternMatcher { } } }, + // Regexes are only searched whole: their matches have no bound. PatternMatcher::Regex { re, fullword, wide } => { - anchored_matches(re, data, &mut out, *fullword, *wide); + anchored_matches(re, data, out, *fullword, *wide); } PatternMatcher::RegexMulti { ascii, wide, fullword, } => { - anchored_matches(ascii, data, &mut out, *fullword, false); - anchored_matches(wide, data, &mut out, *fullword, true); + anchored_matches(ascii, data, out, *fullword, false); + anchored_matches(wide, data, out, *fullword, true); } PatternMatcher::Hex { re } => { - anchored_matches(re, data, &mut out, false, false); + anchored_matches(re, data, out, false, false); } PatternMatcher::Base64 { entries } => { for e in entries { @@ -363,7 +531,7 @@ impl PatternMatcher { continue; } let last = data.len() - e.searched.len(); - for pos in 0..=last { + for pos in keep.start..keep.end.min(last + 1) { if data[pos..].starts_with(&e.searched) { if let Some(r) = verify_base64( &e.pattern, @@ -373,19 +541,109 @@ impl PatternMatcher { &e.engine, e.wide, ) { - out.push(Match { - offset: r.0, - len: r.1 - r.0, - }); + out.push(origin + r.0, r.1 - r.0); } } } } } } - out.sort_unstable_by(|a, b| a.offset.cmp(&b.offset).then(a.len.cmp(&b.len))); - out.dedup(); - out + } + + /// Bytes a literal or base64 match at `pos` is checked against: how many + /// before `pos`, and how many from `pos` on. + fn reach(&self) -> (usize, usize) { + match self { + // The needle, and two bytes each side for `fullword`. + PatternMatcher::Literal { needles, .. } => { + let longest = needles.iter().map(|n| n.bytes.len()).max().unwrap_or(0); + (2, longest + 2) + } + // What `verify_base64` decodes: from up to 3 characters before + // the match (twice that when wide) to its decode length after. + PatternMatcher::Base64 { entries } => { + let after = entries + .iter() + .map(|e| { + let len = base64::encoded_len(e.pattern.len(), false).unwrap_or(0); + e.searched.len().max(2 * (len + 5)) + }) + .max() + .unwrap_or(0); + (6, after) + } + PatternMatcher::Regex { .. } + | PatternMatcher::RegexMulti { .. } + | PatternMatcher::Hex { .. } => (0, 0), + } + } + + /// [`Self::search`] over a source not held in memory. `false` when some + /// of it could not be searched. + fn search_source( + &self, + src: &dyn ByteSource, + stream: RegexSource<'_>, + materialize: usize, + out: &mut Collector, + ) -> bool { + let len = src.len(); + match self { + PatternMatcher::Literal { .. } | PatternMatcher::Base64 { .. } => { + // Chunks, each read with the bytes its matches need around them. + let (before, after) = self.reach(); + let mut base = 0; + while base < len && !out.full { + let end = (base + CHUNK).min(len); + let from = base.saturating_sub(before); + let to = end.saturating_add(after).min(len); + let w = src.window(from, to - from); + if w.len() != to - from { + return false; + } + self.search(&w, base - from..end - from, from, out); + base = end; + } + true + } + PatternMatcher::Regex { fullword, .. } + | PatternMatcher::RegexMulti { fullword, .. } => { + let fullword = *fullword; + self.search_regexes(src, stream, materialize, fullword, out) + } + PatternMatcher::Hex { .. } => self.search_regexes(src, stream, materialize, false, out), + } + } + + fn search_regexes( + &self, + src: &dyn ByteSource, + stream: RegexSource<'_>, + materialize: usize, + fullword: bool, + out: &mut Collector, + ) -> bool { + let found = stream().and_then(|regexes| { + let mut found = Collector::new(); + for (sr, wide) in regexes.iter() { + stream_matches(sr, src, &mut found, fullword, *wide).ok()?; + } + Some(found) + }); + if let Some(found) = found { + out.full |= found.full; + out.out.extend(found.out); + return true; + } + // The lazy DFAs gave up, or could not be built: the object is searched + // whole, as in memory, if it may be read whole. + match src.materialize(materialize) { + Some(data) => { + self.search(&data, 0..data.len(), 0, out); + true + } + None => false, + } } } @@ -400,26 +658,48 @@ impl PatternMatcher { /// byte-for-byte the one an anchored search at `s` would produce. No offset in /// `[start, s)` can begin a match (else it would be the leftmost), so nothing is /// skipped; resuming at `s + 1` then enumerates every match-start in order. -fn anchored_matches(re: &Regex, data: &[u8], out: &mut Vec, fullword: bool, wide: bool) { +fn anchored_matches(re: &Regex, data: &[u8], out: &mut Collector, fullword: bool, wide: bool) { let len = data.len(); let mut start = 0; - while start <= len { + while start <= len && !out.full { let input = Input::new(data).span(start..len); match re.find(input) { Some(m) => { let s = m.start(); let e = m.end(); if e > s && (!fullword || verify_full_word(data, s, e, wide, 0)) { - out.push(Match { - offset: s, - len: e - s, - }); + out.push(s, e - s); + } + start = s + 1; + } + None => break, + } + } +} + +/// [`anchored_matches`] over a source, with the regex stepped through it. +fn stream_matches( + sr: &StreamRegex, + src: &dyn ByteSource, + out: &mut Collector, + fullword: bool, + wide: bool, +) -> std::result::Result<(), GaveUp> { + let mut caches = sr.caches(); + let len = src.len(); + let mut start = 0; + while start <= len && !out.full { + match sr.find(&mut caches, src, start)? { + Some((s, e)) => { + if e > s && (!fullword || full_word_in(src, s, e, wide)) { + out.push(s, e - s); } start = s + 1; } None => break, } } + Ok(()) } /// Verifies that `pattern` actually matches in base64 form at `match_start`. @@ -535,7 +815,7 @@ pub(crate) fn build_regex( // pattern expands during construction, so a rule can ask for far more // memory than its source suggests, and the failure lands at database-load // time where it is least expected. These are failsafes over trusted - // content — the database is a trusted input — so they are set high enough + // content (the database is a trusted input), so they are set high enough // that no reasonable rule meets them, and a rule that does is rejected by // name rather than taking the process with it. // @@ -543,7 +823,7 @@ pub(crate) fn build_regex( // builds finite automata and does not backtrack. The risk here is // compile-time memory, not match-time blowup. let limits = meta::Config::new() - .nfa_size_limit(Some(64 << 20)) + .nfa_size_limit(Some(NFA_SIZE_LIMIT)) .dfa_size_limit(Some(16 << 20)) .onepass_size_limit(Some(8 << 20)); Regex::builder() @@ -577,7 +857,7 @@ pub(crate) fn parse_hir( /// Rewrites an [`Hir`] into its `wide` (UTF-16LE) form: every byte the regexp /// would consume is followed by a mandatory `\x00`. This reproduces yara-x's -/// `wide`-regexp semantics exactly — yara-x matches such a pattern against a +/// `wide`-regexp semantics exactly: yara-x matches such a pattern against a /// stream in which each data byte is interleaved with a zero byte, and reports a /// match length of `2 × (bytes consumed)` (the trailing zero of the last byte is /// part of the match). Widening a literal interleaves a zero after every byte; diff --git a/crates/exav-core/src/yara/mod.rs b/crates/exav-core/src/yara/mod.rs index 01f661df..851d1393 100644 --- a/crates/exav-core/src/yara/mod.rs +++ b/crates/exav-core/src/yara/mod.rs @@ -3,13 +3,13 @@ //! Traditional engines support only a restricted YARA subset (no modules, ≤64 //! strings/rule, etc.). This one targets broad real-world YARA compatibility //! (strings, full conditions, for-loops/`with`, and the `pe`/`math`/`hash`/ -//! `string`/`time` modules) while compiling **natively** — no wasmtime/Cranelift +//! `string`/`time` modules) while compiling **natively**: no wasmtime/Cranelift //! JIT, no runtime codegen at scan time (W^X preserved). //! //! Like the rest of exav's database, [`YaraDb`] serializes its COMPILED form: at //! database-build time ([`YaraDb::finalize`]) the rules are compiled and the //! compiled rule set is serialized into `YaraDb::compiled_blob`, so a prebuilt -//! `.exavdb` loads the YARA engine WITHOUT recompiling — in particular without +//! `.exavdb` loads the YARA engine WITHOUT recompiling, in particular without //! rebuilding the expensive daachorse atom automaton. The daachorse automaton + //! owned IR + per-pattern definitions travel in the blob; only the per-pattern //! regexes (whose compiled automata are not serializable) are recompiled from @@ -24,7 +24,7 @@ //! rather than mis-evaluating them. To avoid one unsupported rule silently //! dropping a whole third-party feed, we compile **per rule** (see //! [`Compiler::add_source_lenient`]) and record every rejection in -//! [`YaraDb::rejected`], so the coverage gap is always visible — never silent. +//! [`YaraDb::rejected`], so the coverage gap is always visible, never silent. //! //! # Where the native engine lives //! @@ -91,17 +91,18 @@ use serde::{Deserialize, Serialize}; /// compiled-rules layout (the `Rules` blob: IR, pattern defs, atom automaton) /// changes in an incompatible way. A loaded blob whose version does not match /// this build is ignored and the rules are recompiled from source, so a stale -/// blob is never misread — the coverage is preserved, just without the load-time +/// blob is never misread: the coverage is preserved, just without the load-time /// shortcut. (This is independent of the outer `.exavdb` format serial in -/// `crate::database`, which guards the whole file.) +/// `crate::database`, which guards the whole file.) Version 2 stores the atom +/// automatons as binary blobs, with their run tables. #[cfg(feature = "yara")] -const YARA_BLOB_VERSION: u32 = 1; +const YARA_BLOB_VERSION: u32 = 2; /// The data fields are always present (so the on-disk database format is identical /// whether or not the `yara` feature is built), but the actual compilation and /// matching are gated on the feature. Without it, rule sources are /// still stored (a database built with yara can be loaded without it and vice -/// versa) — they're simply never matched. +/// versa); they're simply never matched. #[derive(Default, Serialize, Deserialize)] pub struct YaraDb { /// Accumulated rule sources (one per loaded `.yar`/`.yara` file). @@ -114,21 +115,21 @@ pub struct YaraDb { /// Rules the engine could not compile, recorded at [`Self::finalize`] /// time (i.e. when the database is built) as `": "`. This /// is serialized into the database so an operator loading a prebuilt `.exavdb` - /// can still SEE which rules were dropped and why — a rule that will not + /// can still SEE which rules were dropped and why. A rule that will not /// compile is always accounted for, never invisibly missing. #[serde(default)] rejected: Vec, /// The serialized COMPILED rule set, produced by [`Self::finalize`] at /// database-build time (only when built with the `yara` feature). When - /// present and version-compatible, [`Self::rules`] deserializes it — skipping - /// the expensive daachorse atom-automaton build — instead of recompiling from + /// present and version-compatible, [`Self::rules`] deserializes it (skipping + /// the expensive daachorse atom-automaton build) instead of recompiling from /// `sources`. This is what makes a prebuilt `.exavdb` load its YARA engine /// without recompiling, consistent with the rest of the database (which also /// serializes its compiled form). Absent when the database was built without /// the `yara` feature, or predates this field; then `rules()` falls back to /// compiling `sources`. To a build WITHOUT the `yara` feature these are opaque /// bytes that round-trip through the database untouched. - #[serde(default)] + #[serde(default, with = "crate::database::opt_blob")] compiled_blob: Option>, /// Format version of `compiled_blob` (see [`YARA_BLOB_VERSION`]). A mismatch /// makes `rules()` ignore the blob and recompile from source. @@ -176,9 +177,17 @@ impl YaraDb { /// validate them and record which rules were rejected (so the coverage gap is /// captured in the built database) and (b) SERIALIZE the compiled rule set /// into `Self::compiled_blob`, so a prebuilt `.exavdb` loads the YARA engine - /// without recompiling — skipping the expensive atom-automaton build — exactly + /// without recompiling (skipping the expensive atom-automaton build), exactly /// as the rest of the database serializes its compiled form. Idempotent. pub fn finalize(&mut self) { + #[cfg(not(feature = "yara"))] + if !self.sources.is_empty() { + eprintln!( + "[exav yara] WARN: {} YARA rule file(s) loaded, but this build has no `yara` \ + feature: they will never match.", + self.sources.len(), + ); + } #[cfg(feature = "yara")] { if self.sources.is_empty() { @@ -228,7 +237,7 @@ impl YaraDb { } } // Fallback: no blob (database built without the `yara` feature, or - // predating it), or an unusable blob — recompile from source. The + // predating it), or an unusable blob: recompile from source. The // coverage is identical; only the load-time shortcut is skipped. let (rules, _rejected) = compile_lenient(&self.sources); Some(rules) @@ -250,9 +259,9 @@ impl YaraDb { /// When present, the standard THOR/signature-base external variables are set /// for this scan so rules that reference them can match: /// - /// * `filepath` — the path as given (original case), - /// * `filename` — the basename (original case), - /// * `extension` — the lowercased extension *including* the leading dot + /// * `filepath`: the path as given (original case), + /// * `filename`: the basename (original case), + /// * `extension`: the lowercased extension *including* the leading dot /// (e.g. `".js"`), per the signature-base convention. /// /// `filetype`/`owner` are left undefined (exav does not model them). When @@ -260,6 +269,19 @@ impl YaraDb { /// them simply do not match. #[cfg(feature = "yara")] pub fn scan(&self, data: &[u8], filename: Option<&str>) -> Vec { + self.scan_source(&data, filename, usize::MAX) + } + + /// As [`Self::scan`], over an object that need not be held in memory. + /// `materialize` is the largest such object read whole where a regex or + /// a module needs it so. + #[cfg(feature = "yara")] + pub(crate) fn scan_source( + &self, + src: &dyn crate::byte_source::ByteSource, + filename: Option<&str>, + materialize: usize, + ) -> Vec { let Some(rules) = self.rules() else { return Vec::new(); }; @@ -272,17 +294,17 @@ impl YaraDb { scanner.set_global("extension", ext); } } - let Ok(results) = scanner.scan(data) else { - return Vec::new(); - }; - // A condition that ran out of steps evaluated to undefined, so a rule - // may have failed to match for that reason alone. - if results.budget_exhausted() { + let results = scanner.scan_source(src, materialize); + // A condition that ran out of steps evaluated to undefined, and one + // that saw part of a pattern's matches or no module values may be + // wrong, so a rule may have failed to match for that reason alone. + if results.steps_exhausted || !results.complete { crate::engine::mark_scan_truncated(); } results - .matching_rules() - .map(|r| format!("YARA.{}", r.identifier())) + .matching + .iter() + .map(|name| format!("YARA.{name}")) .collect() } @@ -291,6 +313,16 @@ impl YaraDb { pub fn scan(&self, _data: &[u8], _filename: Option<&str>) -> Vec { Vec::new() } + + #[cfg(not(feature = "yara"))] + pub(crate) fn scan_source( + &self, + _src: &dyn crate::byte_source::ByteSource, + _filename: Option<&str>, + _materialize: usize, + ) -> Vec { + Vec::new() + } } /// The basename of a path, splitting on both `/` and `\` so Windows-style paths @@ -361,7 +393,7 @@ fn preview(rejected: &[String]) -> String { // `crate::pe`, hashing via `md-5`/`sha1`/`sha2` + `crate::hexsig`, entropy via // `crate::pe::shannon_entropy`, `memchr`). It deliberately does NOT reuse the AV // matcher or the bytecode VM for evaluating YARA rule *conditions*. A future -// contributor should not "unify" these — doing so breaks the yara-x A/B +// contributor should not "unify" these: doing so breaks the yara-x A/B // differential (~17k rules, 0 disagreements). The reasons are structural: // // * The AV pattern matcher is non-exhaustive by design (it reports the first @@ -467,7 +499,7 @@ mod tests { /// The COMPILED rule set must survive a full `YaraDb` MessagePack round trip /// (the `.exavdb` boundary) and load WITHOUT recompiling from source: after /// `finalize`, the serialized blob is present, travels through serialize + - /// deserialize, and the reloaded database scans identically — driven by the + /// deserialize, and the reloaded database scans identically, driven by the /// deserialized compiled blob, not a source recompile. #[test] fn compiled_blob_survives_database_serialization() { @@ -505,7 +537,7 @@ mod tests { } /// If the blob version does not match this build, the loader must ignore the - /// stale blob and recompile from source — never misread it, never go silent. + /// stale blob and recompile from source, never misreading it or going silent. #[test] fn stale_blob_version_falls_back_to_source() { let mut db = YaraDb::new(); diff --git a/crates/exav-core/src/yara/modules/hash.rs b/crates/exav-core/src/yara/modules/hash.rs index 09212402..e7031419 100644 --- a/crates/exav-core/src/yara/modules/hash.rs +++ b/crates/exav-core/src/yara/modules/hash.rs @@ -12,6 +12,7 @@ use sha1::Sha1; use sha2::Sha256; use super::FuncId; +use crate::byte_source::ByteSource; use crate::yara::ir::Value; /// The `hash` module exposes only functions, so its root is an empty struct. @@ -19,7 +20,7 @@ pub(crate) fn root() -> Value { Value::Struct(Rc::new(HashMap::new())) } -pub(crate) fn call(func: FuncId, data: &[u8], args: &[Value]) -> Option { +pub(crate) fn call(func: FuncId, data: &dyn ByteSource, args: &[Value]) -> Option { use FuncId::*; match func { HashMd5Data => hex_over_range::(data, args).map(Value::Str), @@ -29,33 +30,48 @@ pub(crate) fn call(func: FuncId, data: &[u8], args: &[Value]) -> Option { HashSha256Data => hex_over_range::(data, args).map(Value::Str), HashSha256Str => Some(Value::Str(hex::(args[0].as_bytes()?))), HashCrc32Data => { - let s = range(data, args)?; - Some(Value::Int(crc32fast::hash(s) as i64)) + let (start, end) = range(data, args)?; + let mut h = crc32fast::Hasher::new(); + data.chunks(start, end, &mut |_, c| { + h.update(c); + true + }); + Some(Value::Int(h.finalize() as i64)) } HashCrc32Str => Some(Value::Int(crc32fast::hash(args[0].as_bytes()?) as i64)), HashChecksum32Data => { - let s = range(data, args)?; - Some(Value::Int(checksum32(s) as i64)) + let (start, end) = range(data, args)?; + let mut sum = 0u32; + data.chunks(start, end, &mut |_, c| { + sum = sum.wrapping_add(checksum32(c)); + true + }); + Some(Value::Int(sum as i64)) } HashChecksum32Str => Some(Value::Int(checksum32(args[0].as_bytes()?) as i64)), _ => unreachable!("non-hash FuncId dispatched to hash::call"), } } -/// Resolves `(offset, size)` args to a byte slice, matching yara-x: the range +/// Resolves `(offset, size)` args to a byte range, matching yara-x: the range /// is `offset..(offset+size)`, and any out-of-bounds/negative bound yields /// undefined (`None`). -fn range<'d>(data: &'d [u8], args: &[Value]) -> Option<&'d [u8]> { +fn range(data: &dyn ByteSource, args: &[Value]) -> Option<(usize, usize)> { let offset = args[0].to_i64()?; let size = args[1].to_i64()?; let start: usize = offset.try_into().ok()?; let end: usize = offset.checked_add(size)?.try_into().ok()?; - data.get(start..end) + (start <= end && end <= data.len()).then_some((start, end)) } -fn hex_over_range(data: &[u8], args: &[Value]) -> Option> { - let s = range(data, args)?; - Some(hex::(s)) +fn hex_over_range(data: &dyn ByteSource, args: &[Value]) -> Option> { + let (start, end) = range(data, args)?; + let mut d = D::new(); + data.chunks(start, end, &mut |_, c| { + d.update(c); + true + }); + Some(crate::hexsig::encode_hex(&d.finalize()).into_bytes()) } /// One-shot digest of `data` with `D`, lowercase-hex-encoded (as YARA returns @@ -160,4 +176,37 @@ mod tests { b"foobarbaz" )); } + + /// Every hash of a range gives, over a source read in chunks, exactly the + /// value it gives over the bytes in memory. + #[test] + fn a_range_read_in_chunks_hashes_as_the_slice_does() { + use super::super::FuncId::*; + use super::{call, Value}; + use crate::byte_source::{BlockCache, CHUNK}; + + let h: Vec = (0..3 * CHUNK + 5).map(|i| (i * 31 % 251) as u8).collect(); + let cache = BlockCache::with_sizes(std::io::Cursor::new(h.clone()), 7, 64).unwrap(); + let n = h.len(); + for (off, len) in [ + (0, n), + (1, CHUNK + 1), + (CHUNK - 5, 2 * CHUNK + 3), + (n, 0), + (n - 1, 2), + ] { + let args = [Value::Int(off as i64), Value::Int(len as i64)]; + for func in [ + HashMd5Data, + HashSha1Data, + HashSha256Data, + HashCrc32Data, + HashChecksum32Data, + ] { + let want = format!("{:?}", call(func, &h, &args)); + let got = format!("{:?}", call(func, &cache, &args)); + assert_eq!(want, got, "{func:?} over {off}+{len}"); + } + } + } } diff --git a/crates/exav-core/src/yara/modules/math.rs b/crates/exav-core/src/yara/modules/math.rs index 5c74b1b1..fe738f3a 100644 --- a/crates/exav-core/src/yara/modules/math.rs +++ b/crates/exav-core/src/yara/modules/math.rs @@ -12,6 +12,7 @@ use std::rc::Rc; use memchr::memchr_iter; use super::FuncId; +use crate::byte_source::ByteSource; use crate::yara::ir::Value; type ByteDistribution = [u64; 256]; @@ -22,7 +23,7 @@ pub(crate) fn root() -> Value { Value::Struct(Rc::new(HashMap::new())) } -pub(crate) fn call(func: FuncId, data: &[u8], args: &[Value]) -> Option { +pub(crate) fn call(func: FuncId, data: &dyn ByteSource, args: &[Value]) -> Option { use FuncId::*; match func { MathMin => Some(Value::Int(i64::min(args[0].to_i64()?, args[1].to_i64()?))), @@ -54,7 +55,11 @@ pub(crate) fn call(func: FuncId, data: &[u8], args: &[Value]) -> Option { // identical to the yara-x port that lived here (same 0..=255 count // iteration order, same `p * log2(p)` accumulation, same empty→0.0), // so the difftest against yara-x stays at 0 disagreements. - Some(Value::Float(crate::pe::shannon_entropy(&data[start..end]))) + let dist = range_distribution(data, start, end); + Some(Value::Float(crate::pe::entropy_of_counts( + &dist, + end - start, + ))) } MathEntropyStr => Some(Value::Float(crate::pe::shannon_entropy( args[0].as_bytes()?, @@ -62,7 +67,7 @@ pub(crate) fn call(func: FuncId, data: &[u8], args: &[Value]) -> Option { MathMeanData => { let (start, end) = data_range(data, args[0].to_i64()?, args[1].to_i64()?)?; - let dist = byte_distribution(&data[start..end]); + let dist = range_distribution(data, start, end); Some(Value::Float(mean_from_distribution(&dist, end - start)?)) } MathMeanStr => Some(Value::Float(mean(args[0].as_bytes()?)?)), @@ -70,7 +75,7 @@ pub(crate) fn call(func: FuncId, data: &[u8], args: &[Value]) -> Option { MathDeviationData => { let (start, end) = data_range(data, args[0].to_i64()?, args[1].to_i64()?)?; let mean = args[2].to_f64()?; - let dist = byte_distribution(&data[start..end]); + let dist = range_distribution(data, start, end); Some(Value::Float(deviation_from_distribution( &dist, end - start, @@ -83,35 +88,45 @@ pub(crate) fn call(func: FuncId, data: &[u8], args: &[Value]) -> Option { } MathSerialCorrelationData => { - let s = slice_range(data, args[0].to_i64()?, args[1].to_i64()?)?; - Some(Value::Float(serial_correlation(s)?)) + let (start, end) = data_range(data, args[0].to_i64()?, args[1].to_i64()?)?; + let mut sc = SerialCorrelation::default(); + data.chunks(start, end, &mut |_, c| { + sc.feed(c); + true + }); + Some(Value::Float(sc.finish())) } - MathSerialCorrelationStr => Some(Value::Float(serial_correlation(args[0].as_bytes()?)?)), + MathSerialCorrelationStr => Some(Value::Float(serial_correlation(args[0].as_bytes()?))), MathMonteCarloPiData => { - let s = slice_range(data, args[0].to_i64()?, args[1].to_i64()?)?; - Some(Value::Float(monte_carlo_pi(s)?)) + let (start, end) = data_range(data, args[0].to_i64()?, args[1].to_i64()?)?; + let mut mc = MonteCarloPi::default(); + data.chunks(start, end, &mut |_, c| { + mc.feed(c); + true + }); + Some(Value::Float(mc.finish()?)) } MathMonteCarloPiStr => Some(Value::Float(monte_carlo_pi(args[0].as_bytes()?)?)), MathCountRange => { let byte: u8 = args[0].to_i64()?.try_into().ok()?; - let s = slice_range(data, args[1].to_i64()?, args[2].to_i64()?)?; - Some(Value::Int(memchr_iter(byte, s).count() as i64)) + let (start, end) = data_range(data, args[1].to_i64()?, args[2].to_i64()?)?; + Some(Value::Int(count_in(data, byte, start, end) as i64)) } MathCountGlobal => { let byte: u8 = args[0].to_i64()?.try_into().ok()?; - Some(Value::Int(memchr_iter(byte, data).count() as i64)) + Some(Value::Int(count_in(data, byte, 0, data.len()) as i64)) } MathPercentageRange => { let byte: u8 = args[0].to_i64()?.try_into().ok()?; - let s = slice_range(data, args[1].to_i64()?, args[2].to_i64()?)?; - if s.is_empty() { + let (start, end) = data_range(data, args[1].to_i64()?, args[2].to_i64()?)?; + if start == end { return None; } Some(Value::Float( - memchr_iter(byte, s).count() as f64 / s.len() as f64, + count_in(data, byte, start, end) as f64 / (end - start) as f64, )) } MathPercentageGlobal => { @@ -120,17 +135,17 @@ pub(crate) fn call(func: FuncId, data: &[u8], args: &[Value]) -> Option { return None; } Some(Value::Float( - memchr_iter(byte, data).count() as f64 / data.len() as f64, + count_in(data, byte, 0, data.len()) as f64 / data.len() as f64, )) } MathModeGlobal => { - let dist = byte_distribution(data); + let dist = range_distribution(data, 0, data.len()); mode_from_distribution(&dist, data.len()).map(Value::Int) } MathModeRange => { let (start, end) = data_range(data, args[0].to_i64()?, args[1].to_i64()?)?; - let dist = byte_distribution(&data[start..end]); + let dist = range_distribution(data, start, end); mode_from_distribution(&dist, end - start).map(Value::Int) } @@ -142,29 +157,46 @@ fn truthy(v: &Value) -> bool { matches!(v, Value::Bool(true)) || matches!(v, Value::Int(i) if *i != 0) } -fn data_range(data: &[u8], offset: i64, length: i64) -> Option<(usize, usize)> { +/// `offset..offset+length` of the data, cut where the data ends. Undefined +/// when `offset` is negative or past the end. +fn data_range(data: &dyn ByteSource, offset: i64, length: i64) -> Option<(usize, usize)> { let length: usize = length.try_into().ok()?; let start: usize = offset.try_into().ok()?; let end = cmp::min(data.len(), start.saturating_add(length)); - data.get(start..end)?; - Some((start, end)) -} - -fn slice_range(data: &[u8], offset: i64, length: i64) -> Option<&[u8]> { - let length: usize = length.try_into().ok()?; - let start: usize = offset.try_into().ok()?; - let end = cmp::min(data.len(), start.saturating_add(length)); - data.get(start..end) + (start <= end).then_some((start, end)) } fn byte_distribution(data: &[u8]) -> ByteDistribution { let mut distribution = [0u64; 256]; + add_to_distribution(&mut distribution, data); + distribution +} + +fn add_to_distribution(distribution: &mut ByteDistribution, data: &[u8]) { for byte in data { distribution[*byte as usize] += 1; } +} + +fn range_distribution(data: &dyn ByteSource, start: usize, end: usize) -> ByteDistribution { + let mut distribution = [0u64; 256]; + data.chunks(start, end, &mut |_, c| { + add_to_distribution(&mut distribution, c); + true + }); distribution } +/// Occurrences of `byte` in `[start, end)` of the data. +fn count_in(data: &dyn ByteSource, byte: u8, start: usize, end: usize) -> usize { + let mut n = 0; + data.chunks(start, end, &mut |_, c| { + n += memchr_iter(byte, c).count(); + true + }); + n +} + fn deviation(data: &[u8], mean: f64) -> Option { deviation_from_distribution(&byte_distribution(data), data.len(), mean) } @@ -212,45 +244,108 @@ fn mode_from_distribution(distribution: &ByteDistribution, len: usize) -> Option Some(mode as i64) } -fn serial_correlation(data: &[u8]) -> Option { - let Some((&first, rest)) = data.split_first() else { - return Some(-100000.0); - }; - let first = first as f64; - let mut prev = first; - let mut adjacent_product_sum = 0.0; - let mut byte_sum = first; - let mut byte_square_sum = first * first; - - for byte in rest { - let byte = *byte as f64; - adjacent_product_sum += prev * byte; - byte_sum += byte; - byte_square_sum += byte * byte; - prev = byte; - } +fn serial_correlation(data: &[u8]) -> f64 { + let mut sc = SerialCorrelation::default(); + sc.feed(data); + sc.finish() +} - adjacent_product_sum += first * prev; +/// Serial correlation, fed the bytes in order a piece at a time. +#[derive(Default)] +struct SerialCorrelation { + first: Option, + prev: f64, + adjacent_product_sum: f64, + byte_sum: f64, + byte_square_sum: f64, + len: usize, +} - let len = data.len() as f64; - let byte_sum_squared = byte_sum * byte_sum; - let scc = (len * adjacent_product_sum - byte_sum_squared) - / (len * byte_square_sum - byte_sum_squared); +impl SerialCorrelation { + fn feed(&mut self, data: &[u8]) { + let rest = match self.first { + Some(_) => data, + None => { + let Some((&first, rest)) = data.split_first() else { + return; + }; + let first = first as f64; + self.first = Some(first); + self.prev = first; + self.byte_sum = first; + self.byte_square_sum = first * first; + self.len = 1; + rest + } + }; + for byte in rest { + let byte = *byte as f64; + self.adjacent_product_sum += self.prev * byte; + self.byte_sum += byte; + self.byte_square_sum += byte * byte; + self.prev = byte; + } + self.len += rest.len(); + } - if scc.is_nan() { - Some(-100000.0) - } else { - Some(scc) + fn finish(self) -> f64 { + let Some(first) = self.first else { + return -100000.0; + }; + let adjacent_product_sum = self.adjacent_product_sum + first * self.prev; + + let len = self.len as f64; + let byte_sum_squared = self.byte_sum * self.byte_sum; + let scc = (len * adjacent_product_sum - byte_sum_squared) + / (len * self.byte_square_sum - byte_sum_squared); + + if scc.is_nan() { + -100000.0 + } else { + scc + } } } fn monte_carlo_pi(data: &[u8]) -> Option { - const INCIRC: f64 = 281474943156225.0_f64; // ((256 ^ 3) - 1) ^ 2 + let mut mc = MonteCarloPi::default(); + mc.feed(data); + mc.finish() +} - let mut inmont = 0; - let mut mcount = 0; +/// The Monte Carlo value of Pi, fed the bytes in order a piece at a time. +/// Each six bytes are one point; a trailing partial point is ignored. +#[derive(Default)] +struct MonteCarloPi { + inmont: u64, + mcount: u64, + /// The start of a point cut by the end of the last piece. + partial: Vec, +} + +impl MonteCarloPi { + fn feed(&mut self, mut data: &[u8]) { + if !self.partial.is_empty() { + let take = (6 - self.partial.len()).min(data.len()); + self.partial.extend_from_slice(&data[..take]); + data = &data[take..]; + if self.partial.len() < 6 { + return; + } + let point: [u8; 6] = self.partial[..].try_into().expect("six bytes"); + self.point(&point); + self.partial.clear(); + } + let (points, rest) = data.as_chunks::<6>(); + for point in points { + self.point(point); + } + self.partial.extend_from_slice(rest); + } + + fn point(&mut self, chunk: &[u8; 6]) { + const INCIRC: f64 = 281474943156225.0_f64; // ((256 ^ 3) - 1) ^ 2 - for chunk in data.as_chunks::<6>().0 { let mut mx = 0.0_f64; let mut my = 0.0_f64; @@ -260,17 +355,19 @@ fn monte_carlo_pi(data: &[u8]) -> Option { } if mx * mx + my * my < INCIRC { - inmont += 1; + self.inmont += 1; } - mcount += 1; + self.mcount += 1; } - if mcount == 0 { - return None; - } + fn finish(self) -> Option { + if self.mcount == 0 { + return None; + } - let mpi = 4.0_f64 * (inmont as f64 / mcount as f64); - Some((mpi - PI).abs() / PI) + let mpi = 4.0_f64 * (self.inmont as f64 / self.mcount as f64); + Some((mpi - PI).abs() / PI) + } } #[cfg(test)] @@ -493,4 +590,46 @@ mod tests { b"" )); } + + /// Every function over a range of the data gives, over a source read in + /// chunks, exactly the value it gives over the bytes in memory. + #[test] + fn a_range_read_in_chunks_computes_what_the_slice_does() { + use super::super::FuncId::*; + use super::{call, Value}; + use crate::byte_source::{BlockCache, CHUNK}; + + let h: Vec = (0..3 * CHUNK + 5).map(|i| (i * 31 % 251) as u8).collect(); + let cache = BlockCache::with_sizes(std::io::Cursor::new(h.clone()), 7, 64).unwrap(); + let int = Value::Int; + // Ranges starting and ending off the chunk seams, with lengths that + // leave a partial Monte Carlo point at each seam. + let ranges = [ + (0, h.len()), + (1, CHUNK + 1), + (CHUNK - 5, 2 * CHUNK + 3), + (7, 11), + (h.len(), 4), + ]; + for (off, len) in ranges { + let (o, l) = (off as i64, len as i64); + for (func, args) in [ + (MathEntropyData, vec![int(o), int(l)]), + (MathMeanData, vec![int(o), int(l)]), + (MathDeviationData, vec![int(o), int(l), Value::Float(100.5)]), + (MathSerialCorrelationData, vec![int(o), int(l)]), + (MathMonteCarloPiData, vec![int(o), int(l)]), + (MathCountRange, vec![int(0x1f), int(o), int(l)]), + (MathPercentageRange, vec![int(0x1f), int(o), int(l)]), + (MathModeRange, vec![int(o), int(l)]), + (MathCountGlobal, vec![int(0x1f)]), + (MathPercentageGlobal, vec![int(0x1f)]), + (MathModeGlobal, vec![]), + ] { + let want = format!("{:?}", call(func, &h, &args)); + let got = format!("{:?}", call(func, &cache, &args)); + assert_eq!(want, got, "{func:?} over {off}+{len}"); + } + } + } } diff --git a/crates/exav-core/src/yara/modules/mod.rs b/crates/exav-core/src/yara/modules/mod.rs index 789a8303..7fb1c13e 100644 --- a/crates/exav-core/src/yara/modules/mod.rs +++ b/crates/exav-core/src/yara/modules/mod.rs @@ -13,6 +13,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; +use crate::byte_source::ByteSource; use crate::yara::error::{Error, Result}; use crate::yara::ir::{Cond, EvalCtx, Value}; @@ -80,7 +81,41 @@ impl ModuleCtx { /// Builds the module state for one scan. `imported` lists the modules the /// rule set imported; `need_entrypoint` is set when any rule uses the /// `entrypoint` keyword (which needs a PE parse even without `import "pe"`). - pub(crate) fn build(data: &[u8], imported: &[ModuleKind], need_entrypoint: bool) -> Self { + /// + /// The format parsers read the object whole. One not held in memory is + /// read whole when it is at most `materialize` bytes. A larger one gets + /// the value the parser gives when the object is not of its format, which + /// is exact when the object does not start with that format's magic, and + /// the build is otherwise incomplete (the `bool`). + pub(crate) fn build( + src: &dyn ByteSource, + imported: &[ModuleKind], + need_entrypoint: bool, + materialize: usize, + ) -> (Self, bool) { + let wants_pe = need_entrypoint + || imported.contains(&ModuleKind::Pe) + || imported.contains(&ModuleKind::DotNet); + let wants_elf = imported.contains(&ModuleKind::Elf); + let whole = if wants_pe || wants_elf { + src.materialize(materialize) + } else { + None + }; + let mut complete = true; + let data: &[u8] = match &whole { + Some(data) => data, + None => { + let head = src.window(0, 4); + complete = !(wants_pe && head.starts_with(b"MZ") + || wants_elf && head.starts_with(b"\x7fELF")); + &[] + } + }; + (Self::build_from(data, imported, need_entrypoint), complete) + } + + fn build_from(data: &[u8], imported: &[ModuleKind], need_entrypoint: bool) -> Self { let mut roots = HashMap::new(); let mut pe_extra = None; diff --git a/crates/exav-core/src/yara/scanner.rs b/crates/exav-core/src/yara/scanner.rs index 65b0f40b..92e643ab 100644 --- a/crates/exav-core/src/yara/scanner.rs +++ b/crates/exav-core/src/yara/scanner.rs @@ -1,14 +1,16 @@ //! Scanning: running compiled [`Rules`] over a data buffer. use std::collections::HashMap; +use std::sync::OnceLock; use serde::{Deserialize, Serialize}; +use crate::byte_source::ByteSource; use crate::yara::atoms::PatternGate; use crate::yara::compiler::ExternalValue; use crate::yara::error::{Error, Result}; use crate::yara::ir::{Cond, EvalCtx, Value}; -use crate::yara::matcher::{Match, PatternDef, PatternMatcher}; +use crate::yara::matcher::{find_all_windowed, Match, PatternDef, PatternMatcher, StreamRegexes}; use crate::yara::modules::{ModuleCtx, ModuleKind}; /// A compiled rule (internal representation). @@ -22,13 +24,16 @@ pub(crate) struct CompiledRule { pub cond: Cond, } -/// A compiled set of YARA rules, ready to scan. `'static` — owns everything. +/// A compiled set of YARA rules, ready to scan. `'static`: owns everything. pub struct Rules { pub(crate) patterns: Vec, + /// Per pattern, its regexes as a search over a source not held in memory + /// runs them, built from `defs` the first time one is needed. + pub(crate) stream_regexes: Vec>>, /// The serializable definition of each pattern (index == pattern id), kept /// in lockstep with `patterns`. Not used at scan time; carried so the rule /// set can be re-serialized (the compiled `patterns` are recompiled from - /// these on load — see [`Rules::serialize`]/[`Rules::deserialize`]). + /// these on load; see [`Rules::serialize`]/[`Rules::deserialize`]). pub(crate) defs: Vec, pub(crate) pattern_names: Vec, pub(crate) rules: Vec, @@ -100,6 +105,7 @@ impl Rules { } Ok(Rules { patterns, + stream_regexes: blob.defs.iter().map(|_| OnceLock::new()).collect(), defs: blob.defs, pattern_names: blob.pattern_names, rules: blob.rules, @@ -160,25 +166,89 @@ impl<'r> Scanner<'r> { } fn scan_inner<'d>(&self, data: &'d [u8]) -> ScanResults<'r, 'd> { + let e = self.evaluate(&data, usize::MAX); + ScanResults { + rules: self.rules, + data, + matches: e.matches, + matching: e.matching, + steps_exhausted: e.steps_exhausted, + complete: e.complete, + } + } + + /// Scans an object that need not be held in memory. `materialize` is the + /// largest such object read whole where a search or a module needs it so. + pub(crate) fn scan_source( + &self, + src: &dyn ByteSource, + materialize: usize, + ) -> SourceResults<'r> { + let e = self.evaluate(src, materialize); + SourceResults { + matching: e + .matching + .iter() + .map(|&i| self.rules.rules[i].name.as_str()) + .collect(), + steps_exhausted: e.steps_exhausted, + complete: e.complete, + } + } + + fn evaluate(&self, data: &dyn ByteSource, materialize: usize) -> Evaluated { let rules = self.rules; // 1. Find matches for every pattern. The atom prefilter first tells us // which patterns can possibly match this buffer; the rest are known to // have zero matches (a required literal is absent), so we skip their - // O(n) scan and hand them an empty match list — an identical result. + // O(n) scan and hand them an empty match list, an identical result. let run = rules.gate.select(data); + let mut complete = true; + // Over an object not held in memory, the literal and base64 patterns + // are searched in one read of it together. + let mut windowed = vec![Vec::new(); rules.patterns.len()]; + let together: Vec = match data.as_slice() { + Some(_) => Vec::new(), + None => (0..rules.patterns.len()) + .filter(|&i| run[i] && rules.patterns[i].windowed()) + .collect(), + }; + let pats: Vec<&PatternMatcher> = together.iter().map(|&i| &rules.patterns[i]).collect(); + for (&i, found) in together.iter().zip(find_all_windowed(&pats, data)) { + complete &= found.complete; + windowed[i] = found.matches; + } let matches: Vec> = rules .patterns .iter() .enumerate() - .map(|(i, p)| if run[i] { p.find_all(data) } else { Vec::new() }) + .map(|(i, p)| { + if !run[i] { + return Vec::new(); + } + if together.contains(&i) { + return std::mem::take(&mut windowed[i]); + } + let stream = || { + rules.stream_regexes[i] + .get_or_init(|| rules.defs[i].stream_regexes()) + .as_ref() + }; + let found = p.find_all(data, &stream, materialize); + complete &= found.complete; + found.matches + }) .collect(); // 2. Build the per-scan module state (parse PE, etc.) once. let modules = if rules.imported.is_empty() && !rules.uses_entrypoint { ModuleCtx::empty() } else { - ModuleCtx::build(data, &rules.imported, rules.uses_entrypoint) + let (modules, built) = + ModuleCtx::build(data, &rules.imported, rules.uses_entrypoint, materialize); + complete &= built; + modules }; // 3. Evaluate rule conditions in order. Rule references only ever point @@ -225,16 +295,33 @@ impl<'r> Scanner<'r> { Vec::new() }; - ScanResults { - rules, - data, + Evaluated { matches, matching, steps_exhausted, + complete, } } } +/// What evaluating the rules over one object found. +struct Evaluated { + matches: Vec>, + matching: Vec, + steps_exhausted: bool, + complete: bool, +} + +/// The outcome of [`Scanner::scan_source`]. +pub(crate) struct SourceResults<'r> { + /// The rules that matched, in declaration order. + pub matching: Vec<&'r str>, + /// As [`ScanResults::budget_exhausted`]. + pub steps_exhausted: bool, + /// As [`ScanResults::complete`], and the modules could parse the object. + pub complete: bool, +} + /// The outcome of a scan. pub struct ScanResults<'r, 'd> { rules: &'r Rules, @@ -242,6 +329,7 @@ pub struct ScanResults<'r, 'd> { matches: Vec>, matching: Vec, steps_exhausted: bool, + complete: bool, } impl<'r, 'd> ScanResults<'r, 'd> { @@ -263,6 +351,13 @@ impl<'r, 'd> ScanResults<'r, 'd> { pub fn budget_exhausted(&self) -> bool { self.steps_exhausted } + + /// Whether every match of every pattern was recorded. A pattern keeps at + /// most a million matches, as yara-x does; past that the conditions see a + /// partial list and the results are not to be trusted either way. + pub fn complete(&self) -> bool { + self.complete + } } /// Iterator over matching rules. Implements [`ExactSizeIterator`] so callers can @@ -452,6 +547,258 @@ mod budget_tests { } } +#[cfg(test)] +mod source_tests { + use std::io::Cursor; + + use crate::byte_source::{BlockCache, CHUNK}; + use crate::yara::compiler::Compiler; + use crate::yara::scanner::{Evaluated, Rules, Scanner}; + + const RULES: &str = r#" + import "math" + import "hash" + + rule literals { + strings: + $a = "needle" + $b = "NeEdLe" nocase + $c = "wide" wide ascii + $d = "word" fullword + $e = "wword" wide fullword + $f = "xored" xor(1-3) + $g = "xorword" xor(1-2) fullword + condition: any of them + } + rule b64 { + strings: + $p = "password123" base64 + $q = "secret!" base64wide + condition: $p or $q + } + rule hexes { + strings: + $h = { 4D 5A ?? ?? [2-40] 50 45 } + $j = { 6E 65 ( 65 | 64 ) [-] 6C 65 } + condition: #h > 0 or #j > 1 + } + rule regexes { + strings: + $r = /ne+dle[0-9]*/ + $s = /h.llo/is + $t = /c[a-z]+e/ wide ascii + $u = /word[0-9]?/ fullword + $v = /x[^x]{1,300}y/s + condition: any of them + } + rule counts { + strings: + $a = "needle" + condition: #a >= 3 and @a[2] > 10 and !a[1] == 6 and $a in (0..filesize) + } + rule reads { + condition: + uint32(0) != 0 and uint16be(filesize - 2) != 1 and int8(filesize - 1) < 128 + and not defined uint8(filesize) + } + rule maths { + condition: + math.entropy(0, filesize) > 1.0 + and math.mean(3, filesize) >= 0.0 + and math.deviation(0, filesize, 127.5) >= 0.0 + and math.serial_correlation(1, filesize) != 7.0 + and math.monte_carlo_pi(5, filesize) >= 0.0 + and math.count(0x61, 0, filesize) >= 0 + and math.percentage(0x61) >= 0.0 + and math.mode(0, filesize) >= 0 + } + rule hashes { + condition: + hash.md5(0, filesize) != "" + and hash.sha1(1, filesize - 1) != "" + and hash.sha256(0, filesize) != "" + and hash.crc32(2, filesize - 2) != 0 + and hash.checksum32(0, filesize) != 1 + } + "#; + + fn rules(src: &str) -> Rules { + let mut c = Compiler::new(); + c.add_source(src).expect("compile"); + c.build() + } + + fn evaluate( + rules: &Rules, + src: &dyn crate::byte_source::ByteSource, + materialize: usize, + ) -> Evaluated { + Scanner::new(rules).evaluate(src, materialize) + } + + /// Pattern text, one of each kind the rules look for, and near misses + /// that only the bytes around them rule out. + fn plants() -> Vec> { + let wide = |s: &[u8]| -> Vec { s.iter().flat_map(|&b| [b, 0]).collect() }; + let mut plants: Vec> = [ + &b"needle"[..], + b"NEEDLE7", + b" word ", + b"xword", + b"wordx", + b"cGFzc3dvcmQxMjM=", + // "Apassword123" and "ABpassword123": decoded from before the match. + b"QXBhc3N3b3JkMTIz", + b"QUJwYXNzd29yZDEyMw==", + // A character past the encoded text that fails the decode. + b"cGFzc3dvcmQxMjM!", + b"MZ\x90\x00 some bytes PE", + b"x long gap y", + b"hElLo", + b"neddle", + ] + .iter() + .map(|p| p.to_vec()) + .collect(); + for s in [&b"wide"[..], b"cake", b" wword ", b"awword", b"wwordz"] { + plants.push(wide(s)); + } + // "secret!", "Asecret!" and "ABsecret!", base64 then widened. + for s in [ + &b"c2VjcmV0IQ=="[..], + b"QXNlY3JldCE=", + b"QUJzZWNyZXQh", + b"c2VjcmV0IQ!!", + ] { + plants.push(wide(s)); + } + plants.push(b"xored".iter().map(|b| b ^ 2).collect()); + plants.push(b" xorword ".iter().map(|b| b ^ 1).collect()); + plants.push(b"axorword".iter().map(|b| b ^ 1).collect()); + plants + } + + /// A haystack of noise with `plants()[kind]` planted at each `(at, kind)`. + fn haystack(len: usize, spots: &[(usize, usize)], seed: u64) -> Vec { + let mut state = seed | 1; + let mut h: Vec = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + b"abcdehlnorwxyz \x00\n.MZPE"[(state % 20) as usize] + }) + .collect(); + let plants = plants(); + for &(at, kind) in spots { + let plant = &plants[kind % plants.len()]; + let at = at.min(len.saturating_sub(plant.len())); + h[at..at + plant.len()].copy_from_slice(plant); + } + h + } + + fn assert_same(rules: &Rules, h: &[u8], block: usize, what: &str) { + let want = evaluate(rules, &h, usize::MAX); + let cache = BlockCache::with_sizes(Cursor::new(h.to_vec()), block, 8 * block).unwrap(); + // Nothing may be read whole: every search runs through the cache. + let got = evaluate(rules, &cache, 0); + assert!( + want.complete && got.complete, + "{what}: a search was cut short" + ); + for (i, (w, g)) in want.matches.iter().zip(&got.matches).enumerate() { + assert_eq!(w, g, "{what}: pattern {} ({})", i, rules.pattern_names[i]); + } + assert_eq!(want.matching, got.matching, "{what}"); + } + + #[test] + fn a_streamed_object_matches_as_in_memory() { + let rules = rules(RULES); + let kinds = plants().len(); + for seed in 1..6 { + let spots: Vec<(usize, usize)> = + (0..kinds).map(|k| (k * 23, k + seed as usize)).collect(); + let h = haystack(kinds * 23 + seed as usize * 7, &spots, seed); + assert_same(&rules, &h, 7, &format!("small seed {seed}")); + } + // Each kind of plant straddling a seam between two search windows, at + // every split, and a few more plants away from the seams. + let mut spots = Vec::new(); + for kind in 0..kinds { + let len = plants()[kind].len(); + for cut in 1..len { + let seam = (1 + spots.len()) * CHUNK; + spots.push((seam - cut, kind)); + } + } + let seams = spots.len(); + spots.extend((0..kinds).map(|k| (5 + k * 31, k))); + let h = haystack((seams + 2) * CHUNK, &spots, 99); + // Blocks that never line up with the windows. + assert_same(&rules, &h, 509, "seams"); + // Every rule must have had something to find, or the comparison proves + // little. + let e = evaluate(&rules, &h, usize::MAX); + let names: Vec<&str> = e + .matching + .iter() + .map(|&i| rules.rules[i].name.as_str()) + .collect(); + assert_eq!( + names, + ["literals", "b64", "hexes", "regexes", "counts", "reads", "maths", "hashes"] + ); + } + + #[test] + fn a_regex_the_stepped_search_cannot_follow_is_read_whole_or_reported() { + // A Unicode word boundary against non-ASCII bytes stops the lazy DFA. + let rules = rules(r#"rule u { strings: $a = /(?u)\bcaf/ condition: $a }"#); + let h = b"\xe9caf caf".to_vec(); + let want = evaluate(&rules, &h, usize::MAX); + let cache = BlockCache::with_sizes(Cursor::new(h.clone()), 3, 12).unwrap(); + let whole = evaluate(&rules, &cache, usize::MAX); + assert!(whole.complete); + assert_eq!(want.matches, whole.matches); + let cut = evaluate(&rules, &cache, 0); + assert!(!cut.complete, "a search that could not run must say so"); + } + + #[test] + fn modules_read_a_large_object_only_when_it_may_be_of_their_format() { + let rules = rules( + r#"import "pe" + rule p { condition: pe.is_pe and pe.number_of_sections > 0 } + rule n { condition: not pe.is_pe }"#, + ); + let pe = include_bytes!("../../tests/testdata/tiny_pe32.exe").to_vec(); + let cache = BlockCache::with_sizes(Cursor::new(pe.clone()), 64, 1024).unwrap(); + let whole = evaluate(&rules, &cache, usize::MAX); + assert!(whole.complete); + assert_eq!(whole.matching, evaluate(&rules, &pe, usize::MAX).matching); + assert_eq!(whole.matching, [0]); + // Too large to read whole, and it starts as a PE does. + assert!(!evaluate(&rules, &cache, 0).complete); + // Not a PE: the parser's answer is known without reading it. + let other = b"PK\x03\x04 not a PE".to_vec(); + let cache = BlockCache::with_sizes(Cursor::new(other), 4, 16).unwrap(); + let e = evaluate(&rules, &cache, 0); + assert!(e.complete); + assert_eq!(e.matching, [1]); + } + + #[test] + fn a_pattern_past_the_match_cap_is_incomplete_in_memory_too() { + let rules = rules(r#"rule z { strings: $a = { 00 00 } condition: #a > 5 }"#); + let h = vec![0u8; super::super::matcher::MAX_MATCHES + 10]; + let e = evaluate(&rules, &h, usize::MAX); + assert_eq!(e.matches[0].len(), super::super::matcher::MAX_MATCHES); + assert!(!e.complete); + } +} + #[cfg(test)] mod serialize_tests { use crate::yara::compiler::Compiler; @@ -627,7 +974,7 @@ mod serialize_tests { return; }; - // Compile from source (lenient, per-rule) — the cost a load must avoid. + // Compile from source (lenient, per-rule): the cost a load must avoid. let t0 = Instant::now(); let mut c = Compiler::new(); let _rejected = c.add_source_lenient(&src); @@ -643,7 +990,7 @@ mod serialize_tests { let serialize = t1.elapsed(); // Deserialize (the load path): reconstruct the atom automaton + IR, and - // recompile only the per-pattern regexes — skipping the automaton BUILD. + // recompile only the per-pattern regexes, skipping the automaton BUILD. let t2 = Instant::now(); let loaded = Rules::deserialize(&bytes).expect("deserialize"); let deserialize = t2.elapsed(); diff --git a/crates/exav-core/tests/suites/allmatch_parity.rs b/crates/exav-core/tests/suites/allmatch_parity.rs index 4ee33212..8fe4c5ce 100644 --- a/crates/exav-core/tests/suites/allmatch_parity.rs +++ b/crates/exav-core/tests/suites/allmatch_parity.rs @@ -1,10 +1,9 @@ //! `--all-matches` must be a **superset** of a normal scan, never a subset. //! -//! It is easy to assume this holds by construction and it does not. The two -//! paths are separate walks — `deep_analyze` stops at the first detection, -//! `collect_all` visits everything — and any capability added to one and not the -//! other silently changes *which signatures run*, not just how many names come -//! back. +//! It holds by construction now (one walk, driven by a sink that stops or +//! collects), and it did not when the two paths were separate walks: any +//! capability added to one and not the other silently changed *which +//! signatures run*, not just how many names come back. //! //! That happened: `collect_all` typed every extracted member with plain content //! detection, while the first-match path forces textual OLE2 streams to type as @@ -79,8 +78,8 @@ fn allmatch_is_a_superset_of_a_normal_scan() { assert!( all.iter().any(|n| n == "Test.OleMacro"), "--all-matches dropped a detection a normal scan makes: {all:?}. \ - `collect_all` must apply the same forced member type as `deep_analyze` — \ - a textual OLE2 stream is scanned AS `Ole` so `Target:2` signatures run." + A textual OLE2 stream is scanned AS `Ole` so `Target:2` signatures run, \ + in both modes." ); } diff --git a/crates/exav-core/tests/suites/broken_executable.rs b/crates/exav-core/tests/suites/broken_executable.rs new file mode 100644 index 00000000..d5616c44 --- /dev/null +++ b/crates/exav-core/tests/suites/broken_executable.rs @@ -0,0 +1,733 @@ +//! `Heuristics.Broken.Executable` judges the headers a loader maps, not the +//! data they point at. +//! +//! Every expectation below was checked against clamscan 1.5.4 +//! (`--alert-broken`) on these exact bytes. The rules were found the same way, +//! by mutating one header field at a time; the corpus check behind them is 4,134 +//! PE, ELF and Mach-O files extracted from archives, where exav and clamscan +//! now agree on all of them. +//! +//! Before, any file goblin could not parse in full was "broken": a DLL with a +//! non-UTF-8 export name, a resource directory pointing nowhere, an Android +//! `.so` with junk section headers. Meanwhile PEs whose alignment or section +//! layout no loader accepts parsed fine and passed. + +use exav_core::{analyze, loader, ScanOptions, Scanner, Verdict}; + +fn empty_db() -> Scanner { + let mut l = loader::Builder::new(); + l.add_named_bytes("t.ndb", b"Zzz.Never:0:*:deadbeefdeadbeef\n", true); + l.build().expect("build database") +} + +fn broken(db: &Scanner, blob: &[u8]) -> bool { + let mut opts = ScanOptions::default(); + opts.alert_broken = true; + opts.clamav_compat = true; + match analyze(db, blob, &opts).verdict { + Verdict::Infected { signature, .. } => { + assert_eq!(signature, "Heuristics.Broken.Executable"); + true + } + _ => false, + } +} + +fn check(cases: Vec<(&str, Vec, bool)>) { + let db = empty_db(); + let wrong: Vec = cases + .iter() + .filter(|(_, blob, want)| broken(&db, blob) != *want) + .map(|(name, _, want)| format!("{name}: expected broken={want}")) + .collect(); + assert!( + wrong.is_empty(), + "{} of {} wrong:\n{}", + wrong.len(), + cases.len(), + wrong.join("\n") + ); +} + +fn put16(b: &mut [u8], at: usize, v: u16) { + b[at..at + 2].copy_from_slice(&v.to_le_bytes()); +} + +fn put32(b: &mut [u8], at: usize, v: u32) { + b[at..at + 4].copy_from_slice(&v.to_le_bytes()); +} + +// ---------------------------------------------------------------------- PE + +#[derive(Clone, Copy)] +struct Sec { + vs: u32, + va: u32, + rs: u32, + rp: u32, +} + +const fn sec(vs: u32, va: u32, rs: u32, rp: u32) -> Sec { + Sec { vs, va, rs, rp } +} + +/// A two-section image laid out the way a linker would: headers in the first +/// 0x200 bytes, `.text` and `.data` at 0x200 and 0x400 in the file and at +/// 0x1000 and 0x2000 in memory. Each case changes what it needs. +#[derive(Clone)] +struct Pe { + pe64: bool, + nsec: Option, + optsz: Option, + magic: Option, + subsystem: u16, + ep: u32, + salign: u32, + falign: u32, + soh: u32, + nrva: u32, + dirs: Vec<(usize, u32, u32)>, + secs: Vec, + len: Option, +} + +impl Default for Pe { + fn default() -> Self { + Pe { + pe64: false, + nsec: None, + optsz: None, + magic: None, + subsystem: 2, + ep: 0x1000, + salign: 0x1000, + falign: 0x200, + soh: 0x200, + nrva: 16, + dirs: Vec::new(), + secs: vec![ + sec(0x200, 0x1000, 0x200, 0x200), + sec(0x200, 0x2000, 0x200, 0x400), + ], + len: None, + } + } +} + +impl Pe { + fn build(&self) -> Vec { + let fixed: usize = if self.pe64 { 112 } else { 96 }; + let optsz = self.optsz.map_or(fixed + 128, usize::from); + let nsec = self.nsec.unwrap_or(self.secs.len() as u16); + let opt = 0x80 + 24; + let table = opt + optsz; + let body = table + self.secs.len() * 40; + let data_end = self + .secs + .iter() + .filter(|s| s.rp < 0x10000) + .map(|s| (s.rp + s.rs) as usize); + let mut b = vec![0u8; data_end.max().unwrap_or(0).max(body).max(0x600)]; + b[..2].copy_from_slice(b"MZ"); + put32(&mut b, 0x3c, 0x80); + b[0x80..0x84].copy_from_slice(b"PE\0\0"); + put16(&mut b, 0x84, if self.pe64 { 0x8664 } else { 0x14c }); + put16(&mut b, 0x86, nsec); + put16(&mut b, 0x94, optsz as u16); + put16(&mut b, 0x96, 0x0102); + // Optional header fields that stay in place across both widths, then + // the data directories. + let mut o = vec![0u8; fixed + 128]; + put16( + &mut o, + 0, + self.magic.unwrap_or(if self.pe64 { 0x20b } else { 0x10b }), + ); + put32(&mut o, 16, self.ep); + put32(&mut o, 32, self.salign); + put32(&mut o, 36, self.falign); + put32(&mut o, 56, 0x3000); + put32(&mut o, 60, self.soh); + put16(&mut o, 68, self.subsystem); + put32(&mut o, fixed - 4, self.nrva); + for &(i, rva, size) in &self.dirs { + put32(&mut o, fixed + i * 8, rva); + put32(&mut o, fixed + i * 8 + 4, size); + } + o.resize(optsz, 0); + b[opt..table].copy_from_slice(&o); + for (i, s) in self.secs.iter().enumerate() { + let at = table + i * 40; + b[at..at + 5].copy_from_slice(b".sect"); + put32(&mut b, at + 8, s.vs); + put32(&mut b, at + 12, s.va); + put32(&mut b, at + 16, s.rs); + put32(&mut b, at + 20, s.rp); + } + b[0x200] = 0xc3; + if let Some(len) = self.len { + b.resize(len, 0); + } + b + } +} + +fn pe(f: impl FnOnce(&mut Pe)) -> Vec { + let mut p = Pe::default(); + f(&mut p); + p.build() +} + +#[test] +fn pe_rules() { + let many: Vec = std::iter::once(sec(0x200, 0x2000, 0x200, 0x2000)) + .chain((1..1000).map(|i| sec(0x1000, 0x1000 * (i + 2), 0, 0))) + .collect(); + check(vec![ + ("PE32", pe(|_| {}), false), + ("PE32+", pe(|p| p.pe64 = true), false), + // What a loader does not need to map the image. + ( + "import directory pointing nowhere", + pe(|p| p.dirs = vec![(1, 0x7fff0000, 0x100)]), + false, + ), + ( + "resource directory pointing nowhere", + pe(|p| p.dirs = vec![(2, 0x7fff0000, 0x100)]), + false, + ), + ( + "relocations pointing nowhere", + pe(|p| p.dirs = vec![(5, 0x7fff0000, 0x100)]), + false, + ), + ( + "CLR header pointing nowhere", + pe(|p| p.dirs = vec![(14, 0x7fff0000, 0x100)]), + false, + ), + ( + "unknown optional-header magic", + pe(|p| p.magic = Some(0x999)), + false, + ), + ( + "NumberOfRvaAndSizes above 16", + pe(|p| p.nrva = 0x100), + false, + ), + ( + "raw data running past EOF", + pe(|p| p.secs[1].rs = 0x10000), + false, + ), + ( + "last section starting past EOF", + pe(|p| p.secs[1].rp = 0x10000), + false, + ), + ( + "empty section starting past EOF", + pe(|p| (p.secs[1].rp, p.secs[1].rs) = (0x10000, 0)), + false, + ), + ("entry point 0", pe(|p| p.ep = 0), false), + ("entry point in the headers", pe(|p| p.ep = 0x10), false), + ( + "entry point in raw-size padding", + pe(|p| (p.ep, p.secs[0].rs) = (0x1180, 0x150)), + false, + ), + ("FileAlignment 0x600", pe(|p| p.falign = 0x600), false), + ( + "1000 sections", + pe(|p| (p.soh, p.ep, p.secs) = (0x2000, 0x2000, many.clone())), + false, + ), + ( + "cut inside the COFF header", + pe(|p| p.len = Some(0x8a)), + false, + ), + ( + "cut inside the data directories", + pe(|p| p.len = Some(0x80 + 24 + 100)), + false, + ), + ( + "native driver with 0x80 alignment", + pe(|p| { + (p.subsystem, p.salign, p.falign, p.soh, p.ep) = (1, 0x80, 0x80, 0x200, 0x200); + p.secs = vec![ + sec(0x200, 0x200, 0x200, 0x200), + sec(0x200, 0x400, 0x200, 0x400), + ]; + }), + false, + ), + ( + "native with FileAlignment 0", + pe(|p| (p.subsystem, p.falign) = (1, 0)), + false, + ), + // What it does. + ("no sections", pe(|p| p.nsec = Some(0)), true), + ( + "optional header without its data directories", + pe(|p| p.optsz = Some(0x60)), + true, + ), + ( + "optional header below its fixed part", + pe(|p| (p.optsz, p.nrva) = (Some(95), 0)), + true, + ), + ( + "PE32+ with a PE32-sized optional header", + pe(|p| (p.pe64, p.optsz) = (true, Some(224))), + true, + ), + ( + "cut inside the fixed optional header", + pe(|p| p.len = Some(0x80 + 24 + 50)), + true, + ), + ( + "section table cut short", + pe(|p| p.len = Some(0x80 + 24 + 224 + 60)), + true, + ), + ("FileAlignment 0", pe(|p| p.falign = 0), true), + ("FileAlignment 0x100", pe(|p| p.falign = 0x100), true), + ("FileAlignment 0x300", pe(|p| p.falign = 0x300), true), + ("SectionAlignment 0x800", pe(|p| p.salign = 0x800), true), + ( + "native with SectionAlignment 0", + pe(|p| (p.subsystem, p.salign) = (1, 0)), + true, + ), + ( + "first section not right after the headers", + pe(|p| p.secs[0].va = 0x2000), + true, + ), + ("gap between sections", pe(|p| p.secs[1].va = 0x3000), true), + ("overlapping sections", pe(|p| p.secs[1].va = 0x1000), true), + ("entry point outside the image", pe(|p| p.ep = 0x8000), true), + ( + "entry point past the raw data", + pe(|p| (p.ep, p.secs[0].vs) = (0x1400, 0x800)), + true, + ), + ( + "entry point in raw data past EOF", + pe(|p| p.secs[0].rp = 0x10000), + true, + ), + ( + "only section starting past EOF", + pe(|p| (p.nsec, p.ep, p.secs[0].rp) = (Some(1), 0x10, 0x10000)), + true, + ), + // A section whose raw data starts past EOF leaves the layout, so the + // one after it is misplaced: the shape of every such file in the corpus. + ( + "middle section starting past EOF", + pe(|p| { + p.secs = vec![ + sec(0x200, 0x1000, 0x200, 0x200), + sec(0x800, 0x2000, 0x200, 0x10000), + sec(0x200, 0x3000, 0x200, 0x400), + ] + }), + true, + ), + ]); +} + +// --------------------------------------------------------------------- ELF + +/// One `PT_LOAD` over the whole file, `.text` and `.shstrtab` sections. +#[derive(Clone)] +struct Elf { + wide: bool, + be: bool, + class: Option, + data: Option, + entry: Option, + phoff: Option, + phentsize: Option, + phnum: u16, + shoff: Option, + shentsize: Option, + shnum: u16, + ptype: u32, + memsz: Option, + text_off: Option, + len: Option, +} + +impl Default for Elf { + fn default() -> Self { + Elf { + wide: true, + be: false, + class: None, + data: None, + entry: None, + phoff: None, + phentsize: None, + phnum: 1, + shoff: None, + shentsize: None, + shnum: 3, + ptype: 1, + memsz: None, + text_off: None, + len: None, + } + } +} + +impl Elf { + fn build(&self) -> Vec { + let (eh, ph, sh) = if self.wide { + (64, 56, 64) + } else { + (52, 32, 40) + }; + let base = 0x40_0000u64; + let code = eh + ph; + let shstr = b"\0.text\0.shstrtab\0"; + let table = (code + 16 + shstr.len() + 7) & !7; + let total = table + 3 * sh; + let mut b = vec![0u8; total]; + let w = |b: &mut Vec, at: usize, v: u64, n: usize| { + let bytes = if self.be { + v.to_be_bytes() + } else { + v.to_le_bytes() + }; + let bytes = if self.be { + &bytes[8 - n..] + } else { + &bytes[..n] + }; + b[at..at + n].copy_from_slice(bytes); + }; + let word = if self.wide { 8 } else { 4 }; + b[..4].copy_from_slice(b"\x7fELF"); + b[4] = self.class.unwrap_or(if self.wide { 2 } else { 1 }); + b[5] = self.data.unwrap_or(if self.be { 2 } else { 1 }); + b[6] = 1; + w(&mut b, 16, 2, 2); // EXEC + w(&mut b, 18, if self.wide { 0x3e } else { 3 }, 2); + w(&mut b, 20, 1, 4); + w(&mut b, 24, self.entry.unwrap_or(base + code as u64), word); + w(&mut b, 24 + word, self.phoff.unwrap_or(eh as u64), word); + w( + &mut b, + 24 + 2 * word, + self.shoff.unwrap_or(table as u64), + word, + ); + let f = 28 + 3 * word; // e_ehsize + w(&mut b, f, eh as u64, 2); + w( + &mut b, + f + 2, + u64::from(self.phentsize.unwrap_or(ph as u16)), + 2, + ); + w(&mut b, f + 4, u64::from(self.phnum), 2); + w( + &mut b, + f + 6, + u64::from(self.shentsize.unwrap_or(sh as u16)), + 2, + ); + w(&mut b, f + 8, u64::from(self.shnum), 2); + w(&mut b, f + 10, 2, 2); + let memsz = self.memsz.unwrap_or(total as u64); + if self.wide { + w(&mut b, eh, u64::from(self.ptype), 4); + w(&mut b, eh + 4, 5, 4); + w(&mut b, eh + 16, base, 8); + w(&mut b, eh + 24, base, 8); + w(&mut b, eh + 32, total as u64, 8); + w(&mut b, eh + 40, memsz, 8); + } else { + w(&mut b, eh, u64::from(self.ptype), 4); + w(&mut b, eh + 8, base, 4); + w(&mut b, eh + 12, base, 4); + w(&mut b, eh + 16, total as u64, 4); + w(&mut b, eh + 20, memsz, 4); + w(&mut b, eh + 24, 5, 4); + } + b[code] = 0xc3; + b[code + 16..code + 16 + shstr.len()].copy_from_slice(shstr); + // Section headers: null, .text, .shstrtab. Name, type, then the file + // offset and size at their width-dependent places. + let (off_at, size_at) = if self.wide { (24, 32) } else { (16, 20) }; + for (i, (name, kind, off, size)) in [ + (1u64, 1u64, self.text_off.unwrap_or(code as u64), 16u64), + (7, 3, (code + 16) as u64, shstr.len() as u64), + ] + .into_iter() + .enumerate() + { + let at = table + (i + 1) * sh; + w(&mut b, at, name, 4); + w(&mut b, at + 4, kind, 4); + w(&mut b, at + off_at, off, word); + w(&mut b, at + size_at, size, word); + } + if let Some(len) = self.len { + b.truncate(len); + } + b + } +} + +#[test] +fn elf_rules() { + let mut cases = Vec::new(); + for (wide, be) in [(true, false), (true, true), (false, false), (false, true)] { + let t = format!( + "ELF{}{}", + if wide { 64 } else { 32 }, + if be { "BE" } else { "LE" } + ); + let elf = |f: &dyn Fn(&mut Elf)| { + let mut e = Elf { + wide, + be, + ..Elf::default() + }; + f(&mut e); + e.build() + }; + let (eh, sh) = if wide { (64, 64) } else { (52, 40) }; + for (what, blob, want) in [ + ("base", elf(&|_| {}), false), + // Section contents and segment data are not the loader's concern. + ( + "section data past EOF", + elf(&|e| e.text_off = Some(0x10_0000)), + false, + ), + ( + "header shorter than its own size", + elf(&|e| e.len = Some(eh - 10)), + false, + ), + ("memsz below filesz", elf(&|e| e.memsz = Some(0x100)), false), + ("entry point 0", elf(&|e| e.entry = Some(0)), false), + ( + "entry point in a non-LOAD segment", + elf(&|e| e.ptype = 4), + false, + ), + ( + "no program headers, entry anywhere", + elf(&|e| (e.phnum, e.entry) = (0, Some(0x10))), + false, + ), + ( + "no sections, table offset past EOF", + elf(&|e| (e.shnum, e.shoff) = (0, Some(0x10_0000))), + false, + ), + // The tables the loader and the section walk read. + ("class 3", elf(&|e| e.class = Some(3)), true), + ( + "program header table past EOF", + elf(&|e| e.phoff = Some(0x10_0000)), + true, + ), + ( + "program header entry size wrong", + elf(&|e| e.phentsize = Some(24)), + true, + ), + ("129 program headers", elf(&|e| e.phnum = 129), true), + ( + "section header table past EOF", + elf(&|e| e.shoff = Some(0x10_0000)), + true, + ), + ( + "section header entry size wrong", + elf(&|e| e.shentsize = Some(sh - 8)), + true, + ), + ( + "entry point outside every segment", + elf(&|e| e.entry = Some(0x10)), + true, + ), + ( + "entry point at the end of its segment", + elf(&|e| (e.memsz, e.entry) = (Some(0x100), Some(0x40_0100))), + true, + ), + ( + "cut inside the program headers", + elf(&|e| e.len = Some(eh + 10)), + true, + ), + ( + "cut inside the section headers", + { + let b = elf(&|_| {}); + b[..b.len() - 10].to_vec() + }, + true, + ), + ] { + cases.push((format!("{t}: {what}"), blob, want)); + } + // EI_DATA other than 1 reads the file as big-endian. + cases.push((format!("{t}: EI_DATA 0"), elf(&|e| e.data = Some(0)), !be)); + } + check( + cases + .iter() + .map(|(n, b, w)| (n.as_str(), b.clone(), *w)) + .collect(), + ); +} + +// ------------------------------------------------------------------ Mach-O + +/// A `__TEXT` segment with one section, then `LC_MAIN` (or `LC_UNIXTHREAD`). +fn macho(wide: bool, be: bool, cpu: Option, thread: bool) -> Vec { + let w32 = |v: u32| if be { v.to_be_bytes() } else { v.to_le_bytes() }; + let w64 = |v: u64| if be { v.to_be_bytes() } else { v.to_le_bytes() }; + let mut seg = Vec::new(); + seg.extend(w32(if wide { 0x19 } else { 0x1 })); + seg.extend(w32(if wide { 72 + 80 } else { 56 + 68 })); + seg.extend(*b"__TEXT\0\0\0\0\0\0\0\0\0\0"); + if wide { + for v in [0x1_0000_0000u64, 0x1000, 0, 0x1000] { + seg.extend(w64(v)); + } + } else { + for v in [0x1000u32, 0x1000, 0, 0x1000] { + seg.extend(w32(v)); + } + } + for v in [5u32, 5, 1, 0] { + seg.extend(w32(v)); // maxprot, initprot, nsects, flags + } + seg.extend(*b"__text\0\0\0\0\0\0\0\0\0\0__TEXT\0\0\0\0\0\0\0\0\0\0"); + seg.resize(if wide { 72 + 80 } else { 56 + 68 }, 0); + let mut cmd = Vec::new(); + if thread { + cmd.extend(w32(0x5)); + cmd.extend(w32(16 + 64)); + cmd.extend(w32(1)); + cmd.extend(w32(16)); + cmd.resize(16 + 64, 0); + } else { + cmd.extend(w32(0x8000_0028)); + cmd.extend(w32(24)); + cmd.extend(w64(0x800)); + cmd.extend(w64(0)); + } + let mut b = Vec::new(); + b.extend(w32(if wide { 0xfeed_facf } else { 0xfeed_face })); + b.extend(w32(cpu.unwrap_or(if wide { 0x0100_0007 } else { 7 }))); + b.extend(w32(3)); + b.extend(w32(2)); + b.extend(w32(2)); + b.extend(w32((seg.len() + cmd.len()) as u32)); + b.extend(w32(0x85)); + if wide { + b.extend(w32(0)); + } + b.extend(seg); + b.extend(cmd); + b.resize(0x1000, 0); + b[0x800] = 0xc3; + b +} + +#[test] +fn macho_rules() { + let mut cases = Vec::new(); + for (wide, be) in [(true, false), (true, true), (false, false), (false, true)] { + let t = format!( + "Mach-O{}{}", + if wide { 64 } else { 32 }, + if be { "BE" } else { "LE" } + ); + let hs = if wide { 32 } else { 28 }; + let seg = if wide { 72 + 80 } else { 56 + 68 }; + let put = |b: &mut Vec, at: usize, v: u32| { + b[at..at + 4].copy_from_slice(&if be { v.to_be_bytes() } else { v.to_le_bytes() }) + }; + let base = macho(wide, be, None, false); + let with = |at: usize, v: u32| { + let mut b = base.clone(); + put(&mut b, at, v); + b + }; + let i386 = macho(wide, be, Some(7), true); + let x86_64 = macho(wide, be, Some(0x0100_0007), true); + let ppc = macho(wide, be, Some(18), true); + for (what, blob, want) in [ + ("base", base.clone(), false), + ("header cut short", base[..20].to_vec(), false), + ("sizeofcmds wrong", with(20, 0x10_0000), false), + ("segment cmdsize 0", with(hs + 4, 0), false), + ("segment cmdsize past EOF", with(hs + 4, 0x10000), false), + ( + "x86_64 thread state cut short", + x86_64[..hs + seg + 12].to_vec(), + false, + ), + ("i386 thread state whole", i386.clone(), false), + ("no load commands", with(16, 0), true), + ( + "more load commands than the file holds", + with(16, 1000), + true, + ), + ("segment cut short", base[..hs + 30].to_vec(), true), + ( + "nsects past EOF", + with(hs + if wide { 64 } else { 48 }, 100_000), + true, + ), + ( + "LC_MAIN cmdsize past EOF, then a command", + { + let mut b = with(16, 3); + put(&mut b, hs + seg + 4, 0x10000); + b + }, + true, + ), + ( + "i386 thread state cut short", + i386[..hs + seg + 16 + 4].to_vec(), + true, + ), + ( + "PPC thread state cut short", + ppc[..hs + seg + 16 + 4].to_vec(), + true, + ), + ] { + cases.push((format!("{t}: {what}"), blob, want)); + } + } + check( + cases + .iter() + .map(|(n, b, w)| (n.as_str(), b.clone(), *w)) + .collect(), + ); +} diff --git a/crates/exav-core/tests/suites/bytecode_gating.rs b/crates/exav-core/tests/suites/bytecode_gating.rs index 1079a6a9..bdfed809 100644 --- a/crates/exav-core/tests/suites/bytecode_gating.rs +++ b/crates/exav-core/tests/suites/bytecode_gating.rs @@ -1,10 +1,11 @@ //! End-to-end test of bytecode trigger-gating: a synthetic `.cbc` whose //! logical trigger fires on a byte pattern and whose program calls //! `setvirusname`, exercised through [`BytecodeRuntime`]. No ClamAV data is -//! vendored — the program is assembled with the format's own nibble encoders. +//! vendored: the program is assembled with the format's own nibble encoders. use exav_core::bytecode::runtime::BytecodeRuntime; use exav_core::filetype::FileType; +use exav_core::{analyze, analyze_all, ScanOptions, Verdict}; const HEADER_MAGIC: u64 = 0x53e5_493e_9f3d_1c30; @@ -97,25 +98,62 @@ fn synth_cbc(name: &str, marker: &[u8]) -> String { #[test] fn trigger_gated_detection() { let cbc = synth_cbc("Synth.BC.Detect", b"MALWARE"); - let rt = BytecodeRuntime::from_sources(vec![cbc]); + let (rt, triggers) = BytecodeRuntime::standalone(vec![cbc]); assert_eq!(rt.len(), 1, "program should load"); // Input containing the trigger marker -> the program runs and names it. let hit = b"....prefix....MALWARE....suffix...."; assert_eq!( - rt.scan(hit, FileType::Unknown, None).0, + rt.scan(&triggers, hit, FileType::Unknown, None).0, Some(("Synth.BC.Detect".to_string(), 0)) ); // Input without the marker -> trigger doesn't fire -> no detection. let clean = b"nothing to see here, totally benign bytes"; - assert_eq!(rt.scan(clean, FileType::Unknown, None).0, None); + assert_eq!(rt.scan(&triggers, clean, FileType::Unknown, None).0, None); +} + +/// In a real scan the trigger is matched by the scanner's own engine, in the +/// sweep that looks for every other signature: the program runs, its +/// detection is reported, and the trigger itself neither alerts nor counts +/// as a signature. Through a built database too. +#[test] +fn trigger_gated_detection_in_a_scan() { + let mut loader = exav_core::loader::Builder::new(); + loader.add_named_bytes( + "t.cbc", + synth_cbc("Synth.BC.Detect", b"MALWARE").as_bytes(), + true, + ); + loader.add_named_bytes("t.ndb", b"Other.Sig:0:*:4f54484552\n", true); + let built = loader.build().unwrap(); + let mut file = Vec::new(); + exav_core::database::write(&built, &mut file).unwrap(); + let stored = exav_core::database::read(&file[..]).unwrap(); + for db in [&built, &stored] { + // EICAR, the one `.ndb` line, and not the trigger. + assert_eq!(db.signature_count(), 2); + let found = |data: &[u8]| match analyze(db, data, &ScanOptions::default()).verdict { + Verdict::Infected { signature, .. } => Some(signature), + _ => None, + }; + assert_eq!( + found(b"..prefix..MALWARE..suffix.."), + Some("Synth.BC.Detect".to_string()) + ); + assert_eq!(found(b"..OTHER.."), Some("Other.Sig".to_string())); + assert_eq!(found(b"nothing to see here"), None); + let all = analyze_all(db, b"..MALWARE..OTHER..", &ScanOptions::default()); + let mut names: Vec<_> = all.iter().map(|(name, _)| name.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["Other.Sig", "Synth.BC.Detect"]); + } } #[test] fn forced_mode_runs_regardless_of_trigger() { let cbc = synth_cbc("Synth.BC.Detect", b"MALWARE"); - let rt = BytecodeRuntime::from_sources(vec![cbc]); + let (rt, _) = BytecodeRuntime::standalone(vec![cbc]); // Forced execution ignores the trigger: even clean input runs the program, // which unconditionally names the detection. let out = rt.run_forced(0, b"clean input").expect("program exists"); diff --git a/crates/exav-core/tests/suites/heuristic_names.rs b/crates/exav-core/tests/suites/heuristic_names.rs index 504bcbd8..68ba1c0d 100644 --- a/crates/exav-core/tests/suites/heuristic_names.rs +++ b/crates/exav-core/tests/suites/heuristic_names.rs @@ -1,16 +1,16 @@ //! Heuristic alert names must match ClamAV's **exactly**. //! //! A heuristic reaches a client as an ordinary `FOUND` whose *name* carries all -//! the information — there is no separate channel. Gateways that treat +//! the information; there is no separate channel. Gateways that treat //! heuristics differently (tag rather than block, lower score) do it by matching //! the string. So a name that is one suffix off is not cosmetic: it is a policy //! rule that silently stops firing after a migration. //! //! Two names were wrong and are pinned here: //! -//! * `Heuristics.OLE2.ContainsMacros` — ClamAV suffixes the macro dialect, +//! * `Heuristics.OLE2.ContainsMacros`: ClamAV suffixes the macro dialect, //! `.VBA` or `.XLM`. exav emitted the bare name, matching neither. -//! * `Heuristics.Phishing.Email.Cloaked.IP` — ClamAV calls it +//! * `Heuristics.Phishing.Email.Cloaked.IP`: ClamAV calls it //! `Cloaked.NumericIP`. //! //! Both were found by enumerating ClamAV's emitted names rather than by reading @@ -33,7 +33,7 @@ fn detection(db: &Scanner, blob: &[u8], opts: &ScanOptions) -> Option { } } -/// A minimal valid OLE2 document carrying a VBA project — the same construction +/// A minimal valid OLE2 document carrying a VBA project, the same construction /// `heuristic_alerts.rs` uses, so the fixture is known to reach the extractor. /// A hand-rolled OLE2 with a plain-text module stream does NOT: exav's extractor /// needs a real MS-OVBA compressed container, and a fixture that silently fails @@ -72,7 +72,7 @@ fn macro_alert_carries_the_dialect_suffix() { let mut opts = ScanOptions::default(); opts.alert_macros = true; - // The fixture MUST fire, or the assertions below are vacuous — which is the + // The fixture MUST fire, or the assertions below are vacuous, which is the // failure mode this whole file exists to prevent. let name = detection(&db, &doc, &opts) .expect("fixture did not trigger the macro heuristic; the test would prove nothing"); @@ -120,14 +120,14 @@ fn cloaked_ip_uses_clamavs_numericip_name() { } /// The names exav emits that ClamAV does not. These are *additions*, not -/// mismatches — they cannot cause a missed detection, only an extra one — but +/// mismatches (they cannot cause a missed detection, only an extra one), but /// they must stay deliberate rather than drift in unnoticed. #[test] fn exav_only_names_are_the_ones_we_expect() { // `Heuristics.Encrypted.Doc` was on this list and did not belong: it is a // MISMATCH, not an addition. For an encrypted OLE2 document ClamAV emits - // `Heuristics.Encrypted.OLE2` — observed 468 times against 0 for `.Doc` - // across a corpus run — so exav's name matched no gateway filter while + // `Heuristics.Encrypted.OLE2` (observed 468 times against 0 for `.Doc` + // across a corpus run), so exav's name matched no gateway filter while // looking, on this list, like a deliberate extra. The trap is that ClamAV's // config option IS called `AlertEncryptedDoc`; the option name and the // signature name differ, and only the signature name is on the wire. @@ -154,7 +154,7 @@ fn exav_only_names_are_the_ones_we_expect() { /// member, and `--alert-encrypted` turns that into a detection. /// /// The name has to be exact: a gateway filtering on ClamAV's string matches on -/// that string and nothing else. Verified against the oracle — over one corpus +/// that string and nothing else. Verified against the oracle: over one corpus /// run clamd emitted `Heuristics.Encrypted.OLE2` 468 times and /// `Heuristics.Encrypted.Doc` never. #[test] @@ -162,7 +162,7 @@ fn an_encrypted_office_document_uses_clamavs_signature_name() { let cursor = std::io::Cursor::new(Vec::::new()); let mut cf = cfb::CompoundFile::create(cursor).expect("create OLE2"); { - // Version 4 agile header, then bytes that are not a valid descriptor — + // Version 4 agile header, then bytes that are not a valid descriptor: // enough to be recognised as MS-OFFCRYPTO, not enough to decrypt. let mut s = cf.create_stream("/EncryptionInfo").expect("EncryptionInfo"); s.write_all(&[0x04, 0x00, 0x04, 0x00]).unwrap(); @@ -191,7 +191,7 @@ fn an_encrypted_office_document_uses_clamavs_signature_name() { assert_eq!( name, "Heuristics.Encrypted.OLE2", "ClamAV emits `Heuristics.Encrypted.OLE2` for an encrypted OLE2 document. \ - Note its CONFIG OPTION is `AlertEncryptedDoc` — the option name and the \ + Note its CONFIG OPTION is `AlertEncryptedDoc`; the option name and the \ signature name differ, and only the signature name reaches a gateway." ); } @@ -237,7 +237,7 @@ fn matching_schemes_are_not_a_spoof() { // ------------------------------------------------------- Broken.Executable /// A file carrying an executable magic whose headers do not hold together. The -/// signal is the *contradiction* — which is why an arbitrary binary blob must +/// signal is the *contradiction*, which is why an arbitrary binary blob must /// never qualify. #[test] fn broken_executable_needs_a_broken_executable() { @@ -288,6 +288,83 @@ fn ordinary_files_are_not_broken_executables() { } } +/// A PE header claiming 40 sections it does not have, which passes the +/// embedded-PE carving gate. +fn broken_pe() -> Vec { + let mut out = b"MZ".to_vec(); + out.resize(0x3c, 0); + out.extend(64u32.to_le_bytes()); // e_lfanew + out.extend(b"PE\0\0"); + out.extend(0x014cu16.to_le_bytes()); // machine + out.extend(40u16.to_le_bytes()); // sections, none of which follow + out.extend([0u8; 12]); + out.extend(224u16.to_le_bytes()); // optional header size + out.extend(0x0102u16.to_le_bytes()); // characteristics + out.extend([0x0b, 0x01]); // PE32 magic + out.extend([0u8; 222]); + out +} + +/// Minimal 64-bit little-endian ELF header with e_shentsize (offset 58) zeroed. +fn stripped_elf() -> Vec { + let mut elf = vec![0u8; 64]; + elf[..4].copy_from_slice(b"\x7fELF"); + elf[4] = 2; // 64-bit + elf[5] = 1; // little-endian + elf[6] = 1; // version + elf[16..18].copy_from_slice(&2u16.to_le_bytes()); // e_type = EXEC + elf[18..20].copy_from_slice(&0x3Eu16.to_le_bytes()); // x86-64 + elf[52..54].copy_from_slice(&64u16.to_le_bytes()); // e_ehsize + elf[54..56].copy_from_slice(&56u16.to_le_bytes()); // e_phentsize + elf[58..60].copy_from_slice(&0u16.to_le_bytes()); // e_shentsize: STRIPPED + elf +} + +/// An executable carved out of another object at an offset is a candidate found +/// by its magic, not a claim the object makes about itself. One that does not +/// parse (a fragment, a truncated tail, a coincidental header) does not make +/// its host a broken executable. +/// +/// Measured against clamscan 1.5.4 `--alert-broken`: each fixture below is +/// reported on its own and passes when it follows 1 KiB of text or of zeros, or +/// is appended to a valid PE. Before this, exav flagged a clean 40 KB PE whose +/// last 1.5 KB held the start of a second, truncated one. +#[test] +fn an_embedded_executable_does_not_make_its_host_broken() { + let db = empty_db(); + let mut opts = ScanOptions::default(); + opts.alert_broken = true; + let mut compat = opts.clone(); + compat.clamav_compat = true; + let mut truncated_elf = stripped_elf(); + truncated_elf[58..60].copy_from_slice(&64u16.to_le_bytes()); + truncated_elf[40..48].copy_from_slice(&0x1000u64.to_le_bytes()); // e_shoff past EOF + truncated_elf[60..62].copy_from_slice(&4u16.to_le_bytes()); // e_shnum + for (what, exe) in [ + ("a broken PE", broken_pe()), + ("a stripped ELF", stripped_elf()), + ("a truncated ELF", truncated_elf), + ] { + assert_eq!( + detection(&db, &exe, &compat).as_deref(), + Some("Heuristics.Broken.Executable"), + "{what} alone must be reported, or this test proves nothing" + ); + for (host, lead) in [("text", b'A'), ("zeros", 0u8)] { + let mut blob = vec![lead; 1024]; + blob.extend_from_slice(&exe); + for o in [&opts, &compat] { + assert_eq!( + detection(&db, &blob, o), + None, + "{what} after 1 KiB of {host} must not be reported (compat: {})", + o.clamav_compat + ); + } + } + } +} + /// A JPEG whose APPn segments are ordered the way Office and Adobe write them /// must not be reported as broken media. /// @@ -370,8 +447,8 @@ fn a_fullword_subsignature_does_not_match_inside_a_longer_word() { /// A stripped ELF section-header table is reported in BOTH modes; only the name /// changes. /// -/// exav does not consider these broken — section headers are optional for -/// execution and the binaries run — but no toolchain zeroes the entry size, so +/// exav does not consider these broken (section headers are optional for +/// execution and the binaries run), but no toolchain zeroes the entry size, so /// the fact is worth reporting under a name that says what it is. ClamAV files it /// under `Heuristics.Broken.Executable`, and a gateway filtering on that exact /// string has to keep matching under `--clamav-compat`. @@ -379,17 +456,7 @@ fn a_fullword_subsignature_does_not_match_inside_a_longer_word() { /// The point of the pair below: compat changes VOCABULARY, never coverage. #[test] fn a_stripped_elf_section_table_is_reported_in_both_modes() { - // Minimal 64-bit little-endian ELF header with e_shentsize (offset 58) zeroed. - let mut elf = vec![0u8; 64]; - elf[..4].copy_from_slice(b"\x7fELF"); - elf[4] = 2; // 64-bit - elf[5] = 1; // little-endian - elf[6] = 1; // version - elf[16..18].copy_from_slice(&2u16.to_le_bytes()); // e_type = EXEC - elf[18..20].copy_from_slice(&0x3Eu16.to_le_bytes()); // x86-64 - elf[52..54].copy_from_slice(&64u16.to_le_bytes()); // e_ehsize - elf[54..56].copy_from_slice(&56u16.to_le_bytes()); // e_phentsize - elf[58..60].copy_from_slice(&0u16.to_le_bytes()); // e_shentsize: STRIPPED + let elf = stripped_elf(); let mut base = ScanOptions::default(); base.alert_broken = true; diff --git a/crates/exav-core/tests/suites/mod.rs b/crates/exav-core/tests/suites/mod.rs index 4f9f2780..faaa167e 100644 --- a/crates/exav-core/tests/suites/mod.rs +++ b/crates/exav-core/tests/suites/mod.rs @@ -10,6 +10,7 @@ mod allmatch_parity; mod authenticode; #[cfg(feature = "base64scan")] mod base64_scan; +mod broken_executable; #[cfg(feature = "unstable-internals")] mod bytecode_gating; // NB: gate container suites on `all-formats`, never on exav-core's own per-format diff --git a/crates/exav-core/tests/suites/one_scan.rs b/crates/exav-core/tests/suites/one_scan.rs index dcda19fd..457257d6 100644 --- a/crates/exav-core/tests/suites/one_scan.rs +++ b/crates/exav-core/tests/suites/one_scan.rs @@ -35,9 +35,9 @@ fn zip_with(member: &[u8], comment: &str) -> Vec { z.finish().unwrap().into_inner() } -/// Over `--max-input-bytes` the first bytes get the whole scan, not only the -/// literal pass: EICAR deflated inside a ZIP is not in the file's bytes, and -/// only extraction finds it. +/// Over `--max-input-bytes` the first bytes get the whole scan, extraction +/// included: EICAR deflated inside a ZIP is not in the file's bytes, and only +/// extraction finds it. #[test] fn the_start_of_an_oversize_input_gets_the_whole_scan() { // Compressible text first, so the deflater codes EICAR rather than diff --git a/crates/exav-core/tests/suites/oversize_container.rs b/crates/exav-core/tests/suites/oversize_container.rs index 534f5c3e..a70e60d6 100644 --- a/crates/exav-core/tests/suites/oversize_container.rs +++ b/crates/exav-core/tests/suites/oversize_container.rs @@ -1,20 +1,42 @@ -//! A file past `deep_analysis_max` is never clean, whichever entry point sees it. +//! A file past `deep_analysis_max` is scanned in full, or reported as over the +//! size limit, whichever entry point sees it. //! -//! Past that size the full engine does not run: the streaming core reads the -//! bytes, and it matches literal signatures and whole-file hashes only. That is -//! not a complete scan of any file, and for a container the members were never -//! opened either, so reporting `Clean` would break the never-a-silent-clean -//! invariant in the one case where the file is large enough to be worth hiding -//! something in. +//! Past that size the file is read through a block cache rather than held, +//! and everything that can run that way does. What cannot is named as the +//! size limit: a container whose format is read whole, and the parts of a +//! scan that need spill space when the host provides none. //! //! [`scan_path`] and [`scan_seekable`] reach that decision by separate routes — //! one buffers from a `File`, the other from a `Read + Seek` that may be an //! HTTP range source — and the daemon picks between them by verb. Two routes //! and one rule, so the rule needs a test on both or they drift. +use exav_core::spill::{Spill, SpillReader, SpillWriter}; use exav_core::{loader, scan_path, scan_seekable, ScanOptions, Scanner, Verdict}; use std::io::Write; +/// Spill space held in memory. +struct MemSpill; + +struct MemFile(Vec); + +impl Spill for MemSpill { + fn create(&self) -> Result, String> { + Ok(Box::new(MemFile(Vec::new()))) + } +} + +impl SpillWriter for MemFile { + fn write(&mut self, bytes: &[u8]) -> Result<(), String> { + self.0.extend_from_slice(bytes); + Ok(()) + } + + fn finish(self: Box) -> Result, String> { + Ok(Box::new(std::io::Cursor::new(self.0))) + } +} + fn scanner() -> Scanner { let mut l = loader::Builder::new(); l.add_named_bytes("t.ndb", b"Zzz.Never:0:*:deadbeefdeadbeefdead\n", true); @@ -74,9 +96,10 @@ fn scan_path_refuses_the_same_container() { } } -/// A ZIP is walked member by member at any size, but past the cap neither its -/// own bytes nor a member that large get the full engine. That is the same -/// size limit, and it is named as one. +/// A ZIP is walked member by member at any size, and past the cap its own +/// bytes get the full engine read through the cache. A member that large, or +/// text whose normalised views are that large, needs spill space: without it +/// that is the size limit, named as one, and with it the scan is complete. #[test] fn an_oversize_streamed_container_or_member_is_a_limit() { let db = scanner(); @@ -91,11 +114,35 @@ fn an_oversize_streamed_container_or_member_is_a_limit() { z.finish().unwrap().into_inner() }; let many_small: Vec<_> = (0..64) - .map(|i| (format!("m{i}.txt"), vec![b'a'; 200])) + .map(|i| (format!("m{i}.bin"), vec![0u8; 200])) .collect(); + let many_small = zip_of(&many_small); + let report = scan_seekable( + &db, + std::io::Cursor::new(many_small.clone()), + many_small.len() as u64, + &tiny_deep_analysis(), + ) + .expect("scan"); + assert_eq!( + report.verdict, + Verdict::Clean, + "every member and byte was scanned" + ); + let one_large = vec![("big.txt".to_string(), vec![b'a'; 64 * 1024])]; let flat = vec![b'a'; 64 * 1024]; - for blob in [zip_of(&many_small), zip_of(&one_large), flat] { + for blob in [zip_of(&one_large), flat] { + let mut spilling = tiny_deep_analysis(); + spilling.spill = Some(std::sync::Arc::new(MemSpill)); + let report = scan_seekable( + &db, + std::io::Cursor::new(blob.clone()), + blob.len() as u64, + &spilling, + ) + .expect("scan"); + assert_eq!(report.verdict, Verdict::Clean, "a spill completes the scan"); let size = blob.len() as u64; let report = scan_seekable( &db, @@ -123,10 +170,35 @@ fn an_oversize_streamed_container_or_member_is_a_limit() { } } +/// A DMG is walked off its source, decompressing only what the filesystem +/// reaches, so its files are scanned at any size. +#[cfg(feature = "all-formats")] +#[test] +fn an_oversize_disk_image_has_its_files_scanned() { + let mut l = loader::Builder::new(); + // "hello hfs+", the one file on the image. + l.add_named_bytes("t.ndb", b"Zzz.InDmg:0:*:68656c6c6f206866732b\n", true); + let db = l.build().expect("build database"); + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../exav-unpack/tests/fixtures/dmg/hfs_plus_udzo.dmg" + ); + let blob = std::fs::read(path).unwrap(); + assert!(blob.len() > 4096); + assert!(!blob.windows(10).any(|w| w == b"hello hfs+"), "compressed"); + let size = blob.len() as u64; + let report = + scan_seekable(&db, std::io::Cursor::new(blob), size, &tiny_deep_analysis()).expect("scan"); + match report.verdict { + Verdict::Infected { signature, .. } => assert_eq!(signature, "Zzz.InDmg"), + other => panic!("expected the file on the image to be found, got {other:?}"), + } +} + #[test] fn oversize_flat_content_is_a_limit_too() { - // Flat content past the cap got the literal-and-hash pass only, so it is - // not fully scanned either. + // Text past the cap has its normalised views made in spill space. With + // none, those views were not scanned, so the file was not fully scanned. let db = scanner(); let blob = vec![b'a'; 64 * 1024]; let size = blob.len() as u64; @@ -134,6 +206,6 @@ fn oversize_flat_content_is_a_limit_too() { scan_seekable(&db, std::io::Cursor::new(blob), size, &tiny_deep_analysis()).expect("scan"); match report.verdict { Verdict::LimitsExceeded { .. } => {} - other => panic!("flat content past the cap had a literal-only scan; got {other:?}"), + other => panic!("text past the cap had no views scanned; got {other:?}"), } } diff --git a/crates/exav-core/tests/suites/size_caps.rs b/crates/exav-core/tests/suites/size_caps.rs index 8101e759..1ac76aa3 100644 --- a/crates/exav-core/tests/suites/size_caps.rs +++ b/crates/exav-core/tests/suites/size_caps.rs @@ -183,9 +183,10 @@ fn an_iso_too_large_to_buffer_is_walked() { #[test] #[cfg(feature = "all-formats")] fn walking_an_input_too_large_to_buffer_still_reports_the_limit() { - // Its own bytes got the literal pass only, whatever the walk found in it. - // The loader's baseline EICAR signature is ignored, so the payloads in the - // fixtures do not count as a find. + // Something too large for the limit went unscanned (a member, with no + // spill; a PE's structure), whatever the walk found. The loader's baseline + // EICAR signature is ignored, so the payloads in the fixtures do not count + // as a find. let mut l = exav_core::loader::Builder::new(); l.add_named_bytes("t.ndb", b"Zzz.Never:0:*:deadbeefdeadbeefdead\n", true); l.add_named_bytes("t.ign2", b"Eicar-Test-Signature\n", true); diff --git a/crates/exav-grep/Cargo.toml b/crates/exav-grep/Cargo.toml index 902201b8..d6be1134 100644 --- a/crates/exav-grep/Cargo.toml +++ b/crates/exav-grep/Cargo.toml @@ -16,7 +16,7 @@ name = "exav-grep" path = "src/main.rs" [dependencies] -exav-unpack = { path = "../exav-unpack", version = "0.0.1", default-features = false, features = ["all-formats"] } +exav-unpack = { path = "../exav-unpack", version = "0.0.2", default-features = false, features = ["all-formats"] } regex = "1" clap = { workspace = true } walkdir = { workspace = true } diff --git a/crates/exav-grep/README.md b/crates/exav-grep/README.md index 4c354798..eab7bbe5 100644 --- a/crates/exav-grep/README.md +++ b/crates/exav-grep/README.md @@ -4,11 +4,11 @@ Ordinary `grep -r` stops at the container: a ZIP, ISO, 7z or Word document is one opaque binary blob to it. `exav-grep` walks *into* containers, recursively, and -searches every member as if it were a file on disk — reporting matches with a +searches every member as if it were a file on disk, reporting matches with a path that shows the nesting. ```sh -exav-grep 'BEGIN RSA PRIVATE KEY' backups/ +exav-grep -r 'BEGIN RSA PRIVATE KEY' backups/ backups/2024.zip!home/dev/keys.tar!id_rsa:1:-----BEGIN RSA PRIVATE KEY----- ``` @@ -27,7 +27,7 @@ searcher.search_path(std::path::Path::new("backup.zip"), &mut |ev| { ## Everything happens in memory No member is ever written to disk, so the zip-slip / path-traversal / symlink -class of bug does not arise — there is no path to traverse to. Extraction runs +class of bug does not arise: there is no path to traverse to. Extraction runs under the same decompression-bomb budgets the exav scanner uses, so a malicious archive cannot turn a search into an unbounded allocation. @@ -35,8 +35,8 @@ archive cannot turn a search into an unbounded allocation. ## Members that cannot be read are reported -A member that cannot be decoded — an unsupported codec, encryption without a -working password, a budget stop — is **not silently omitted from the results**. +A member that cannot be decoded (an unsupported codec, encryption without a +working password, a budget stop) is **not silently omitted from the results**. It is surfaced as its own event, because "no matches" and "I could not look" are different answers, and quietly conflating them is how a search tool lies to you. @@ -47,7 +47,7 @@ The CLI prints those to stderr and sets a distinct exit code: | 0 | matches found | | 1 | no matches | | 2 | a usage or I/O error | -| 3 | **no matches, but some members could not be read** — the search was incomplete | +| 3 | **no matches, but some members could not be read**: the search was incomplete | Exit code 3 is the one that matters. Anything that scripts a search for secrets or indicators across an archive tree needs to know the difference between a @@ -57,7 +57,7 @@ clean result and an incomplete one. Whatever [`exav-unpack`](https://crates.io/crates/exav-unpack) opens: ZIP, RAR, 7z, tar and every common compressor, ISO and disk images, OLE/CFB and OOXML -documents, PDF, email (MIME/TNEF), CAB, ARJ, LHA, ZOO, EGG, ALZ and more — +documents, PDF, email (MIME/TNEF), CAB, ARJ, LHA, ZOO, EGG, ALZ and more, nested arbitrarily deep, each format behind its own Cargo feature. ## License diff --git a/crates/exav-grep/src/lib.rs b/crates/exav-grep/src/lib.rs index b448fb25..b2b90428 100644 --- a/crates/exav-grep/src/lib.rs +++ b/crates/exav-grep/src/lib.rs @@ -42,7 +42,7 @@ use std::fmt; use std::path::Path; -use exav_unpack::{Budget, Entry, Limits}; +use exav_unpack::{Budget, Limits}; /// Separator between a container and a member in a reported path, mirroring the /// convention `7z`/`unzip` tooling uses so the output is greppable in turn. @@ -279,7 +279,7 @@ impl Searcher { ), }); } - let Some(fmt) = exav_unpack::detect(data).or_else(|| packed_executable(data)) else { + let Some(fmt) = exav_unpack::detect(&data).or_else(|| packed_executable(data)) else { // A leaf: search its bytes. return self.search_one(label, data, sink); }; @@ -290,37 +290,50 @@ impl Searcher { // still gets looked at rather than silently producing no output. let mut members = 0usize; let mut keep_going = true; - let r = - exav_unpack::extract_each(fmt, data, budget, &mut |entry: Entry, b: &mut Budget| { - let child = format!("{label}{NESTING_SEP}{}", entry.name); - // A member exav could not decode is content this search did not see. - // Saying nothing here would report "no matches" for bytes nobody - // looked at. - if let Some(reason) = entry.unsupported { - // Counted as a member even though it yielded nothing. The - // fallback below exists for a buffer the walker found no - // members in at all; a container whose members all failed to - // decode HAS members, and line-searching its own compressed - // bytes would emit a stream of garbage matches contradicting - // the `Unreadable` events just reported for the same file. - members += 1; - self.unreadable += 1; - if !sink(Event::Unreadable { - path: child, - reason: reason.to_string(), - }) { - keep_going = false; - return Some(()); + let r = exav_unpack::walk(fmt, &data, budget, &mut |meta, content, b| { + let child = format!("{label}{NESTING_SEP}{}", meta.name); + // Counted as a member whether or not it yields anything. The + // fallback below exists for a buffer the walker found no members in + // at all; a container whose members all failed to decode HAS + // members, and line-searching its own compressed bytes would emit a + // stream of garbage matches contradicting the `Unreadable` events + // reported for the same file. + members += 1; + // A member exav could not decode is content this search did not + // see. Saying nothing here would report "no matches" for bytes + // nobody looked at. + let (data, unread) = match meta.unsupported { + Some(reason) => (Vec::new(), Some(reason)), + None => match content.map(|c| c.into_bytes(meta, b)).transpose() { + Err(hit) => return Some(Err(hit)), + Ok(None) => (Vec::new(), None), + Ok(Some((data, partial))) => { + (data, partial.then_some("member failed to decode part way")) } - return None; - } - members += 1; - if !self.walk(&child, &entry.data, b, depth + 1, sink) { - keep_going = false; - return Some(()); - } + }, + }; + let mut go = true; + if unread.is_none() || !data.is_empty() { + go = self.walk(&child, &data, b, depth + 1, sink); + } + if let Some(reason) = unread.filter(|_| go) { + self.unreadable += 1; + go = sink(Event::Unreadable { + path: child, + reason: reason.to_string(), + }); + } + if go { None - }); + } else { + keep_going = false; + Some(Ok(())) + } + }); + let r = match r { + Ok(Some(Err(hit))) | Err(hit) => Err(hit), + Ok(_) => Ok(()), + }; // A budget stop or a decoder failure means members went unexamined. if let Err(e) = r { self.unreadable += 1; diff --git a/crates/exav-grep/src/main.rs b/crates/exav-grep/src/main.rs index fdea4896..2dacb599 100644 --- a/crates/exav-grep/src/main.rs +++ b/crates/exav-grep/src/main.rs @@ -24,7 +24,7 @@ use exav_unpack::Limits; #[derive(Parser)] #[command( name = "exav-grep", - about = "grep that searches inside archives (zip, rar, 7z, tar, iso, OLE, PDF, email — recursively)", + about = "grep that searches inside archives, recursively (zip, rar, 7z, tar, iso, OLE, PDF, email)", long_about = None, version )] diff --git a/crates/exav-pe-emu/Cargo.toml b/crates/exav-pe-emu/Cargo.toml index 387a2de3..c893dbad 100644 --- a/crates/exav-pe-emu/Cargo.toml +++ b/crates/exav-pe-emu/Cargo.toml @@ -27,7 +27,7 @@ path = "src/main.rs" # here more than anywhere else in exav: this is the one component where a # scanned file supplies control flow rather than data, so the bytes reach the # decoder directly and in volume. -exav-x86 = { path = "../exav-x86", version = "0.0.1" } +exav-x86 = { path = "../exav-x86", version = "0.0.2" } # The emulator looks pages and export traps up once per executed instruction, # tens of millions of times per packed file. `HashMap`'s DoS-resistant hasher # costs more than the lookup; `FxHashMap` hashes a `u32` key in a multiply and diff --git a/crates/exav-pe-emu/src/main.rs b/crates/exav-pe-emu/src/main.rs index 7fbb83fd..90858ade 100644 --- a/crates/exav-pe-emu/src/main.rs +++ b/crates/exav-pe-emu/src/main.rs @@ -137,7 +137,7 @@ fn main() -> ExitCode { report.ticks ), None => println!( - "{name}: nothing recovered — {} (after {} instructions, {ms} ms)", + "{name}: nothing recovered: {} (after {} instructions, {ms} ms)", report.stop, report.ticks ), } diff --git a/crates/exav-pe-emu/src/run.rs b/crates/exav-pe-emu/src/run.rs index 797b9eac..da697596 100644 --- a/crates/exav-pe-emu/src/run.rs +++ b/crates/exav-pe-emu/src/run.rs @@ -88,10 +88,11 @@ const MAX_UNIMPLEMENTED_CALLS: u32 = 64; /// run can end is a number here, and a run that hits any of them stops and /// reports rather than continuing on a guess. /// -/// An embedder should set these deliberately. [`Default`] is tuned for the scan -/// path, where the emulator is one stage among many and a stub that misbehaves -/// should cost little; a triage tool that wants to watch a single sample to its -/// conclusion will want larger budgets and `trace` on. +/// An embedder should set these deliberately. [`Default`] suits one sample on +/// its own; exav's scan path sets tighter budgets, since there the emulator is +/// one stage among many and a stub that misbehaves should cost little. A triage +/// tool that wants to watch a single sample to its conclusion may want larger +/// budgets still, and `trace` on. /// /// The bounds are independent, and a stub defeats the emulator by reaching any /// one of them. Raising a single limit rarely changes an outcome on its own. diff --git a/crates/exav-unpack-wasm/Cargo.lock b/crates/exav-unpack-wasm/Cargo.lock index dac79c79..af08e26a 100644 --- a/crates/exav-unpack-wasm/Cargo.lock +++ b/crates/exav-unpack-wasm/Cargo.lock @@ -10,12 +10,12 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aes" -version = "0.8.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ - "cfg-if", "cipher", + "cpubits", "cpufeatures", ] @@ -88,20 +88,20 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] name = "block-padding" -version = "0.3.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -116,21 +116,11 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" -[[package]] -name = "bzip2-rs" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beeb59e7e4c811ab37cc73680c798c7a5da77fc9989c62b09138e31ee740f735" -dependencies = [ - "crc32fast", - "tinyvec", -] - [[package]] name = "cbc" -version = "0.1.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" dependencies = [ "cipher", ] @@ -175,14 +165,26 @@ dependencies = [ [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "crypto-common", "inout", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -195,11 +197,17 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -230,12 +238,20 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", ] [[package]] @@ -257,22 +273,23 @@ dependencies = [ [[package]] name = "des" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" dependencies = [ "cipher", ] [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", + "const-oid", "crypto-common", - "subtle", + "ctutils", ] [[package]] @@ -283,14 +300,15 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "exav-pe-emu" -version = "0.0.1" +version = "0.0.2" dependencies = [ "exav-x86", + "rustc-hash", ] [[package]] name = "exav-unpack" -version = "0.0.1" +version = "0.0.2" dependencies = [ "aes", "apfs", @@ -298,7 +316,6 @@ dependencies = [ "bincode", "bitstream-io", "byteorder", - "bzip2-rs", "cbc", "cfb", "constant_time_eq", @@ -330,6 +347,7 @@ dependencies = [ "stringprep", "tar", "thiserror", + "tinyvec", "unshield", "xz4rust", "zip", @@ -337,7 +355,7 @@ dependencies = [ [[package]] name = "exav-unpack-wasm" -version = "0.0.1" +version = "0.0.2" dependencies = [ "exav-unpack", "flate2", @@ -349,7 +367,7 @@ dependencies = [ [[package]] name = "exav-x86" -version = "0.0.1" +version = "0.0.2" [[package]] name = "explode" @@ -437,16 +455,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -477,13 +485,22 @@ dependencies = [ [[package]] name = "hmac" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ "digest", ] +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -520,12 +537,12 @@ dependencies = [ [[package]] name = "inout" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ "block-padding", - "generic-array", + "hybrid-array", ] [[package]] @@ -589,9 +606,9 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", "digest", @@ -639,9 +656,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "pbkdf2" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ "digest", "hmac", @@ -680,6 +697,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustversion" version = "1.0.22" @@ -733,9 +756,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", "cpufeatures", @@ -744,9 +767,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures", @@ -782,12 +805,6 @@ dependencies = [ "unicode-properties", ] -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "2.0.118" @@ -909,12 +926,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -1061,6 +1072,7 @@ version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ + "bitstream-io", "crc32fast", "flate2", "indexmap", diff --git a/crates/exav-unpack-wasm/Cargo.toml b/crates/exav-unpack-wasm/Cargo.toml index 787c2623..b49aa69a 100644 --- a/crates/exav-unpack-wasm/Cargo.toml +++ b/crates/exav-unpack-wasm/Cargo.toml @@ -1,7 +1,7 @@ # Excluded from the workspace (a wasm32-target artifact), so values are explicit. [package] name = "exav-unpack-wasm" -version = "0.0.1" +version = "0.0.2" edition = "2021" license = "MIT" repository = "https://github.com/sylvinus/exav" @@ -117,7 +117,7 @@ wasm-opt = [ ] [dependencies] -exav-unpack = { path = "../exav-unpack", version = "0.0.1", default-features = false } +exav-unpack = { path = "../exav-unpack", version = "0.0.2", default-features = false } wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" js-sys = "0.3" diff --git a/crates/exav-unpack-wasm/README.md b/crates/exav-unpack-wasm/README.md index d4515247..f267f279 100644 --- a/crates/exav-unpack-wasm/README.md +++ b/crates/exav-unpack-wasm/README.md @@ -138,11 +138,13 @@ const archive = await Archive.open(file, undefined, { }); ``` -Where an archive carries an index — a ZIP's central directory, a tar's headers — -`list()` reads it and decompresses nothing, and `extract(i)` seeks straight to -the member. A single compressed stream such as gzip or xz has no index by -construction: there `list()` walks the archive, costing what `extractAll()` -costs, and the result is reused rather than walked twice. +`list()` decodes nothing it can avoid: where an archive carries an index (a +ZIP's central directory, a tar's headers) it decompresses nothing, and +`uncompressedSize` is what the archive declares, or -1 where it declares none +(a gzip or xz stream). `extract(i)` walks the archive up to member `i`, skipping +the members before it undecoded where the format allows; a format read whole +(7z, RAR) is decoded to be listed or extracted from, so for those, +`extractAll()` is one pass where `extract(i)` over every `i` is many. ### Limits diff --git a/crates/exav-unpack-wasm/js/index.d.ts b/crates/exav-unpack-wasm/js/index.d.ts index 1efd8d6b..6ae8fff7 100644 --- a/crates/exav-unpack-wasm/js/index.d.ts +++ b/crates/exav-unpack-wasm/js/index.d.ts @@ -25,6 +25,7 @@ export interface MemberInfo { name: string; index: number; compressedSize: number; + /** As the archive declares it; -1 where it declares none (a gzip or xz stream). */ uncompressedSize: number; encrypted: boolean; } @@ -79,12 +80,12 @@ export class Archive { /** The detected format's name. Known since `open`. */ format(): string; /** - * Every member's metadata. Where the archive carries an index this extracts - * nothing; an index-less single stream (gzip, xz) is walked once — taking - * `passwords`, costing what `extractAll` costs — and the walk is kept. + * Every member's metadata, decoding nothing it can avoid: where the archive + * carries an index (a ZIP's central directory, a tar's headers) nothing is + * extracted. A format read whole (7z, RAR) is decoded to be listed. */ list(passwords?: string[]): Promise; - /** Extract one member by index. */ + /** Extract one member by index. The archive is walked up to it. */ extract(index: number, passwords?: string[]): Promise; /** * Extract every member, under one budget for the whole archive. Where a limit diff --git a/crates/exav-unpack-wasm/js/index.js b/crates/exav-unpack-wasm/js/index.js index 1c8b3607..3f075233 100644 --- a/crates/exav-unpack-wasm/js/index.js +++ b/crates/exav-unpack-wasm/js/index.js @@ -1,15 +1,14 @@ // The public entry point. // -// The wasm module beneath this is synchronous throughout, because -// `exav_unpack::Archive` is `Read + Seek` and meeting it on its own terms is -// what lets a browser build share exav's archive readers instead of carrying a -// second set. Synchronous reads of a `File` need `FileReaderSync`, which exists -// only inside a Worker — so this file routes a `File` to a Worker and presents -// the async API a page actually wants. +// The wasm module beneath this is synchronous throughout, because exav's +// archive readers read by offset, synchronously, and meeting them on their own +// terms is what lets a browser build share them instead of carrying a second +// set. Synchronous reads of a `File` need `FileReaderSync`, which exists only +// inside a Worker — so this file routes a `File` to a Worker and presents the +// async API a page actually wants. // -// Bytes already in memory take no Worker at all: a `Cursor` is already -// `Read + Seek`, so that path runs in-process and the caller cannot tell the -// difference except in how fast it returns. +// Bytes already in memory take no Worker at all, so that path runs in-process +// and the caller cannot tell the difference except in how fast it returns. import init, { Archive as WasmArchive, @@ -226,13 +225,12 @@ export class Archive { } /** - * Every member's metadata. + * Every member's metadata, decoding nothing it can avoid. * - * Where the archive carries an index — a ZIP's central directory, a tar's - * headers — this decompresses nothing. A single compressed stream such as - * gzip or xz has no index by construction, so its members are only knowable - * by walking the archive; there this takes passwords and costs what - * `extractAll` costs. The result is held, so calling both is not two walks. + * Where the archive carries an index (a ZIP's central directory, a tar's + * headers) this decompresses nothing, and `uncompressedSize` is what the + * archive declares, or -1 where it declares none (a gzip or xz stream). A + * format read whole (7z, RAR) is decoded to be listed. */ async list(passwords) { this.#check(); diff --git a/crates/exav-unpack-wasm/js/worker.js b/crates/exav-unpack-wasm/js/worker.js index 7754f1fa..97b723cf 100644 --- a/crates/exav-unpack-wasm/js/worker.js +++ b/crates/exav-unpack-wasm/js/worker.js @@ -1,7 +1,8 @@ // The Worker side of the File path. // -// WHY THIS EXISTS: `exav_unpack::Archive` is `Read + Seek`, and the only -// synchronous way to read a `Blob` is `FileReaderSync`, which browsers expose +// WHY THIS EXISTS: exav's archive readers read by offset, synchronously, and +// the only synchronous way to read a `Blob` is `FileReaderSync`, which browsers +// expose // only inside a Worker. Running here is what lets a `File` be read through the // same archive code as everything else, a member at a time, without loading it // whole. The alternative was a second set of archive readers written against diff --git a/crates/exav-unpack-wasm/package.json b/crates/exav-unpack-wasm/package.json index 2f4e1e01..6aeb1594 100644 --- a/crates/exav-unpack-wasm/package.json +++ b/crates/exav-unpack-wasm/package.json @@ -1,6 +1,6 @@ { "name": "@exav/unpack-wasm", - "version": "0.0.1", + "version": "0.0.2", "type": "module", "description": "WebAssembly bindings for exav-unpack: in-browser, memory-safe universal archive extraction. Supports zip, rar, 7z, tar, gz, bz2, xz, cab, arj, lha, iso, cpio, ar, xar, ole, pdf, email, UPX.", "license": "MIT", diff --git a/crates/exav-unpack-wasm/src/lib.rs b/crates/exav-unpack-wasm/src/lib.rs index 3b18ec98..a7527fe3 100644 --- a/crates/exav-unpack-wasm/src/lib.rs +++ b/crates/exav-unpack-wasm/src/lib.rs @@ -8,14 +8,13 @@ //! extraction with no native/C dependencies. //! //! This module is SYNCHRONOUS throughout, and that is the design rather than an -//! omission. `exav_unpack::Archive` is `Read + Seek`; meeting it on its own -//! terms is what lets a browser build share exav's archive readers instead of -//! carrying a second set, and a second set is a second set of answers about the -//! same bytes. Reading a `File` synchronously needs `FileReaderSync`, which -//! exists only inside a Worker — so the `File` path runs in a Worker, and -//! `js/index.js` presents the async API a caller on the main thread uses. -//! Bytes already in memory need no Worker and take the same code through a -//! `Cursor`. +//! omission. `exav_unpack::walk` reads its source by offset, synchronously; +//! meeting it on its own terms is what lets a browser build share exav's +//! archive readers instead of carrying a second set, and a second set is a +//! second set of answers about the same bytes. Reading a `File` synchronously +//! needs `FileReaderSync`, which exists only inside a Worker, so the `File` +//! path runs in a Worker, and `js/index.js` presents the async API a caller on +//! the main thread uses. Bytes already in memory need no Worker. //! //! From JavaScript, through the facade: //! @@ -32,7 +31,10 @@ mod source; -use exav_unpack::{Budget, Entry as UnpackEntry, Format, Limits}; +use exav_unpack::source::BlockCache; +use exav_unpack::{ + walk, Budget, ByteSource, Entry as UnpackEntry, Format, LimitHit, Limits, Member, MemberMeta, +}; use js_sys::{Array, Object, Reflect, Uint8Array}; use source::Src; use wasm_bindgen::prelude::*; @@ -144,24 +146,49 @@ fn limit_marker_to_js(why: &str) -> Result { entry_to_js(&format!("<{why}>"), &[], false, why) } -fn member_to_js(m: &exav_unpack::MemberInfo) -> Result { +/// One member's metadata, as JS receives it. `size` is its decoded size where +/// that is known without decoding it, else -1. +fn member_to_js(m: &MemberMeta, index: usize, size: Option) -> Result { let obj = Object::new(); Reflect::set(&obj, &"name".into(), &JsValue::from_str(&m.name))?; - Reflect::set(&obj, &"index".into(), &JsValue::from_f64(m.index as f64))?; + Reflect::set(&obj, &"index".into(), &JsValue::from_f64(index as f64))?; Reflect::set( &obj, &"compressedSize".into(), - &JsValue::from_f64(m.compressed_size as f64), + &JsValue::from_f64(m.comp_size as f64), )?; Reflect::set( &obj, &"uncompressedSize".into(), - &JsValue::from_f64(m.uncompressed_size as f64), + &JsValue::from_f64(size.map_or(-1.0, |n| n as f64)), )?; Reflect::set(&obj, &"encrypted".into(), &JsValue::from_bool(m.encrypted))?; Ok(obj.into()) } +/// A member a walk handed over, as an entry: a streamed one is read whole +/// under the budget, and a limit is an error that stops the walk. +fn read_entry( + meta: &MemberMeta, + content: Option>, + budget: &mut Budget, +) -> Result { + let mut entry = UnpackEntry::new(meta.name.clone(), Vec::new()); + entry.comp_size = meta.comp_size; + entry.encrypted = meta.encrypted; + entry.unsupported = meta.unsupported; + if let Some(content) = content { + let (data, partial) = content.into_bytes(meta, budget)?; + entry.data = data; + if partial && entry.unsupported.is_none() { + entry.unsupported = Some( + "member failed to decode part way; the bytes before the failure are kept", + ); + } + } + Ok(entry) +} + fn parse_passwords(passwords: &JsValue) -> Vec { let Some(arr) = passwords.dyn_ref::() else { return Vec::new(); @@ -260,32 +287,39 @@ fn format_by_name(name: &str) -> Option { /// `js/index.js` is what a page talks to. #[wasm_bindgen] pub struct Archive { - inner: exav_unpack::Archive, - /// One budget for the whole archive lifetime, so `maxMembers`, - /// `maxExtractedBytes` and the scan budget bound the ARCHIVE rather than - /// each member. A fresh budget per `extract` call would reset the counters - /// and let sequential extracts exceed every cumulative limit. - budget: Budget, - /// The members of an archive that carries no index, once walked. - /// - /// `exav_unpack::Archive::list` reports an index where the format has one. - /// Where it does not, it reports an empty slice — and passing that straight - /// out would tell a page the archive holds NOTHING, which is the one answer - /// this crate must never give. So an archive without an index is walked - /// once and held: its members are not knowable any other way, since finding - /// them and extracting them are the same operation. - /// - /// Nothing here is decided per format. Whatever `exav_unpack` can index, it - /// indexes, and this never runs for it. - walked: Option, + bytes: Bytes, + fmt: Format, + limits: Limits, + /// Bytes extracted so far, across calls: `maxExtractedBytes` bounds the + /// ARCHIVE rather than each call, so sequential extracts cannot exceed it. + extracted: u64, +} + +/// Where an archive's bytes are. +enum Bytes { + Memory(Vec), + /// Read by offset as a walk needs them, through a cache of bounded size. + Reader(BlockCache), +} + +impl Bytes { + fn source(&self) -> &dyn ByteSource { + match self { + Bytes::Memory(v) => v, + Bytes::Reader(c) => c, + } + } } -/// What one walk of a directory-less archive found, and why it stopped. -struct Walk { - entries: Vec, - /// Set when a limit ended the walk early, so the caller is told the list is - /// short rather than left to read it as complete. - stopped: Option, +impl Archive { + /// The budget for one walk. Each call walks the archive afresh, so member + /// counts are per walk; extracted bytes are counted across calls. + fn budget(&self, passwords: Option) -> Budget { + let mut limits = self.limits.clone(); + limits.max_extracted_bytes = limits.max_extracted_bytes.saturating_sub(self.extracted); + let passwords = passwords.map(|p| parse_passwords(&p)).unwrap_or_default(); + Budget::with_passwords(limits, passwords) + } } #[wasm_bindgen] @@ -294,8 +328,8 @@ impl Archive { /// caller-supplied `{ read(offset, length): Uint8Array, size: number }`. /// /// A supplied `read` is SYNCHRONOUS and returns bytes rather than a promise - /// of them: the archive readers are `Read + Seek`, and there is nowhere in - /// a `read` that returns bytes to await anything. + /// of them: the archive readers read by offset, synchronously, and there is + /// nowhere in a `read` that returns bytes to await anything. /// /// `limits` is optional and bounds every extraction from this archive: /// `{ maxExtractedBytes, maxBufferBytes, maxScannedBytes, maxMembers, @@ -306,151 +340,134 @@ impl Archive { #[wasm_bindgen(js_name = "open")] pub fn open(source: JsValue, limits: Option) -> Result { let limits = limits_from_js(&limits.unwrap_or(JsValue::UNDEFINED)); - let src = if let Some(bytes) = source.dyn_ref::() { - Src::Memory(std::io::Cursor::new(bytes.to_vec())) + let reader = |src: Src| BlockCache::new(src).map(Bytes::Reader).map_err(err); + let bytes = if let Some(bytes) = source.dyn_ref::() { + Bytes::Memory(bytes.to_vec()) } else if let Some(blob) = source.dyn_ref::() { // `File` is a `Blob`, so one branch covers both. - Src::from_blob(blob.clone())? + reader(Src::from_blob(blob.clone())?)? } else if let Some(obj) = source.dyn_ref::() { - Src::from_js_reader(obj)? + reader(Src::from_js_reader(obj)?)? } else { return Err(JsValue::from_str( "expected a Uint8Array, a File/Blob inside a Worker, \ or a { read(offset, length), size } object", )); }; - - let inner = exav_unpack::Archive::open(src).map_err(err)?; - check_allowed(inner.format(), &limits)?; - let budget = Budget::new(limits); + let fmt = exav_unpack::detect(bytes.source()) + .ok_or_else(|| JsValue::from_str("unrecognised archive format"))?; + check_allowed(fmt, &limits)?; Ok(Archive { - inner, - budget, - walked: None, + bytes, + fmt, + limits, + extracted: 0, }) } - /// Whether this archive carries an index that can be read without - /// extracting anything. - /// - /// A capability, asked of the archive, rather than a list of formats kept - /// here — as `exav_unpack` learns to index another format, this starts - /// reporting it with no change on this side. - fn is_indexed(&self) -> bool { - !self.inner.list().is_empty() - } - - /// Walk a directory-less archive once, and keep what it found. - /// - /// The walk is what extraction does, so doing it twice would decompress - /// everything twice — and for a source that only reads forward, the second - /// walk would find nothing at all. - fn walk(&mut self, passwords: Vec) -> &Walk { - if self.walked.is_none() { - self.budget.passwords = passwords; - let mut entries = Vec::new(); - let mut stopped = None; - loop { - match self.inner.extract_next(&mut self.budget) { - Ok(Some(e)) => entries.push(e), - Ok(None) => break, - Err(hit) => { - stopped = Some(hit.to_string()); - break; - } - } - } - self.walked = Some(Walk { entries, stopped }); - } - self.walked.as_ref().expect("just populated") - } - /// The detected format's name. pub fn format(&self) -> String { - format_name(self.inner.format()) + format_name(self.fmt) } /// The members, as metadata. /// - /// Where the archive carries an index — a ZIP's central directory, a tar's - /// headers — this reads what `open` already parsed: no further I/O, and no - /// member decompressed. Where it does not, the members are only knowable by - /// walking, so the walk happens here and is kept. + /// A listing decodes nothing it can avoid: a member decoded as it is read + /// is left unread, and its size is what the archive declares for it. Where + /// the archive carries an index (a ZIP's central directory, a tar's + /// headers) that is all there is to it; a format read whole is decoded to + /// be listed. pub fn list(&mut self, passwords: Option) -> Result { + let mut budget = self.budget(passwords); let out = Array::new(); - if self.is_indexed() { - for m in self.inner.list() { - out.push(&member_to_js(m)?); - } - return Ok(out); - } - let pw = passwords.map(|p| parse_passwords(&p)).unwrap_or_default(); - let walk = self.walk(pw); - for (index, e) in walk.entries.iter().enumerate() { - out.push(&member_to_js(&exav_unpack::MemberInfo { - name: e.name.clone(), - index, - compressed_size: e.comp_size, - uncompressed_size: e.data.len() as u64, - encrypted: e.encrypted, - })?); + let mut index = 0; + let mut failed = None; + let walked = walk(self.fmt, self.bytes.source(), &mut budget, &mut |meta, content, _| { + let size = match &content { + Some(Member::Bytes(data)) => Some(data.len() as u64), + _ => meta.size, + }; + match member_to_js(meta, index, size) { + Ok(m) => out.push(&m), + Err(e) => { + failed = Some(e); + return Some(()); + } + }; + index += 1; + None + }); + if let Some(e) = failed { + return Err(e); } - if let Some(why) = walk.stopped.clone() { - out.push(&limit_marker_to_js(&why)?); + // A list cut short by a limit says so, rather than reading as the + // whole archive. + if let Err(hit) = walked { + out.push(&limit_marker_to_js(&hit.to_string())?); } Ok(out) } - /// Extract one member by index. + /// Extract one member by index: the archive is walked up to it. pub fn extract( &mut self, index: usize, passwords: Option, ) -> Result { - let pw = passwords.map(|p| parse_passwords(&p)).unwrap_or_default(); - if self.is_indexed() { - self.budget.passwords = pw; - let entry = self.inner.extract(index, &mut self.budget).map_err(err)?; - return unpack_entry_to_js(&entry); - } - let walk = self.walk(pw); - match walk.entries.get(index) { - Some(e) => unpack_entry_to_js(e), - None => Err(JsValue::from_str(&format!("index {index} out of bounds"))), + let mut budget = self.budget(passwords); + let mut at = 0; + let found = walk(self.fmt, self.bytes.source(), &mut budget, &mut |meta, content, b| { + if at < index { + at += 1; + return None; + } + Some(read_entry(meta, content, b)) + }); + match found { + Ok(Some(Ok(entry))) => { + self.extracted += entry.data.len() as u64; + unpack_entry_to_js(&entry) + } + Ok(Some(Err(hit))) | Err(hit) => Err(err(hit)), + Ok(None) => Err(JsValue::from_str(&format!("index {index} out of bounds"))), } } /// Extract every member. /// - /// One budget across the whole archive, so `maxMembers`, - /// `maxExtractedBytes` and the compression-ratio guard bound the ARCHIVE - /// rather than each member of it. A limit reached part-way is reported as a - /// final member saying so, because a list that simply ends reads as an - /// archive that simply ended. + /// One budget across the walk, so `maxMembers`, `maxExtractedBytes` and the + /// compression-ratio guard bound the ARCHIVE rather than each member of it. + /// A limit reached part-way is reported as a final member saying so, + /// because a list that simply ends reads as an archive that simply ended. #[wasm_bindgen(js_name = "extractAll")] pub fn extract_all(&mut self, passwords: Option) -> Result { - let pw = passwords.map(|p| parse_passwords(&p)).unwrap_or_default(); + let mut budget = self.budget(passwords); let out = Array::new(); - if !self.is_indexed() { - let walk = self.walk(pw); - for e in &walk.entries { - out.push(&unpack_entry_to_js(e)?); - } - if let Some(why) = walk.stopped.clone() { - out.push(&limit_marker_to_js(&why)?); + let mut taken = 0; + let mut failed = None; + let walked = walk(self.fmt, self.bytes.source(), &mut budget, &mut |meta, content, b| { + let entry = match read_entry(meta, content, b) { + Ok(entry) => entry, + Err(hit) => return Some(Err(hit)), + }; + taken += entry.data.len() as u64; + match unpack_entry_to_js(&entry) { + Ok(e) => { + out.push(&e); + None + } + Err(e) => { + failed = Some(e); + Some(Ok(())) + } } - return Ok(out); + }); + self.extracted += taken; + if let Some(e) = failed { + return Err(e); } - self.budget.passwords = pw; - loop { - match self.inner.extract_next(&mut self.budget) { - Ok(Some(e)) => out.push(&unpack_entry_to_js(&e)?), - Ok(None) => break, - Err(hit) => { - out.push(&limit_marker_to_js(&hit.to_string())?); - break; - } - }; + if let Ok(Some(Err(hit))) | Err(hit) = walked { + out.push(&limit_marker_to_js(&hit.to_string())?); } Ok(out) } @@ -463,7 +480,7 @@ impl Archive { /// Detect the archive/container format from magic bytes. #[wasm_bindgen(js_name = "detectFormat")] pub fn detect_format(data: &[u8]) -> Option { - exav_unpack::detect(data).map(format_name) + exav_unpack::detect(&data).map(format_name) } /// Extract every member from bytes already in memory. diff --git a/crates/exav-unpack-wasm/src/source.rs b/crates/exav-unpack-wasm/src/source.rs index e54a5439..f0717b37 100644 --- a/crates/exav-unpack-wasm/src/source.rs +++ b/crates/exav-unpack-wasm/src/source.rs @@ -1,15 +1,15 @@ -//! The byte sources `exav_unpack::Archive` reads through. +//! The sources an archive not held in memory is read through. //! -//! `Archive` is `Read + Seek`, which is to say synchronous, and that is not an -//! obstacle to be worked around — it is the reason this crate can share ONE set +//! The archive readers read by offset, synchronously, and that is not an +//! obstacle to be worked around: it is the reason this crate can share ONE set //! of archive readers with the rest of exav instead of carrying a second that -//! drifts. What it needs is a synchronous way to read a `Blob`, and browsers +//! drifts. What they need is a synchronous way to read a `Blob`, and browsers //! have exactly one: [`web_sys::FileReaderSync`], which exists only inside a //! Worker. So the `File` path runs in a Worker, and `js/index.js` turns that //! into the async API a caller on the main thread sees. //! -//! Bytes already in memory need none of this: a `Cursor` is `Read + Seek` -//! already, so that path stays on the main thread and costs no Worker at all. +//! Bytes already in memory need none of this, so that path stays on the main +//! thread and costs no Worker at all. use std::io::{self, Read, Seek, SeekFrom}; @@ -127,8 +127,8 @@ impl Fetch for BlobFetch { /// A caller-supplied `{ read(offset, length): Uint8Array, size: number }`. /// -/// `read` is SYNCHRONOUS. An async one cannot be called from here at all — the -/// archive readers are `Read + Seek`, and there is no way to await inside a +/// `read` is SYNCHRONOUS. An async one cannot be called from here at all: the +/// archive readers read synchronously, and there is no way to await inside a /// `read` that returns bytes. Supplying a source is still the escape hatch it /// always was; what it hands back is bytes rather than a promise of them. pub(crate) struct JsFetch { @@ -160,11 +160,9 @@ impl Fetch for JsFetch { /// Whichever source this archive was opened from. /// -/// One concrete type rather than a generic parameter: `exav_unpack::Archive` -/// would otherwise be a different type per source, and the exported `Archive` -/// can only be one of them. +/// One concrete type rather than a generic parameter: the exported `Archive` +/// holds one, and can only be one type. pub(crate) enum Src { - Memory(io::Cursor>), Blob(Box>), Js(Box>), } @@ -211,7 +209,6 @@ impl Src { impl Read for Src { fn read(&mut self, out: &mut [u8]) -> io::Result { match self { - Src::Memory(c) => c.read(out), Src::Blob(b) => b.read(out), Src::Js(j) => j.read(out), } @@ -221,7 +218,6 @@ impl Read for Src { impl Seek for Src { fn seek(&mut self, to: SeekFrom) -> io::Result { match self { - Src::Memory(c) => c.seek(to), Src::Blob(b) => b.seek(to), Src::Js(j) => j.seek(to), } diff --git a/crates/exav-unpack/Cargo.toml b/crates/exav-unpack/Cargo.toml index 3ecb7b59..8edcdc19 100644 --- a/crates/exav-unpack/Cargo.toml +++ b/crates/exav-unpack/Cargo.toml @@ -13,14 +13,14 @@ homepage.workspace = true keywords = ["archive", "extract", "unzip", "decompression", "zip"] categories = ["filesystem", "compression", "parser-implementations"] readme = "README.md" -# The test corpus is 9.4 MB of crafted archives — the bulk of this crate on +# The test corpus is 9.4 MB of crafted archives: the bulk of this crate on # disk, and of no use to anything that depends on it. Every `cargo install exav` # would otherwise download it. Clone the repository to run the suites. exclude = ["tests/fixtures/**", "test_data/**"] # The extractor CLI ships with the library rather than as a separate package. # It is a thin shell over the public API and needs nothing the library does not -# already have — no argument parser, no error crate — so a library consumer pays +# already have (no argument parser, no error crate), so a library consumer pays # nothing for it (Cargo builds only the lib target for a dependency), and # `cargo install exav-unpack` gives the command, which is where people look for # it. @@ -37,23 +37,27 @@ memchr = "2" # The x86 emulator that runs a packer's stub (`exav-pe-emu`). Its own crate because # it is useful on its own and because it is the only thing that needs a CPU # decoder: keeping it out of here means a build without `pepack` never compiles -# `iced-x86` at all. -exav-pe-emu = { version = "0.0.1", path = "../exav-pe-emu", optional = true } +# `exav-x86` at all. +exav-pe-emu = { version = "0.0.2", path = "../exav-pe-emu", optional = true } # --- Per-format optional deps. Each is pulled in only by the feature(s) below # that need it; a dep shared by several formats (e.g. `flate2`) is compiled if # ANY enabling feature is active. See `[features]`. --- -zip = { version = "8", default-features = false, features = ["deflate-flate2"], optional = true } +# `legacy-zip`: PKZIP 1.x Shrink, Reduce and Implode (methods 1-6), no +# `unsafe`. These decode a member whole; `formats/zip.rs` checks its sizes +# against the buffer limit first. +zip = { version = "8", default-features = false, features = ["deflate-flate2", "legacy-zip"], optional = true } # `default-features = false` drops `tar`'s `xattr` feature (extended-attribute -# *restore* on disk extraction) — we extract to memory and never restore xattrs, +# *restore* on disk extraction): we extract to memory and never restore xattrs, # so it's unused. This removes the `rustix` + `linux-raw-sys` syscall crates # entirely (~8,600 `unsafe` ops, the single largest source in the dep tree). tar = { version = "0.4", default-features = false, optional = true } -bzip2-rs = { version = "0.1.2", optional = true } +# For the bzip2 decoder vendored from `bzip2-rs` in `formats/bzip2_rs`. +tinyvec = { version = "1.11", optional = true } byteorder = { version = "1", optional = true } lzxd = { version = "0.2", optional = true } -# FAT filesystem reader. Zero `unsafe`; `chrono` (timestamps) is left off, so the -# only transitive deps are `bitflags` and `log`. +# FAT filesystem reader (a few `unsafe` uses of its own); `chrono` (timestamps) +# is left off, so the only transitive deps are `bitflags` and `log`. fatfs = { version = "0.3", default-features = false, features = ["std", "alloc"], optional = true } # LZNT1, the compression NTFS applies per compression unit. Single-purpose and # dependency-free; the MFT walk itself is in-tree. @@ -73,13 +77,13 @@ delharc = { version = "0.6.2", optional = true } bitstream-io = { version = "4.10.0", optional = true } # Pure-Rust crypto for decrypting password-known encrypted archive members # (ZIP ZipCrypto + WinZip AES). DECRYPTION needs only a cipher + a KDF, no RNG, -# so none of these pull `getrandom` (verified) — the hard no-getrandom default +# so none of these pull `getrandom` (verified) and the hard no-getrandom default # build is preserved. We do NOT enable the `zip` crate's `aes-crypto`/encryption # features, which would pull `getrandom` for salt/IV generation we never do. -aes = { version = "0.8", optional = true } -hmac = { version = "0.12", optional = true } -sha1 = { version = "0.10", optional = true } -pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"], optional = true } +aes = { version = "0.9", optional = true } +hmac = { version = "0.13", optional = true } +sha1 = { version = "0.11", optional = true } +pbkdf2 = { version = "0.13", default-features = false, features = ["hmac"], optional = true } constant_time_eq = { version = "0.3", optional = true } # DMG support hfsplus = { version = "0.2", optional = true } @@ -87,8 +91,8 @@ apfs = { version = "0.2", optional = true } lzfse_rust = { version = "0.2.1", optional = true } base64 = { version = "0.22", optional = true } quick-xml = { version = "0.41", optional = true } -des = { version = "0.8", default-features = false, optional = true } -cbc = { version = "0.1", optional = true } +des = { version = "0.9", default-features = false, optional = true } +cbc = { version = "0.2", optional = true } bincode = { version = "1", optional = true } serde = { version = "1.0.228", features = ["derive"], optional = true } flate2 = { version = "1.1.9", optional = true } @@ -100,9 +104,9 @@ flate2 = { version = "1.1.9", optional = true } deflate64 = { version = "0.1.12", optional = true } lzma-rust2 = { version = "0.16.4", default-features = false, features = ["std", "lzip"], optional = true } # PDF crypto primitives (replacing hand-rolled implementations) -md-5 = { version = "0.10", optional = true } -sha2 = { version = "0.10", optional = true } -digest = { version = "0.10", optional = true } +md-5 = { version = "0.11", optional = true } +sha2 = { version = "0.11", optional = true } +digest = { version = "0.11", optional = true } stringprep = { version = "0.1", optional = true } xz4rust = { version = "0.2.1", default-features = false, features = ["std", "bcj", "crc64", "delta", "no_unsafe"], optional = true } ruzstd = { version = "0.8", optional = true } @@ -111,13 +115,17 @@ crc32fast = { version = "1.5.0", optional = true } [features] default = ["all-formats", "decrypt"] # Makes a reserved byte prefix panic from inside the format dispatch, so the -# `catch_unwind` boundary in `extract_each` can be tested for what it is: the +# `catch_unwind` boundary in `walk` can be tested for what it is: the # thing that turns an arbitrary decoder panic into a reported limit instead of a # dead process. Never enabled by a shipped build; the test suite turns it on. testing-faults = [] +# `source::CHUNK` at 4 KiB instead of 1 MiB, so a test's small input crosses +# many chunk seams. Never enabled by a shipped build; exav-core's tests turn it +# on. +small-chunks = [] # Every container/format exav can extract. Disable this (with # `default-features = false`) and enable only the formats you need for a smaller -# build — e.g. `features = ["zip"]` for a ZIP-only extractor. A format exav +# build, e.g. `features = ["zip"]` for a ZIP-only extractor. A format exav # recognises but wasn't compiled to unpack is reported as unsupported (never # silently treated as clean), same as any undecodable member. all-formats = ["all-formats-no-emu", "pe-emu"] @@ -139,13 +147,16 @@ all-formats-no-emu = [ zip = ["dep:zip", "dep:flate2", "dep:deflate64"] gzip = ["dep:flate2"] tar = ["dep:tar"] -bzip2 = ["dep:bzip2-rs"] +bzip2 = ["bzip2-decoder"] +# Internal: the vendored bzip2 decoder, shared by every format that carries +# bzip2. Not a format of its own; select `bzip2` instead. +bzip2-decoder = ["dep:tinyvec", "dep:crc32fast"] xz = ["dep:xz4rust"] zstd = ["dep:ruzstd"] lzip = ["dep:lzma-rust2"] cab = ["dep:lzxd", "dep:flate2", "dep:byteorder"] chm = ["dep:lzxd"] -sevenz = ["dep:lzma-rust2", "dep:bzip2-rs", "dep:flate2", "dep:crc32fast"] +sevenz = ["dep:lzma-rust2", "bzip2-decoder", "dep:flate2", "dep:crc32fast"] rar = [] arj = ["dep:delharc", "dep:bitstream-io", "dep:crc32fast"] lha = ["dep:delharc"] @@ -165,14 +176,14 @@ binhex = [] lnk = [] partition = [] pyc = [] -nsis = ["dep:flate2", "dep:lzma-rust2", "dep:bzip2-rs"] +nsis = ["dep:flate2", "dep:lzma-rust2", "bzip2-decoder"] autoit = [] onenote = [] rtf = [] machofat = [] sfx = [] # PE runtime packers: detection plus the aPLib-family static unpack -# (Petite/FSG2/NsPack). Pure Rust, parses only — nothing is executed. +# (Petite/FSG2/NsPack). Pure Rust, parses only: nothing is executed. pepack = [] # Emulation-based unpacking: run the packer's own stub under `exav-pe-emu` and # capture the image it rebuilds. Its own switch, because it is the one thing in @@ -199,16 +210,15 @@ arc = [] ace = [] stuffit = [] # ESTsoft/Hancom Korean formats, all decoded. -alz = ["dep:flate2", "dep:bzip2-rs"] -egg = ["dep:flate2", "dep:bzip2-rs", "dep:lzma-rust2"] +alz = ["dep:flate2", "bzip2-decoder"] +egg = ["dep:flate2", "bzip2-decoder", "dep:lzma-rust2"] hwp3 = ["dep:flate2"] -# FAT filesystem reader. Zero `unsafe`; `chrono` (timestamps) is left off, so the -# only transitive deps are `bitflags` and `log`. -fat = ["dep:fatfs"] +# FAT filesystem reader. +fat =["dep:fatfs"] # ext2/3/4, walked as a filesystem for the same reason as FAT: a fragmented file # comes back whole from the inode's block map and in pieces from a carve. ext = ["dep:ext4-view"] -# ZOO carries LZD (LZW) and LZH — the latter is `lh5` on the wire, so `delharc` +# ZOO carries LZD (LZW) and LZH. The latter is `lh5` on the wire, so `delharc` # already covers it. zoo = ["dep:salzweg", "dep:delharc"] # InstallShield `.z`. NOT the `ISc(` InstallScript cabinet, whose only @@ -228,7 +238,7 @@ email = ["dep:mail-parser", "dep:base64"] # Apple disk images (UDIF + HFS+/APFS). The encrypted-DMG cipher stack is under # the `decrypt` feature, not here. dmg = [ - "dep:hfsplus", "dep:apfs", "dep:lzfse_rust", "dep:xz4rust", "dep:bzip2-rs", + "dep:hfsplus", "dep:apfs", "dep:lzfse_rust", "dep:xz4rust", "bzip2-decoder", "dep:flate2", "dep:base64", "dep:quick-xml", "dep:bincode", "dep:serde", ] @@ -236,7 +246,7 @@ dmg = [ # security handler, encrypted DMG). ON by default. This is the ONLY thing that # pulls the cipher/KDF crates (all decryption-only: no RNG, no getrandom). Drop # it (with the format features you want, e.g. `--no-default-features --features -# zip`) for a build with no crypto stack — encrypted members are then reported +# zip`) for a build with no crypto stack; encrypted members are then reported # `PASSWORD-PROTECTED`/unsupported (never decrypted, never silently clean), the # same as when no password is supplied. Effective only combined with a format # that has encryptable members (zip/pdf/dmg). @@ -251,7 +261,7 @@ rar5-crc-check = [] # Enforce container checksums (CRCs) during extraction. OFF by default: a malware # scanner scans decompressed content regardless of integrity metadata (a wrong -# CRC must never stop a member's bytes from being scanned — that would let an +# CRC must never stop a member's bytes from being scanned, as that would let an # attacker downgrade a detection by corrupting a checksum; this matches ClamAV). # With this feature compiled in, `Budget::set_verify_checksums(true)` re-enables # strict verification (e.g. for the extract-for-real CLI, where a bad CRC is a @@ -285,7 +295,11 @@ required-features = ["zip"] [dev-dependencies] # Used only in tests to compare decoder output against external-oracle digests. -sha2 = "0.10" +sha2 = "0.11" +# Test helpers build ZIP fixtures and gunzip stored disk images; as optional +# normal dependencies they would be missing from a single-format test build. +flate2 = "1.1.9" +zip = { version = "8", default-features = false, features = ["deflate-flate2"] } # Reference PPMd7 (var.H) encoder, used only to validate the vendored PPMd7 # model in `formats/ppmd7` against a known-good 7z-coder stream (CC0-1.0/MIT-0). ppmd-rust = "1" @@ -293,6 +307,9 @@ ppmd-rust = "1" sevenz-rust2 = { version = "0.20.2", default-features = false, features = ["bzip2", "ppmd", "util", "compress"] } # Used only in tests to build CAB fixtures for roundtrip testing. cab = "0.6.0" +# Reference bzip2 encoder (a port of libbzip2), used only in tests to build +# streams the vendored decoder must read. +bzip2 = "0.6" # Used only in tests to prove the strict CFB reader rejects a malformed # compound file that exav's lenient OLE fallback still recovers. cfb = "0.14.0" diff --git a/crates/exav-unpack/MEMORY.md b/crates/exav-unpack/MEMORY.md index 88a915bd..34e456f7 100644 --- a/crates/exav-unpack/MEMORY.md +++ b/crates/exav-unpack/MEMORY.md @@ -12,7 +12,7 @@ never held in RAM. **1. `--max-object-bytes`: the largest *single* buffer.** - API: `unpack::Limits::max_buffer_bytes` and core `ScanOptions::deep_analysis_max`, both **256 MiB** by default. - `--max-object-bytes` sets both. + `--max-object-bytes` sets both, and a scan holds the first to the second. - Every forced-materialization site caps its largest allocation here. It is **not** a cap on peak memory. Several buffers are alive at once (a @@ -32,82 +32,81 @@ never held in RAM. **2. `--max-matcher-bytes`: scan reach (CPU/time).** - API: `Limits::max_scanned_bytes`, default **10 GiB**. - Bounds the cumulative bytes fed to the matcher for one top-level input - (streamed members plus re-carved/re-scanned regions). Not a memory cost: a - streamed member is decoded on demand (`stream_members` + `BudgetReader`), and - `exav-core` holds at most `deep_analysis_max` of it. Raising this knob buys - scan time only; its job is DoS resistance. + (members plus re-carved/re-scanned regions). Not a memory cost: a member + decoded as it is read is bounded by a `BudgetReader`, and `exav-core` holds + at most `deep_analysis_max` of it. Raising this knob buys scan time only; + its job is DoS resistance. ## How an input reaches the extractors Every entry point runs `exav-core::scan_seekable` (`scan_path` opens the file and -hands it over). - -- **Typing.** From the first 4 KiB. An input over `deep_analysis_max` whose - type is not walkable off the reader is typed again from a wider head, because - an ISO 9660 descriptor sits at 32 KiB and a UDF one up to 64 KiB in; an - executable is also searched for an appended archive in its first - `max_buffer_bytes`, read a window at a time (`unpack::is_sfx`). -- **Walkable containers** (`exav-core::streams_natively`) are walked member by - member off the reader, at any size. Up to `deep_analysis_max` the container is - also read whole, once, for the full engine and the whole-object checks. Past - it the constant-memory `stream_core` (literal signatures and whole-file - hashes) runs over its bytes, the members are walked, and the result is - `LIMITS-EXCEEDED` unless something is found. -- **Everything else** up to `deep_analysis_max` is read whole and analysed. - Past it, `stream_core` runs over it and the result is `LIMITS-EXCEEDED` unless - something is found. - -## Streaming coverage - -`stream_members` walks every `is_streamable` format with each member produced -on demand, never materialized whole: gzip, bzip2, xz, zstd, lzip, tar, zip, -lha, ar, cpio, machofat, pyc, sfx, tnef, partition, iso, onenote, swf, szdd, 7z -and cab. - -- **Single-stream compressors**: a `Read` decoder over the source. bzip2 and xz - buffer their compressed input (bounded by `max_buffer_bytes`); the win is on - the output side. +hands it over), and every object, the input and whatever is unpacked from it, +goes through the same pipeline (`scan_object`). + +- **Up to `deep_analysis_max`** the input is read whole and analysed in memory. +- **Past it** the input is read through a `source::BlockCache` (64 KiB blocks, + 8 MiB held) and gets the same scan: every check runs over a + `source::ByteSource`, reading what it needs of the object. The cache bounds + memory, not reach: a block evicted is read again. Typing is exact (`detect` + reads the start, the end, or searches the object). +- **Containers** are walked by `walk` over the same source, at every depth. +- **What cannot run over a source** needs the object in memory and is skipped + past `deep_analysis_max`, which makes the result `LIMITS-EXCEEDED` naming the + size limit unless something is found: goblin-parsed PE structure (layout, + icons, section hashes, imphash, Authenticode, UPX and packer detection), + YARA's `pe`/`elf`/`dotnet` modules, the structural heuristics that parse a + whole file, and every container format read whole (`read_whole`). +- **What a scan makes** of an object that large, its normalised text views, + goes to spill files when the host provides them (`ScanOptions::spill`; the + `exav` binary passes its `--spill-dir` ones). Without them those views are + skipped and the result is `LIMITS-EXCEEDED`. + +## The walk + +`walk` is the one way in. Each member reaches the visitor as a `Member`: +`Bytes` for one decoded whole, `Stream` for one decoded as it is read. + +- **Single-stream compressors** (gzip, bzip2, xz, zstd, lzip, lzw, lz4): a + `Read` decoder straight off the source, no buffering on either side. - **Stored-offset containers** parse a small header or table through `Read + Seek`, then seek and `take` each member (`stream_stored`): ar, cpio, partition, machofat, tnef, onenote, sfx, pyc, and the ISO 9660 trees of iso. - Where a test holds a streamed walk to the buffered `extract`, it is - `assert_stream_matches_buffered` in `stream.rs`, or the `udf` suite for ISO. + A part of a table left unwalked (too many partitions or ISO directories, a + truncated table) is a member of its own, reported unsupported. - **ISO** walks its ISO 9660 trees and then its UDF tree, skipping files the - first already emitted, as the buffered walk does. The UDF walk reads the image - through the same code in both cases (`udf::Image`) and streams each file from - the runs of the image it occupies. -- **SFX** finds the appended archive in the first `max_buffer_bytes` a window at - a time and streams `[offset, EOF)` as one member. + first already emitted. The UDF walk streams each file from the runs of the + image it occupies. +- **SFX** finds the appended archive as far into the file as detection looks, a + window at a time, and streams `[offset, EOF)` as one member. - **swf**: CWS/ZWS bodies decode through a zlib/LZMA `Read` behind the rebuilt - FWS header. **szdd**: SZDD's LZSS as a streaming `Read`; KWAJ falls back to a - bounded buffered decode. + FWS header. **szdd**: SZDD's LZSS as a streaming `Read`; KWAJ is decoded whole. - **Solid blocks (7z, cab)**: the block or folder is a forward-only `Read`; each file is a `take(size)` window after skipping to its offset, so the - decompressed solid unit is never buffered. 7z still buffers its compressed + decompressed solid unit is never buffered. 7z still holds its compressed input, and AES members take the buffered CRC/password path. +- **DMG**: the UDIF disk is a `Read + Seek` over its decoded runs, and its + HFS+/APFS files are read from it. +- **Everything else** is read whole (`read_whole`: zero-copy when the source is + in memory, bounded by `max_buffer_bytes`), and its members come out as + `Bytes`. -**At every depth.** `deep_analyze` walks any `is_streamable` format through -`scan_streamed_container` over a `Cursor` of the member, so a small nested -member decompressing to gigabytes is scanned in full (memory bounded by -`deep_analysis_max`). Formats with no streaming walk stay on the buffered -`extract_each` path. +Every member is charged to the scan budget; a streamed one is also held to the +compression-ratio guard. A read that failed on the source ends the walk as an +error, never as the end of the container. -**Members.** One that fits in `deep_analysis_max` is held and fully analysed. -A larger one is not materialized: its buffered prefix is chained with the -still-streaming tail through `stream_core`, and the member is reported -`LIMITS-EXCEEDED` unless that finds something. +**Members, in exav-core.** One that fits in `deep_analysis_max` is held and +fully analysed. A larger one is written to a spill file and scanned from there +like any object too large to hold. With no spill it is not scanned, and the +scan is reported `LIMITS-EXCEEDED`. -**Panic containment**: `stream_members` runs its dispatch inside -`catch_unwind`, like `extract_each`, so a decoder panic becomes `Unscannable`. - -Every format not in `is_streamable` is buffered: the container is held whole, -bounded by `max_buffer_bytes`. +**Panic containment**: `walk` runs its dispatch inside `catch_unwind`, so a +decoder panic becomes `Unscannable`. ## Why some buffering is unavoidable A site must hold a whole object when: - a decoder or parser needs **random access** over the decoded bytes (a - ZIP/7z central directory or TOC, a PDF xref, the filesystem inside a DMG); + ZIP/7z central directory or TOC, a PDF xref); - **decryption** needs the full ciphertext (ZipCrypto, WinZip-AES, an encrypted 7z header); - an **LZ sliding window** is inherent to the codec (RAR3/RAR5, LZX); @@ -122,9 +121,10 @@ it. ### Bounded by the knob (`max_buffer_bytes` / `deep_analysis_max`) -- **exav-core buffers**: the whole input, a container read for the full engine, - and a member's structural prefix are each read up to `deep_analysis_max`; one - larger goes through `stream_core` instead. +- **exav-core buffers**: the whole input and a member are each held up to + `deep_analysis_max`; one larger is read through a block cache (the input, a + spilled member). A check that needs a larger object whole reads it only up + to the same limit. - **Streaming member API**: `BudgetReader` caps each member and errors (never truncates) past it. - **Per-format member buffers**: every `Entry.data` and decoded blob goes @@ -133,15 +133,14 @@ it. max_buffer_bytes)`. The UDF walk checks a file's size before copying it out, so unwritten extents (zeroes) cost nothing until then. - **Decoders that thread `max_buffer`**: CAB (`Cabinet::new`, each folder and the - combined buffer), 7z solid blocks, PPMd input and output, BCJ filters, UDIF - (`decompress_udif`), DMG HFS+/APFS files (`reserve`/`commit` before - `Entry::new`), encrypted ZIP members + combined buffer), 7z solid blocks, PPMd input and output, BCJ filters, DMG + HFS+/APFS files (read into a writer capped at `reserve()`), an encrypted + DMG (decrypted whole), encrypted ZIP members (`read_encrypted_member`), ARJ (the whole archive on entry), VBA (`decompress`, `build_artifacts`), the `ar` name table, the RAR3 window (rejected past the limit), ISO and UDF directory reads. - **Bounded by the default limit, not the knob**: the 7z encoded header (a - header carries no `Budget`), the DMG decrypt block size, and - `Archive::open`'s buffered fallback, a public API that predates the budget. + header carries no `Budget`) and the DMG decrypt block size. ### Input-bounded @@ -149,14 +148,15 @@ These copy a slice of their input, which the caller already capped at `max_buffer_bytes`, so they cannot exceed it: - PDF stream bodies and filter working copies (the decoded output is bounded like any member); -- the RAR3/RAR5 compressed-member copy and its padding; -- `cab.rs` `repair_cab_size`, a clone of the input CAB to patch 4 bytes. +- the RAR3/RAR5 compressed-member copy and its padding. ### Fixed literals, at or below the knob's default Routing these through the knob would only let an operator lower them: - `rar5_unpack.rs` `MAX_WINDOW_SIZE` (64 MiB), the `xar.rs` TOC (64 MiB), - `udif.rs` runs and xz dictionary (64 MiB); + `udif.rs` runs and xz dictionary (64 MiB), the decoded runs `DmgReader` + keeps (32 MiB), the LZ4 streaming reader's block (8 MiB decoded, 16 MiB + read); - `PREALLOC_CAP` (16 MiB): a pre-allocation clamp only, growth is capped elsewhere; - core's JavaScript normaliser output (32 MiB). The raw bytes are scanned in @@ -172,6 +172,6 @@ Routing these through the knob would only let an operator lower them: scan input; hashed and deserialized whole, bounded by `CvdLimits` when unpacked. - Fixed small protocol and detection buffers: the 64 KiB HTTP range block - (`source.rs`), the 64 KiB head `Archive::open` detects from, the typing heads. + (`source.rs`), the typing heads. - Format constants: the deflate 32 KiB and Deflate64 64 KiB dictionaries, the 4096-entry LZW table, 255-byte names. diff --git a/crates/exav-unpack/README.md b/crates/exav-unpack/README.md index e2d405d7..df2a8c8c 100644 --- a/crates/exav-unpack/README.md +++ b/crates/exav-unpack/README.md @@ -1,24 +1,28 @@ # exav-unpack -**Bounded, memory-safe, pure-Rust archive & container extraction** — the +**Bounded, memory-safe, pure-Rust archive & container extraction**: the extraction layer of [**exav**](https://github.com/sylvinus/exav), usable on its own. The entire crate is `#![forbid(unsafe_code)]`. It extracts a wide range of containers to memory under a shared [decompression-bomb `Budget`](https://docs.rs/exav-unpack) (output-byte, ratio, file-count, recursion-depth, and cumulative-scan-byte -caps), with per-member panic containment so hostile input can never crash the -process — an unsupported codec or encrypted member is reported as unsupported, -never silently dropped. +caps), with panic containment per container walk, so a decoder panic becomes an +error for that container instead of a crash. An unsupported codec or encrypted +member is reported as unsupported, never silently dropped. -**Formats:** zip, gzip, tar, xz, bzip2, zstd, lzip, cab, 7z, RAR (RAR3 LZ+PPMd, -RAR5 LZ), ARJ, LHA, ISO, ar, cpio, xar, OLE2, PDF, MIME email, Apple DMG -(UDIF + HFS+/APFS), and UPX. +**Formats:** ZIP (every method in common use, including Deflate64, LZMA, bzip2, +zstd, XZ, PPMd and the PKZIP 1.x ones), gzip, tar, xz, bzip2, zstd, lzip, LZ4, +Unix `compress`, CAB, 7z, RAR (RAR3 LZ+PPMd, RAR5 LZ), ARJ, LHA, ZOO, ARC, EGG, +ALZ, ISO 9660 and UDF, ar, cpio, xar, WIM, OLE2, PDF, MIME email and TNEF, Apple +DMG (UDIF + HFS+/APFS), virtual disks (VHD, VHDX, QCOW2, VMDK) with the NTFS and +FAT filesystems inside them, UPX and other PE packers, and more. The full list +is at [exav.org](https://exav.org/reference/formats/). Every format is a **Cargo feature**, so you can build only what you need (`--no-default-features --features zip` for a ZIP-only extractor). Decryption -(`decrypt`, on by default) and checksum verification (`checksums`, off by -default) are also features. +(`decrypt`, on by default) and making a checksum mismatch an error +(`checksums`, off by default) are also features. ```rust use exav_unpack::{extract, detect, Budget, Limits}; @@ -32,6 +36,10 @@ if let Some(fmt) = detect(&data) { } ``` +`extract` holds every member; `walk` hands them over one at a time, a large one +as a reader, and reads its source through a `ByteSource` (a slice, or a +`source::BlockCache` over a file). + Licensed under MIT. Full documentation: [exav.org](https://exav.org/subprojects/exav-unpack/). See [`NOTICE`](https://github.com/sylvinus/exav/blob/main/NOTICE) for third-party attributions. diff --git a/crates/exav-unpack/examples/dump_members.rs b/crates/exav-unpack/examples/dump_members.rs index dcae24f1..c83b0958 100644 --- a/crates/exav-unpack/examples/dump_members.rs +++ b/crates/exav-unpack/examples/dump_members.rs @@ -11,7 +11,7 @@ //! //! Usage: `cargo run --example dump_members --all-features -- ` -use exav_unpack::{detect, extract_each, Budget, Entry, Limits}; +use exav_unpack::{detect, Budget, Limits, Member}; use std::path::PathBuf; fn main() { @@ -35,19 +35,28 @@ fn walk(data: &[u8], outdir: &PathBuf, depth: u32, n: &mut usize, prefix: &str) if depth > 8 { return; } - let Some(fmt) = detect(data) else { return }; + let Some(fmt) = detect(&data) else { return }; let mut budget = Budget::new(Limits::default()); - let _ = extract_each::<()>(fmt, data, &mut budget, &mut |e: Entry, _b: &mut Budget| { - if e.data.is_empty() { + let _ = exav_unpack::walk::<()>(fmt, &data, &mut budget, &mut |meta, content, _| { + let bytes = match content { + None => return None, + Some(Member::Bytes(bytes)) => bytes, + Some(Member::Stream(reader)) => { + let mut bytes = Vec::new(); + let _ = reader.read_to_end(&mut bytes); + bytes + } + }; + if bytes.is_empty() { return None; } // A member byte-identical to its container is the fixed point the // scanner also refuses to follow; recursing here would never terminate. - if e.data.len() == data.len() && e.data == data { + if bytes == data { return None; } *n += 1; - let safe: String = e + let safe: String = meta .name .chars() .map(|c| { @@ -60,9 +69,9 @@ fn walk(data: &[u8], outdir: &PathBuf, depth: u32, n: &mut usize, prefix: &str) .collect(); let safe = &safe[safe.len().saturating_sub(60)..]; let path = outdir.join(format!("{prefix}{:05}_{safe}", *n)); - if std::fs::write(&path, &e.data).is_ok() { + if std::fs::write(&path, &bytes).is_ok() { let child = format!("{prefix}{:05}_", *n); - walk(&e.data, outdir, depth + 1, n, &child); + walk(&bytes, outdir, depth + 1, n, &child); } None }); diff --git a/crates/exav-unpack/src/formats/aimodel.rs b/crates/exav-unpack/src/formats/aimodel.rs index 72d7e996..988c9765 100644 --- a/crates/exav-unpack/src/formats/aimodel.rs +++ b/crates/exav-unpack/src/formats/aimodel.rs @@ -42,8 +42,8 @@ const SAFETENSORS_SNIFF: usize = 4096; /// proto-0/1 pickles are intentionally *not* sniffed here (their leading byte /// carries no reliable magic), though `extract_aimodel` will still disassemble /// them if routing sends them our way. -pub(crate) fn is_aimodel(data: &[u8]) -> bool { - is_pickle(data) || is_safetensors(data) +pub(crate) fn is_aimodel(p: &crate::Probe) -> bool { + is_pickle(p.head) || is_safetensors_in(p.head, p.len) } /// Pickle protocol 2..5: `0x80` PROTO opcode followed by the protocol number. @@ -55,7 +55,12 @@ fn is_pickle(data: &[u8]) -> bool { /// bounds, the header opening with `{`, and a `"` within the first /// `min(n, 4096)` header bytes. fn is_safetensors(data: &[u8]) -> bool { - if data.len() < 9 { + is_safetensors_in(data, data.len()) +} + +/// [`is_safetensors`] for an object `len` bytes long whose start is `data`. +fn is_safetensors_in(data: &[u8], len: usize) -> bool { + if len < 9 { return false; } let n = u64::from_le_bytes([ @@ -69,7 +74,7 @@ fn is_safetensors(data: &[u8]) -> bool { Some(e) => e, None => return false, }; - if end > data.len() as u64 { + if end > len as u64 { return false; } if data.get(8) != Some(&b'{') { @@ -477,6 +482,10 @@ fn strip_quotes(raw: &[u8]) -> Vec { mod tests { use super::*; + fn is_aimodel(data: &[u8]) -> bool { + super::is_aimodel(&crate::Probe::whole(data)) + } + fn short_binunicode(s: &str) -> Vec { let bytes = s.as_bytes(); let mut v = vec![0x8c, bytes.len() as u8]; diff --git a/crates/exav-unpack/src/formats/alz.rs b/crates/exav-unpack/src/formats/alz.rs index 964f8188..97c13825 100644 --- a/crates/exav-unpack/src/formats/alz.rs +++ b/crates/exav-unpack/src/formats/alz.rs @@ -149,8 +149,7 @@ fn decode_alz_member(method: u8, body: &[u8], cap: u64) -> Option<(Vec, bool use std::io::Cursor; match method { 0 => Some((body.to_vec(), body.len() as u64 > cap)), - #[cfg(feature = "bzip2")] - 1 => crate::bounded_read(bzip2_rs::DecoderReader::new(Cursor::new(body)), cap).ok(), + 1 => crate::bounded_read(super::bzip2_rs::DecoderReader::new(Cursor::new(body)), cap).ok(), 2 => crate::bounded_read(flate2::read::DeflateDecoder::new(Cursor::new(body)), cap).ok(), _ => None, } diff --git a/crates/exav-unpack/src/formats/ar.rs b/crates/exav-unpack/src/formats/ar.rs index 57df98a3..c75f7e4d 100644 --- a/crates/exav-unpack/src/formats/ar.rs +++ b/crates/exav-unpack/src/formats/ar.rs @@ -13,6 +13,17 @@ fn parse_decimal(field: &[u8]) -> u64 { s.parse().unwrap_or(0) } +/// Walk an `ar` archive, each member streamed from where it lies. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let mut source = crate::source::Reader::new(src); + let members = stream_offsets(&mut source, budget.limits().max_buffer_bytes)?; + crate::stream::stream_stored(&mut source, budget, visit, members) +} + /// Parse member offsets from a seekable source (the reader-based streaming /// path): walk the 60-byte headers, resolving GNU long names via the `//` string /// table, and return each *file* member as `(name, data_offset, size)`. Symbol @@ -66,65 +77,6 @@ pub(crate) fn stream_offsets( Ok(out) } -pub(crate) fn extract_ar( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - if !data.starts_with(MAGIC) { - return Err(LimitHit::new("ar: bad magic".to_string())); - } - let mut pos = MAGIC.len(); - // GNU long-name string table ("//" member), if present. - let mut name_table: Vec = Vec::new(); - - while pos + 60 <= data.len() { - let hdr = &data[pos..pos + 60]; - if &hdr[58..60] != b"`\n" { - break; // not a valid header; stop rather than misread - } - let raw_name = &hdr[0..16]; - let size = parse_decimal(&hdr[48..58]) as usize; - let body = pos + 60; - let end = body.saturating_add(size).min(data.len()); - // Clamped to EOF: a member declaring more than the archive holds is - // truncated, and every byte that exists is still scanned. Absent, not - // hidden — so this stays a normal member. See docs/QUIRKS.md. - let member = &data[body..end]; - - let name = resolve_name(raw_name, &name_table); - match name.as_str() { - // GNU string table: holds long names referenced as "/". - // Bounded by the global peak-buffer limit like any materialized blob. - "//" => { - if member.len() as u64 > budget.limits.max_buffer_bytes { - return Err(LimitHit::new( - "ar name table exceeds max-buffer".to_string(), - )); - } - name_table = member.to_vec(); - } - // Symbol tables, not file content. - "/" | "/SYM64/" | "__.SYMDEF" => {} - _ => { - budget.count_entry()?; - let cap = budget.reserve()?; - if member.len() as u64 > cap { - return Err(LimitHit::new(format!("ar member '{name}' exceeds budget"))); - } - budget.commit(member.len() as u64); - if let Some(r) = visit(Entry::new(name, member.to_vec()), budget) { - return Ok(Some(r)); - } - } - } - - // Members are padded to an even byte boundary. - pos = end + (size & 1); - } - Ok(None) -} - /// Resolve an `ar` member name, handling GNU (`/N` → string table) and BSD /// (`#1/N` → name is the first N bytes of the data, not yet stripped here) /// extended-name conventions. BSD long names are left as-is (best effort). diff --git a/crates/exav-unpack/src/formats/arc.rs b/crates/exav-unpack/src/formats/arc.rs index 3e2c912e..08e61d18 100644 --- a/crates/exav-unpack/src/formats/arc.rs +++ b/crates/exav-unpack/src/formats/arc.rs @@ -120,6 +120,12 @@ fn next_header(data: &[u8], off: usize) -> Option { /// somewhere meaningful: the end-of-archive marker, another header, or the end /// of a truncated file. Arbitrary data does not chain. pub(crate) fn is_arc(data: &[u8]) -> bool { + is_arc_in(&crate::Probe::whole(data)) +} + +/// [`is_arc`] for the object `p` looks at. +pub(crate) fn is_arc_in(p: &crate::Probe) -> bool { + let data = p.head; if !plausible_header(data, 0) { return false; } @@ -128,10 +134,11 @@ pub(crate) fn is_arc(data: &[u8]) -> bool { }; // A member whose data runs past the end is a truncated archive, which is // still an archive. - if next >= data.len() { + if next >= p.len { return true; } - data.get(next..next + 2) == Some(&[MARKER, 0][..]) || plausible_header(data, next) + let at_next = p.window(next, HEADER_LEN_V1); + at_next.get(..2) == Some(&[MARKER, 0][..]) || plausible_header(&at_next, 0) } /// Undo the run-length coding: `DLE n` repeats the preceding byte, and `DLE 00` diff --git a/crates/exav-unpack/src/formats/autoit.rs b/crates/exav-unpack/src/formats/autoit.rs index 2e879f0f..26378f52 100644 --- a/crates/exav-unpack/src/formats/autoit.rs +++ b/crates/exav-unpack/src/formats/autoit.rs @@ -6,17 +6,20 @@ //! AutoIt, so carving the embedded script back out lets the engine scan it. //! //! Implemented from the MIT-licensed **AutoIt-Ripper** reference -//! (, MIT) and the public format: +//! (, MIT) and the public format. +//! After the marker, 16 bytes, then a run of `FILE` records: a 4-byte tag that +//! decrypts to `"FILE"`, subtype and name strings, sizes, a checksum, two +//! timestamps and the content, each field under its own key. A compressed +//! member is an `EA05`/`EA06` magic, a big-endian output size and an LZSS +//! bitstream. The two versions differ in: //! -//! * **EA05** (older) — after the marker, a 16-byte region is summed into a -//! checksum, then a run of `FILE` records. Each record's 4-byte tag decrypts -//! (via AutoIt's Mersenne-Twister keystream, seed `0x16FA`) to `"FILE"`; the -//! subtype/name strings, sizes and CRC are keyed likewise, and the content is -//! MT-decrypted with `checksum + 0x22AF`. A compressed member is a `EA05` -//! magic + big-endian output size + an LZSS bitstream. We decode this in full. -//! * **EA06** (newer) — keyed by a floating-point PRNG and a tokenised opcode -//! stream; recognised and reported [`Entry::unsupported`] (`autoit-ea06`), -//! never silently clean. +//! * **EA05** (older): a Mersenne-Twister keystream, byte strings, and a content +//! key that adds the sum of the 16 bytes after the marker; +//! * **EA06** (newer): a floating-point PRNG keystream and UTF-16 strings. +//! +//! A compiled script (`>>>AUTOIT SCRIPT<<<`) is a token stream, turned back into +//! text in the form ClamAV writes it, so that the signatures written against +//! that form match (see [`decompile`]). //! //! Every read is bounds-checked, output grows dynamically (never pre-allocated //! from an attacker size), and members are charged against the [`Budget`], so @@ -24,8 +27,8 @@ use crate::*; -const MARKER_EA05: &[u8; 8] = b"AU3!EA05"; -const MARKER_EA06: &[u8; 8] = b"AU3!EA06"; +pub(crate) const MARKER_EA05: &[u8; 8] = b"AU3!EA05"; +pub(crate) const MARKER_EA06: &[u8; 8] = b"AU3!EA06"; // AutoIt EA05 keystream seeds / XOR keys (format constants). const KEY_FILE_TAG: u32 = 0x16FA; // decrypts the 4-byte record tag to "FILE" @@ -34,11 +37,54 @@ const KEY_SUBTYPE_DATA: u32 = 0xA25E; const KEY_NAME_LEN: u32 = 0x29AC; const KEY_NAME_DATA: u32 = 0xF25E; const KEY_SIZE: u32 = 0x45AA; +#[cfg(test)] const KEY_CRC: u32 = 0xC3D2; const KEY_CONTENT: u32 = 0x22AF; -pub(crate) fn is_autoit(data: &[u8]) -> bool { - find_marker(data, MARKER_EA05).is_some() || find_marker(data, MARKER_EA06).is_some() +#[derive(Clone, Copy)] +enum Cipher { + Mt, + Lame, +} + +/// What differs between the two versions of the record stream. +struct Version { + cipher: Cipher, + /// Strings are UTF-16, their length counted in characters. + unicode: bool, + /// The content key adds the sum of the 16 bytes after the marker. + checksum: bool, + file_tag: u32, + subtype: (u32, u32), + name: (u32, u32), + size: u32, + content: u32, +} + +const EA05: Version = Version { + cipher: Cipher::Mt, + unicode: false, + checksum: true, + file_tag: KEY_FILE_TAG, + subtype: (KEY_SUBTYPE_LEN, KEY_SUBTYPE_DATA), + name: (KEY_NAME_LEN, KEY_NAME_DATA), + size: KEY_SIZE, + content: KEY_CONTENT, +}; + +const EA06: Version = Version { + cipher: Cipher::Lame, + unicode: true, + checksum: false, + file_tag: 0x18EE, + subtype: (0xADBC, 0xB33F), + name: (0xF820, 0xF479), + size: 0x87BC, + content: 0x2477, +}; + +pub(crate) fn is_autoit(p: &crate::Probe) -> bool { + p.find(MARKER_EA05).is_some() || p.find(MARKER_EA06).is_some() } fn find_marker(data: &[u8], needle: &[u8; 8]) -> Option { @@ -60,86 +106,99 @@ pub(crate) fn extract_autoit( visit: Sink, ) -> Result, LimitHit> { if let Some(off) = find_marker(data, MARKER_EA05) { - return ea05(&data[off + 8..], budget, visit); + return records(&data[off + 8..], &EA05, budget, visit); } if let Some(off) = find_marker(data, MARKER_EA06) { - budget.count_entry()?; - let size = (data.len() - (off + 8).min(data.len())) as u64; - let e = Entry::unsupported( - "autoit-ea06".to_string(), - size, - false, - "AutoIt EA06 unsupported", - ); - return Ok(visit(e, budget)); + return records(&data[off + 8..], &EA06, budget, visit); } Ok(None) } -/// Decode the EA05 record stream (`body` starts just past the marker). -fn ea05(body: &[u8], budget: &mut Budget, visit: Sink) -> Result, LimitHit> { +/// Decode the record stream (`body` starts just past the marker). +fn records( + body: &[u8], + v: &Version, + budget: &mut Budget, + visit: Sink, +) -> Result, LimitHit> { if body.len() < 16 { return Ok(None); } - // The first 16 bytes are summed into the content-decrypt checksum. - let checksum: u32 = body[..16] - .iter() - .fold(0u32, |a, &b| a.wrapping_add(b as u32)); + let checksum: u32 = if v.checksum { + body[..16] + .iter() + .fold(0u32, |a, &b| a.wrapping_add(b as u32)) + } else { + 0 + }; let mut pos = 16usize; let mut emitted = 0u32; - // Record tag: 4 bytes that MT-decrypt to "FILE". while let Some(tag_enc) = body.get(pos..pos + 4) { let mut tag = tag_enc.to_vec(); - mt_xor(&mut tag, KEY_FILE_TAG); + xor(&mut tag, v.file_tag, v.cipher); if tag != b"FILE" { break; } pos += 4; - // Subtype and name strings: u32 length (XOR key) then `len` MT-decrypted - // bytes (seed = length + data-key). We only need to advance past them, but - // decode the subtype to recognise the script member. - let Some(subtype) = read_string(body, &mut pos, KEY_SUBTYPE_LEN, KEY_SUBTYPE_DATA) else { + // The subtype says what the member is; the name is only skipped. + let Some(subtype) = read_string(body, &mut pos, v, v.subtype) else { break; }; - if read_string(body, &mut pos, KEY_NAME_LEN, KEY_NAME_DATA).is_none() { + if read_string(body, &mut pos, v, v.name).is_none() { break; } - // Data header: compressed flag (u8), compressed size, uncompressed size, - // CRC (each XOR-keyed), then two 8-byte timestamps. + // Compressed flag (u8), compressed size, uncompressed size and checksum + // (each XOR-keyed), then two 8-byte timestamps. The checksum is not + // verified: a wrong one must not stop the content being scanned. let Some(&comp) = body.get(pos) else { break }; pos += 1; - let (Some(csize_raw), Some(_usize_raw), Some(crc_raw)) = - (le32(body, pos), le32(body, pos + 4), le32(body, pos + 8)) - else { + let Some(csize_raw) = le32(body, pos) else { break; }; - let csize = (csize_raw ^ KEY_SIZE) as usize; - let _crc = crc_raw ^ KEY_CRC; // decoded CRC-32 (not verified) - pos += 12; // csize + usize + crc - pos += 16; // two u64 timestamps + let csize = (csize_raw ^ v.size) as usize; + pos += 12 + 16; if (csize as i32) < 0 { break; } - let Some(enc) = body.get(pos..pos + csize) else { + let Some(enc) = pos.checked_add(csize).and_then(|end| body.get(pos..end)) else { break; }; pos += csize; let mut content = enc.to_vec(); - mt_xor(&mut content, checksum.wrapping_add(KEY_CONTENT)); + xor(&mut content, checksum.wrapping_add(v.content), v.cipher); let cap = budget.reserve()?; - let out = if comp == 1 { - match decompress_ea05(&content, cap as usize) { + let (content, complete) = if comp == 1 { + match decompress(&content, cap as usize) { Some(v) => v, None => continue, } } else { - content + (content, true) + }; + let (out, mut unsupported) = match subtype.as_str() { + ">>>AUTOIT SCRIPT<<<" => match decompile(&content, cap as usize) { + Ok(text) => (text, None), + // Not a token stream at all: the plain script older compilers + // stored under this subtype. + Err(None) => (content, None), + Err(Some(text)) => ( + text, + Some( + "AutoIt script token stream is malformed; the lines before it were scanned", + ), + ), + }, + ">AUTOIT UNICODE SCRIPT<" => (utf16le(&content).into_bytes(), None), + _ => (content, None), }; + if !complete { + unsupported = Some("AutoIt member ends early; the part decoded was scanned"); + } if out.len() < 4 { continue; } @@ -155,25 +214,100 @@ fn ea05(body: &[u8], budget: &mut Budget, visit: Sink) -> Result format!("autoit-{:03}.au3", emitted + 1) }; emitted += 1; - if let Some(r) = visit(Entry::new(name, out), budget) { + let mut e = Entry::new(name, out); + e.unsupported = unsupported; + if let Some(r) = visit(e, budget) { return Ok(Some(r)); } } Ok(None) } -/// Read an XOR-length + MT-encrypted string, advancing `pos`. EA05 strings are -/// single-byte (non-unicode). -fn read_string(body: &[u8], pos: &mut usize, key_len: u32, key_data: u32) -> Option { - let len = (le32(body, *pos)? ^ key_len) as usize; +/// Read an XOR-keyed length, then that many encrypted characters (keyed by the +/// length plus `keys.1`), advancing `pos`. +fn read_string(body: &[u8], pos: &mut usize, v: &Version, keys: (u32, u32)) -> Option { + let chars = le32(body, *pos)? ^ keys.0; *pos += 4; - if (len as i32) < 0 { + if (chars as i32) < 0 { return None; } - let mut bytes = body.get(*pos..*pos + len)?.to_vec(); + let len = if v.unicode { + chars as usize * 2 + } else { + chars as usize + }; + let mut bytes = body.get(*pos..pos.checked_add(len)?)?.to_vec(); *pos += len; - mt_xor(&mut bytes, (len as u32).wrapping_add(key_data)); - Some(String::from_utf8_lossy(&bytes).into_owned()) + xor(&mut bytes, chars.wrapping_add(keys.1), v.cipher); + Some(if v.unicode { + utf16le(&bytes) + } else { + String::from_utf8_lossy(&bytes).into_owned() + }) +} + +fn utf16le(b: &[u8]) -> String { + let units: Vec = b + .as_chunks::<2>() + .0 + .iter() + .map(|c| u16::from_le_bytes(*c)) + .collect(); + String::from_utf16_lossy(&units) +} + +/// XOR `buf` in place with the version's keystream seeded by `seed`. +fn xor(buf: &mut [u8], seed: u32, cipher: Cipher) { + match cipher { + Cipher::Mt => mt_xor(buf, seed), + Cipher::Lame => { + let mut prng = Lame::new(seed); + for b in buf.iter_mut() { + *b ^= prng.next_byte(); + } + } + } +} + +// --- AutoIt EA06 keystream (from AutoIt-Ripper lame.py, MIT) ----------------- + +/// A lagged-Fibonacci generator whose output is read through a double in +/// [0, 1), then scaled to a byte. +struct Lame { + c0: usize, + c1: usize, + grp: [u32; 17], +} + +impl Lame { + fn new(mut seed: u32) -> Self { + let mut grp = [0u32; 17]; + for g in grp.iter_mut() { + seed = 1u32.wrapping_sub(seed.wrapping_mul(0x53A9_B4FB)); + *g = seed; + } + let mut l = Lame { c0: 0, c1: 10, grp }; + for _ in 0..9 { + l.step(); + } + l + } + + fn step(&mut self) -> f64 { + let r = self.grp[self.c0] + .rotate_left(9) + .wrapping_add(self.grp[self.c1].rotate_left(13)); + self.grp[self.c0] = r; + self.c0 = if self.c0 == 0 { 16 } else { self.c0 - 1 }; + self.c1 = if self.c1 == 0 { 16 } else { self.c1 - 1 }; + let bits = (((r >> 12) | 0x3FF0_0000) as u64) << 32 | (r << 20) as u64; + f64::from_bits(bits) - 1.0 + } + + fn next_byte(&mut self) -> u8 { + self.step(); + (self.step() * 256.0) as u8 + } } // --- AutoIt Mersenne-Twister keystream (from AutoIt-Ripper mt.py, MIT) -------- @@ -240,7 +374,7 @@ impl Mt { } } -// --- EA05 LZSS decompressor (from AutoIt-Ripper decompress.py, MIT) ----------- +// --- LZSS decompressor (from AutoIt-Ripper decompress.py, MIT) ---------------- /// MSB-first bit reader over a byte slice. struct Bits<'a> { @@ -273,7 +407,7 @@ impl<'a> Bits<'a> { } } -/// Read an EA05 match length via the variable-length ladder (min 3). +/// Read a match length via the variable-length ladder (min 3). fn read_match_len(bits: &mut Bits) -> usize { // (base, bits, sentinel) const LADDER: &[(usize, u32, u32)] = &[ @@ -306,11 +440,15 @@ fn read_match_len(bits: &mut Bits) -> usize { } } -/// Decompress a `EA05`-magic + big-endian-size + LZSS bitstream, bounded by `cap`. -fn decompress_ea05(content: &[u8], cap: usize) -> Option> { - if content.len() < 8 || &content[0..4] != b"EA05" { - return None; - } +/// Decompress an `EA05`/`EA06` magic + big-endian size + LZSS bitstream, +/// bounded by `cap`. The magic sets the flag bit that marks a literal. The +/// flag is false when the stream ended before the size it declares. +fn decompress(content: &[u8], cap: usize) -> Option<(Vec, bool)> { + let literal = match content.get(0..4)? { + b"EA05" => 0, + b"EA06" => 1, + _ => return None, + }; let mut want = be32(content, 4)? as usize; if want == 0 { want = content.len(); @@ -320,9 +458,12 @@ fn decompress_ea05(content: &[u8], cap: usize) -> Option> { let mut bits = Bits::new(&content[8..]); let mut out: Vec = Vec::new(); while !bits.err && out.len() < want { - if bits.get(1) == 0 { - // literal - out.push(bits.get(8) as u8); + if bits.get(1) == literal { + let b = bits.get(8) as u8; + if bits.err { + break; + } + out.push(b); } else { let offset = bits.get(15) as usize; let len = read_match_len(&mut bits); @@ -335,13 +476,183 @@ fn decompress_ea05(content: &[u8], cap: usize) -> Option> { } } } - Some(out) + let complete = out.len() >= want; + Some((out, complete)) +} + +// --- Token stream to script text (token layout from AutoIt-Ripper, MIT) ------ + +/// Rebuild the text of a compiled script from its token stream, one line per +/// `0x7F` token, in the form ClamAV writes it, which is the form signatures +/// are written against. Observed on ClamAV's own output, byte for byte over +/// 38,000 lines of corpus scripts plus a crafted one for the rarer tokens: +/// +/// * each token is followed by a space, except a user function's name, which +/// runs into the `(` after it; lines end with LF, and nothing is indented; +/// * keywords and built-in functions named by index are upper-cased; names +/// stored as text are written as stored; +/// * a 32-bit integer is `0x%08x`; a 64-bit one is its high word shifted up +/// plus its low word sign-extended, as `0x%016x`; a float is C's `%g`; +/// * a string is quoted without escaping, and each UTF-16 unit is written as +/// its low byte. +/// +/// `Err(None)` when not even one line decodes, which is content that is not a +/// token stream; `Err(Some(text))` holds the lines before a malformed token. +/// Stops once past `cap` bytes; the caller reports that as a limit. +fn decompile(tokens: &[u8], cap: usize) -> Result, Option>> { + use super::autoit_data::{FUNCTIONS, KEYWORDS}; + let fail = |out: Vec| Err(Some(out).filter(|o| !o.is_empty())); + let mut t = Tokens { d: tokens, pos: 0 }; + let mut out = Vec::new(); + let Some(lines) = t.u32() else { + return fail(out); + }; + let mut line = 0u32; + while line < lines && out.len() <= cap { + let Some(op) = t.u8() else { + return fail(out); + }; + let ok = match op { + 0x7F => { + line += 1; + out.push(b'\n'); + continue; + } + 0x00 | 0x01 => { + let table: &[&str] = if op == 0 { &KEYWORDS } else { &FUNCTIONS }; + t.i32() + .and_then(|i| table.get(usize::try_from(i).ok()?)) + .map(|name| out.extend(name.bytes().map(|b| b.to_ascii_uppercase()))) + } + 0x05 => t + .u32() + .map(|n| out.extend_from_slice(format!("0x{n:08x}").as_bytes())), + 0x10 => t.u64().map(|n| { + let v = (n & !0xFFFF_FFFF).wrapping_add(n as u32 as i32 as i64 as u64); + out.extend_from_slice(format!("0x{v:016x}").as_bytes()); + }), + 0x20 => t + .f64() + .map(|f| out.extend_from_slice(c_float_g(f).as_bytes())), + 0x30..=0x37 => { + let prefix: &[u8] = match op { + 0x32 => b"@", + 0x33 => b"$", + 0x35 => b".", + 0x36 => b"\"", + _ => b"", + }; + out.extend_from_slice(prefix); + let ok = t.string(&mut out); + if op == 0x36 { + out.push(b'"'); + } + // A user function's name runs into its `(`. + if op == 0x34 { + if ok.is_none() { + return fail(out); + } + continue; + } + ok + } + 0x40..=0x58 => { + out.extend_from_slice(OPERATORS[(op - 0x40) as usize].as_bytes()); + Some(()) + } + _ => None, + }; + if ok.is_none() { + return fail(out); + } + out.push(b' '); + } + Ok(out) +} + +const OPERATORS: [&str; 25] = [ + ",", "=", ">", "<", "<>", ">=", "<=", "(", ")", "+", "-", "/", "*", "&", "[", "]", "==", "^", + "+=", "-=", "/=", "*=", "&=", "?", ":", +]; + +/// `f` as C's `printf("%g")` writes it: six significant digits, trailing zeros +/// dropped, exponent form below 1e-4 and from 1e6 with a signed exponent of at +/// least two digits. +fn c_float_g(f: f64) -> String { + if !f.is_finite() { + return match (f.is_nan(), f.is_sign_negative()) { + (true, _) => "nan", + (false, true) => "-inf", + (false, false) => "inf", + } + .to_string(); + } + let trim = |s: String| { + if s.contains('.') { + s.trim_end_matches('0').trim_end_matches('.').to_string() + } else { + s + } + }; + // The exponent after rounding to six digits decides the form. + let sci = format!("{f:.5e}"); + let (mantissa, exp) = sci.split_once('e').unwrap_or((&sci, "0")); + let exp: i32 = exp.parse().unwrap_or(0); + if (-4..6).contains(&exp) { + trim(format!("{f:.*}", (5 - exp) as usize)) + } else { + let sign = if exp < 0 { '-' } else { '+' }; + format!("{}e{sign}{:02}", trim(mantissa.to_string()), exp.abs()) + } +} + +struct Tokens<'a> { + d: &'a [u8], + pos: usize, +} + +impl Tokens<'_> { + fn take(&mut self) -> Option<[u8; N]> { + let b = self.d.get(self.pos..self.pos.checked_add(N)?)?; + self.pos += N; + b.try_into().ok() + } + fn u8(&mut self) -> Option { + self.take::<1>().map(|b| b[0]) + } + fn u32(&mut self) -> Option { + self.take().map(u32::from_le_bytes) + } + fn i32(&mut self) -> Option { + self.take().map(i32::from_le_bytes) + } + fn u64(&mut self) -> Option { + self.take().map(u64::from_le_bytes) + } + fn f64(&mut self) -> Option { + self.take().map(f64::from_le_bytes) + } + + /// A length in characters, then that many UTF-16 units XOR-keyed by it; + /// the low byte of each is appended to `out`. + fn string(&mut self, out: &mut Vec) -> Option<()> { + let key = self.u32()?; + let len = (key as usize).checked_mul(2)?; + let raw = self.d.get(self.pos..self.pos.checked_add(len)?)?; + self.pos += len; + out.extend(raw.as_chunks::<2>().0.iter().map(|c| c[0] ^ key as u8)); + Some(()) + } } #[cfg(test)] mod tests { use super::*; + fn is_autoit(data: &[u8]) -> bool { + super::is_autoit(&crate::Probe::whole(data)) + } + /// MT is symmetric, so encrypting == decrypting. fn mt_apply(plain: &[u8], seed: u32) -> Vec { let mut v = plain.to_vec(); @@ -434,8 +745,11 @@ mod tests { fn lzss_literal_roundtrip() { let plain = b"MALWARETEST-lzss-literal-stream"; let stream = ea05_all_literal(plain); - let out = decompress_ea05(&stream, 1 << 20).unwrap(); + let (out, complete) = decompress(&stream, 1 << 20).unwrap(); assert_eq!(out, plain); + assert!(complete); + let (_, complete) = decompress(&stream[..stream.len() - 6], 1 << 20).unwrap(); + assert!(!complete, "a cut stream says so"); } #[test] @@ -461,14 +775,125 @@ mod tests { assert_eq!(entries[0].data, script); } + fn unhex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + /// The EA06 keystream, against AutoIt-Ripper's `LAME` seeded with the tag key. #[test] - fn ea06_marker_unsupported() { - let mut blob = MARKER_EA06.to_vec(); - blob.extend_from_slice(&[0xde, 0xad, 0xbe, 0xef, 0, 1, 2, 3]); + fn ea06_keystream_matches_the_reference() { + let mut ks = [0u8; 16]; + xor(&mut ks, EA06.file_tag, Cipher::Lame); + assert_eq!(ks.to_vec(), unhex("2d0a8617b6b371a0071084f7e5bae729")); + } + + /// A token stream and its text in ClamAV's form: keywords and a function + /// by index, a function by name, quotes inside a string, integers and + /// floats. + const TOKENS: &str = "04000000001c0000003301000000590041013a00000047360c000000670069007e006200690060003f003e0022006800600060004036030000006a006d0077004036040000004600610061007400487f00040000003301000000590000050000007f31060000004b0055004100440049005e00470500000000403611000000620070006800310033005c0050005d004600500043005400450054004200450033004020000000000000f83f4020408cb5781daf1544487f00080000007f"; + const SOURCE: &str = "LOCAL $X = DLLCALL ( \"kernel32.dll\" , \"int\" , \"Beep\" ) \nIF $X THEN \nMSGBOX ( 0x00000000 , \"say \"MALWARETEST\"\" , 1.5 , 1e+20 ) \nENDIF \n"; + + #[test] + fn tokens_decompile_as_clamav_writes_them() { + assert_eq!( + decompile(&unhex(TOKENS), 1 << 20).unwrap(), + SOURCE.as_bytes() + ); + let cut = unhex(TOKENS); + match decompile(&cut[..cut.len() - 3], 1 << 20) { + Err(Some(text)) => assert!(text.starts_with(b"LOCAL $X")), + other => panic!("the lines before the cut are kept: {other:?}"), + } + assert!(matches!( + decompile(b"; plain script text", 1 << 20), + Err(None) + )); + } + + /// The rarer tokens, against what ClamAV wrote for a crafted script: 64-bit + /// integers (whose low word it sign-extends), floats as `%g`, a user + /// function's name running into its `(`, and a non-ASCII character as the + /// low byte of its UTF-16 unit. + #[test] + fn rare_tokens_render_as_clamav_writes_them() { + let mut t = 1u32.to_le_bytes().to_vec(); + for v in [ + 0xffff_ffffu64, + 0x1_0000_0002, + 0x1234_5678_9abc_def0, + 0x8000_0000, + 5, + ] { + t.push(0x10); + t.extend_from_slice(&v.to_le_bytes()); + } + for f in [6.0f64, 0.5, 1e20, 123456.789, -2.5, 1e-7, 1.2345678] { + t.push(0x20); + t.extend_from_slice(&f.to_le_bytes()); + } + let key = 3u32; + t.push(0x34); + t.extend_from_slice(&key.to_le_bytes()); + for c in "F\u{2014}N".encode_utf16() { + t.extend_from_slice(&(c ^ key as u16).to_le_bytes()); + } + t.extend_from_slice(&[0x47, 0x48, 0x7F]); + let want = b"0xffffffffffffffff 0x0000000100000002 0x123456779abcdef0 \ + 0xffffffff80000000 0x0000000000000005 6 0.5 1e+20 123457 -2.5 1e-07 \ + 1.23457 F\x14N( ) \n"; + assert_eq!(decompile(&t, 1 << 20).unwrap(), want); + } + + /// A whole EA06 record: the tag, UTF-16 subtype and name, and a compressed + /// token stream, all under the EA06 keystream, come out as source text. + #[test] + fn ea06_script_is_extracted_as_source() { + let lame = |b: &[u8], seed: u32| { + let mut v = b.to_vec(); + xor(&mut v, seed, Cipher::Lame); + v + }; + let utf16 = |s: &str| { + s.encode_utf16() + .flat_map(u16::to_le_bytes) + .collect::>() + }; + let comp = ea06_all_literal(&unhex(TOKENS)); + let mut d = MARKER_EA06.to_vec(); + d.extend_from_slice(&[0x5a; 16]); // not summed into any key for EA06 + d.extend_from_slice(&lame(b"FILE", EA06.file_tag)); + for (s, keys) in [(">>>AUTOIT SCRIPT<<<", EA06.subtype), ("s.au3", EA06.name)] { + let n = s.encode_utf16().count() as u32; + d.extend_from_slice(&(n ^ keys.0).to_le_bytes()); + d.extend_from_slice(&lame(&utf16(s), n.wrapping_add(keys.1))); + } + d.push(1); + d.extend_from_slice(&(comp.len() as u32 ^ EA06.size).to_le_bytes()); + d.extend_from_slice(&[0; 8 + 16]); + d.extend_from_slice(&lame(&comp, EA06.content)); let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Autoit, &blob, &mut budget).unwrap(); + let entries = extract(Format::Autoit, &d, &mut budget).unwrap(); assert_eq!(entries.len(), 1); - assert!(entries[0].unsupported.is_some()); + assert_eq!(entries[0].unsupported, None); + assert_eq!(entries[0].data, SOURCE.as_bytes()); + } + + /// [`ea05_all_literal`] with EA06's literal flag (1) and magic. + fn ea06_all_literal(plain: &[u8]) -> Vec { + let mut bits: Vec = Vec::new(); + for &b in plain { + bits.push(1); + bits.extend((0..8).rev().map(|k| (b >> k) & 1)); + } + bits.resize(bits.len().div_ceil(8) * 8 + 8, 0); + let mut out = b"EA06".to_vec(); + out.extend_from_slice(&(plain.len() as u32).to_be_bytes()); + let packed = bits.as_slice().chunks(8); + out.extend(packed.map(|c| c.iter().fold(0u8, |a, &b| a << 1 | b))); + out } #[test] @@ -476,7 +901,7 @@ mod tests { for cut in 0..40usize { let full = build_ea05_stored(b"x", b"MALWARETEST short"); let mut budget = Budget::new(Limits::default()); - let _ = extract(Format::Autoit, &full[..cut.min(full.len())], &mut budget).unwrap(); + let _ = extract(Format::Autoit, &&full[..cut.min(full.len())], &mut budget).unwrap(); } let mut budget = Budget::new(Limits::default()); assert!(extract(Format::Autoit, b"not autoit", &mut budget) diff --git a/crates/exav-unpack/src/formats/autoit_data.rs b/crates/exav-unpack/src/formats/autoit_data.rs new file mode 100644 index 00000000..353b78d2 --- /dev/null +++ b/crates/exav-unpack/src/formats/autoit_data.rs @@ -0,0 +1,90 @@ +//! The keywords and built-in functions a compiled AutoIt3 token stream refers +//! to by index. From AutoIt-Ripper's `autoit_data.py` (MIT; see `NOTICE`), in +//! the same order: the index is the token's operand. + +#[rustfmt::skip] +pub(super) const KEYWORDS: [&str; 45] = [ + "", "And", "Or", "Not", "If", "Then", "Else", "ElseIf", "EndIf", "While", "WEnd", + "Do", "Until", "For", "Next", "To", "Step", "In", "ExitLoop", "ContinueLoop", "Select", + "Case", "EndSelect", "Switch", "EndSwitch", "ContinueCase", "Dim", "ReDim", "Local", + "Global", "Const", "Static", "Func", "EndFunc", "Return", "Exit", "ByRef", "With", + "EndWith", "True", "False", "Default", "Null", "Volatile", "Enum", +]; + +#[rustfmt::skip] +pub(super) const FUNCTIONS: [&str; 406] = [ + "Abs", "ACos", "AdlibRegister", "AdlibUnRegister", "Asc", "AscW", "ASin", "Assign", "ATan", + "AutoItSetOption", "AutoItWinGetTitle", "AutoItWinSetTitle", "Beep", "Binary", "BinaryLen", + "BinaryMid", "BinaryToString", "BitAND", "BitNOT", "BitOR", "BitRotate", "BitShift", + "BitXOR", "BlockInput", "Break", "Call", "CDTray", "Ceiling", "Chr", "ChrW", "ClipGet", + "ClipPut", "ConsoleRead", "ConsoleWrite", "ConsoleWriteError", "ControlClick", + "ControlCommand", "ControlDisable", "ControlEnable", "ControlFocus", "ControlGetFocus", + "ControlGetHandle", "ControlGetPos", "ControlGetText", "ControlHide", "ControlListView", + "ControlMove", "ControlSend", "ControlSetText", "ControlShow", "ControlTreeView", "Cos", + "Dec", "DirCopy", "DirCreate", "DirGetSize", "DirMove", "DirRemove", "DllCall", + "DllCallAddress", "DllCallbackFree", "DllCallbackGetPtr", "DllCallbackRegister", "DllClose", + "DllOpen", "DllStructCreate", "DllStructGetData", "DllStructGetPtr", "DllStructGetSize", + "DllStructSetData", "DriveGetDrive", "DriveGetFileSystem", "DriveGetLabel", + "DriveGetSerial", "DriveGetType", "DriveMapAdd", "DriveMapDel", "DriveMapGet", + "DriveSetLabel", "DriveSpaceFree", "DriveSpaceTotal", "DriveStatus", "DUMMYSPEEDTEST", + "EnvGet", "EnvSet", "EnvUpdate", "Eval", "Execute", "Exp", "FileChangeDir", "FileClose", + "FileCopy", "FileCreateNTFSLink", "FileCreateShortcut", "FileDelete", "FileExists", + "FileFindFirstFile", "FileFindNextFile", "FileFlush", "FileGetAttrib", "FileGetEncoding", + "FileGetLongName", "FileGetPos", "FileGetShortcut", "FileGetShortName", "FileGetSize", + "FileGetTime", "FileGetVersion", "FileInstall", "FileMove", "FileOpen", "FileOpenDialog", + "FileRead", "FileReadLine", "FileReadToArray", "FileRecycle", "FileRecycleEmpty", + "FileSaveDialog", "FileSelectFolder", "FileSetAttrib", "FileSetEnd", "FileSetPos", + "FileSetTime", "FileWrite", "FileWriteLine", "Floor", "FtpSetProxy", "FuncName", + "GUICreate", "GUICtrlCreateAvi", "GUICtrlCreateButton", "GUICtrlCreateCheckbox", + "GUICtrlCreateCombo", "GUICtrlCreateContextMenu", "GUICtrlCreateDate", "GUICtrlCreateDummy", + "GUICtrlCreateEdit", "GUICtrlCreateGraphic", "GUICtrlCreateGroup", "GUICtrlCreateIcon", + "GUICtrlCreateInput", "GUICtrlCreateLabel", "GUICtrlCreateList", "GUICtrlCreateListView", + "GUICtrlCreateListViewItem", "GUICtrlCreateMenu", "GUICtrlCreateMenuItem", + "GUICtrlCreateMonthCal", "GUICtrlCreateObj", "GUICtrlCreatePic", "GUICtrlCreateProgress", + "GUICtrlCreateRadio", "GUICtrlCreateSlider", "GUICtrlCreateTab", "GUICtrlCreateTabItem", + "GUICtrlCreateTreeView", "GUICtrlCreateTreeViewItem", "GUICtrlCreateUpdown", + "GUICtrlDelete", "GUICtrlGetHandle", "GUICtrlGetState", "GUICtrlRead", "GUICtrlRecvMsg", + "GUICtrlRegisterListViewSort", "GUICtrlSendMsg", "GUICtrlSendToDummy", "GUICtrlSetBkColor", + "GUICtrlSetColor", "GUICtrlSetCursor", "GUICtrlSetData", "GUICtrlSetDefBkColor", + "GUICtrlSetDefColor", "GUICtrlSetFont", "GUICtrlSetGraphic", "GUICtrlSetImage", + "GUICtrlSetLimit", "GUICtrlSetOnEvent", "GUICtrlSetPos", "GUICtrlSetResizing", + "GUICtrlSetState", "GUICtrlSetStyle", "GUICtrlSetTip", "GUIDelete", "GUIGetCursorInfo", + "GUIGetMsg", "GUIGetStyle", "GUIRegisterMsg", "GUISetAccelerators", "GUISetBkColor", + "GUISetCoord", "GUISetCursor", "GUISetFont", "GUISetHelp", "GUISetIcon", "GUISetOnEvent", + "GUISetState", "GUISetStyle", "GUIStartGroup", "GUISwitch", "Hex", "HotKeySet", + "HttpSetProxy", "HttpSetUserAgent", "HWnd", "InetClose", "InetGet", "InetGetInfo", + "InetGetSize", "InetRead", "IniDelete", "IniRead", "IniReadSection", "IniReadSectionNames", + "IniRenameSection", "IniWrite", "IniWriteSection", "InputBox", "Int", "IsAdmin", "IsArray", + "IsBinary", "IsBool", "IsDeclared", "IsDllStruct", "IsFloat", "IsFunc", "IsHWnd", "IsInt", + "IsKeyword", "IsMap", "IsNumber", "IsObj", "IsPtr", "IsString", "Log", "MapAppend", + "MapExists", "MapKeys", "MapRemove", "MemGetStats", "Mod", "MouseClick", "MouseClickDrag", + "MouseDown", "MouseGetCursor", "MouseGetPos", "MouseMove", "MouseUp", "MouseWheel", + "MsgBox", "Number", "ObjCreate", "ObjCreateInterface", "ObjEvent", "ObjGet", "ObjName", + "OnAutoItExitRegister", "OnAutoItExitUnRegister", "Opt", "Ping", "PixelChecksum", + "PixelGetColor", "PixelSearch", "ProcessClose", "ProcessExists", "ProcessGetStats", + "ProcessList", "ProcessSetPriority", "ProcessWait", "ProcessWaitClose", "ProgressOff", + "ProgressOn", "ProgressSet", "Ptr", "Random", "RegDelete", "RegEnumKey", "RegEnumVal", + "RegRead", "RegWrite", "Round", "Run", "RunAs", "RunAsWait", "RunWait", "Send", + "SendKeepActive", "SetError", "SetExtended", "ShellExecute", "ShellExecuteWait", "Shutdown", + "Sin", "Sleep", "SoundPlay", "SoundSetWaveVolume", "SplashImageOn", "SplashOff", + "SplashTextOn", "Sqrt", "SRandom", "StatusbarGetText", "StderrRead", "StdinWrite", + "StdioClose", "StdoutRead", "String", "StringAddCR", "StringCompare", "StringFormat", + "StringFromASCIIArray", "StringInStr", "StringIsAlNum", "StringIsAlpha", "StringIsASCII", + "StringIsDigit", "StringIsFloat", "StringIsInt", "StringIsLower", "StringIsSpace", + "StringIsUpper", "StringIsXDigit", "StringLeft", "StringLen", "StringLower", "StringMid", + "StringRegExp", "StringRegExpReplace", "StringReplace", "StringReverse", "StringRight", + "StringSplit", "StringStripCR", "StringStripWS", "StringToASCIIArray", "StringToBinary", + "StringTrimLeft", "StringTrimRight", "StringUpper", "Tan", "TCPAccept", "TCPCloseSocket", + "TCPConnect", "TCPListen", "TCPNameToIP", "TCPRecv", "TCPSend", "TCPShutdown", "TCPStartup", + "TimerDiff", "TimerInit", "ToolTip", "TrayCreateItem", "TrayCreateMenu", "TrayGetMsg", + "TrayItemDelete", "TrayItemGetHandle", "TrayItemGetState", "TrayItemGetText", + "TrayItemSetOnEvent", "TrayItemSetState", "TrayItemSetText", "TraySetClick", "TraySetIcon", + "TraySetOnEvent", "TraySetPauseIcon", "TraySetState", "TraySetToolTip", "TrayTip", "UBound", + "UDPBind", "UDPCloseSocket", "UDPOpen", "UDPRecv", "UDPSend", "UDPShutdown", "UDPStartup", + "VarGetType", "WinActivate", "WinActive", "WinClose", "WinExists", "WinFlash", + "WinGetCaretPos", "WinGetClassList", "WinGetClientSize", "WinGetHandle", "WinGetPos", + "WinGetProcess", "WinGetState", "WinGetText", "WinGetTitle", "WinKill", "WinList", + "WinMenuSelectItem", "WinMinimizeAll", "WinMinimizeAllUndo", "WinMove", "WinSetOnTop", + "WinSetState", "WinSetTitle", "WinSetTrans", "WinWait", "WinWaitActive", "WinWaitClose", + "WinWaitNotActive", +]; diff --git a/crates/exav-unpack/src/formats/bzip2.rs b/crates/exav-unpack/src/formats/bzip2.rs index 7d47b331..a7419e03 100644 --- a/crates/exav-unpack/src/formats/bzip2.rs +++ b/crates/exav-unpack/src/formats/bzip2.rs @@ -1,92 +1,81 @@ -#![allow(unused_imports)] -use crate::*; -use std::io::{BufReader, Cursor, Read, Seek, Write}; +//! bzip2: one member, every concatenated stream decoded in turn as it is read. +use crate::source::{ByteSource, Reader}; +use crate::stream::{emit_stream, single_meta, Visit}; +use crate::{Budget, LimitHit}; +use std::io::Read; -pub(crate) fn extract_bzip2( - data: &[u8], +pub(crate) fn walk( + src: &dyn ByteSource, budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { + visit: Visit, +) -> Result, LimitHit> { budget.count_entry()?; - let cap = budget.reserve()?; - let (out, truncated) = decode_bzip2_streams(data, cap)?; - if truncated { - return Err(LimitHit::new("bzip2 member exceeds budget".to_string())); - } - ratio_guard(data.len() as u64, out.len() as u64, budget)?; - budget.commit(out.len() as u64); - Ok(visit(Entry::new("bzip2-content".to_string(), out), budget)) + let mut dec = content_reader(src, budget.limits().max_buffer_bytes); + emit_stream( + &single_meta("bzip2-content", src, None), + &mut dec, + budget, + visit, + ) } -/// Decode every concatenated bzip2 stream (a bzip2 file may be several — `pbzip2` -/// produces multi-stream output, and the payload can live in a later one). -/// `DecoderReader` decodes only the first stream. +/// A reader over the decompressed content of `data`, every concatenated stream +/// of it (`pbzip2` writes multi-stream files, and the payload can live in a +/// later one; `DecoderReader` decodes only the first). /// /// bzip2's decoder buffers a full block ahead, so the exact stream boundary -/// can't be recovered from the decoder. Instead split on the `BZh[1-9]` stream -/// magic and decode each slice independently. This is a no-op for the common -/// single-stream file, and if a `BZh[1-9]` happens to occur *inside* compressed -/// data (≈1 in 2^32 per position) a slice fails to decode and we fall back to a -/// plain single-stream decode — so the result is never worse than before. -fn decode_bzip2_streams(data: &[u8], cap: u64) -> Result<(Vec, bool), LimitHit> { - let starts = bzip2_stream_starts(data); +/// can't be recovered from the decoder. Instead the input is split on the +/// `BZh[1-9]` stream magic. A single stream, the common case, is handed back +/// as a decoder over the input. With several magics, each slice between them +/// must decode on its own, or the input is one stream with a `BZh` inside its +/// compressed data (about 1 in 2^32 per position). A reader that has handed +/// bytes to the scanner cannot take them back, so that is settled first, by +/// decoding each slice with its output discarded, and only then is the +/// content streamed the way it turned out. +pub(crate) fn content_reader(data: &dyn ByteSource, cap: u64) -> Box { + let starts = stream_starts(data); + let one = || -> Box { + Box::new(super::bzip2_rs::DecoderReader::new(Reader::new(data))) + }; if starts.len() <= 1 { - return decode_one_bzip2(data, cap); + return one(); } - let mut out = Vec::new(); - for i in 0..starts.len() { - let s = starts[i]; - let e = starts.get(i + 1).copied().unwrap_or(data.len()); - let remaining = cap - out.len() as u64; - match decode_one_bzip2(&data[s..e], remaining) { - Ok((bytes, truncated)) => { - out.extend_from_slice(&bytes); - if truncated { - return Ok((out, true)); - } - } - // A split landed inside a stream (spurious magic): the multi-stream - // assumption is wrong — decode the whole input as one stream. - Err(_) => return decode_one_bzip2(data, cap), - } + let bounds: Vec<(usize, usize)> = starts + .iter() + .enumerate() + .map(|(i, &s)| (s, starts.get(i + 1).copied().unwrap_or(data.len()))) + .collect(); + let each_decodes = bounds.iter().all(|&(s, e)| { + let mut slice = super::bzip2_rs::DecoderReader::new(Reader::range(data, s, e)); + std::io::copy( + &mut (&mut slice).take(cap.saturating_add(1)), + &mut std::io::sink(), + ) + .is_ok() + }); + if !each_decodes { + return one(); } - Ok((out, false)) -} - -/// A reader over the decompressed content of `data`, for the streaming path. -/// -/// The COMMON case — a single stream — is handed back as a decoder, so content -/// decompressing to any size is scanned without being materialized. That is the -/// whole point: a `.bz2` output can be orders of magnitude larger than the file. -/// -/// The multi-stream case is decoded whole instead. It cannot stream: recovering -/// from a spurious `BZh` needs re-decoding the entire input as one stream (see -/// [`decode_bzip2_streams`]), and a reader that has already handed bytes to the -/// scanner cannot take them back. Multi-stream files are rare, so the common -/// case keeps the win and the rare one keeps the recovery. -pub(crate) fn content_reader(data: &[u8], cap: u64) -> Result, LimitHit> { - if bzip2_stream_starts(data).len() <= 1 { - return Ok(Box::new(bzip2_rs::DecoderReader::new(Cursor::new(data)))); + let mut streams: Box = Box::new(std::io::empty()); + for (s, e) in bounds { + streams = Box::new( + streams.chain(super::bzip2_rs::DecoderReader::new(Reader::range( + data, s, e, + ))), + ); } - let (out, _truncated) = decode_bzip2_streams(data, cap)?; - Ok(Box::new(Cursor::new(out))) + streams } /// Offsets of bzip2 stream headers (`BZh` followed by a `1`-`9` block-size). -fn bzip2_stream_starts(data: &[u8]) -> Vec { +fn stream_starts(data: &dyn ByteSource) -> Vec { let mut out = Vec::new(); - for off in memchr::memmem::find_iter(data, b"BZh") { - if matches!(data.get(off + 3), Some(b'1'..=b'9')) { + let mut from = 0; + while let Some(off) = data.find(b"BZh", from, data.len()) { + if matches!(data.window(off + 3, 1).first(), Some(b'1'..=b'9')) { out.push(off); } + from = off + 1; } out } - -/// Decode a single bzip2 stream, capped at `cap` output bytes. -fn decode_one_bzip2(data: &[u8], cap: u64) -> Result<(Vec, bool), LimitHit> { - // A decode failure (bad block-size, corrupt stream) is undecodable content, - // not a resource limit — `corrupt` → `Unscannable`, never `LimitsExceeded`. - bounded_read(bzip2_rs::DecoderReader::new(Cursor::new(data)), cap) - .map_err(|e| LimitHit::corrupt(format!("bzip2: {e}"))) -} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/bitreader.rs b/crates/exav-unpack/src/formats/bzip2_rs/bitreader.rs new file mode 100644 index 00000000..5e5cedfc --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/bitreader.rs @@ -0,0 +1,163 @@ +pub struct BitReader<'a> { + bytes: &'a [u8], + position: usize, +} + +impl<'a> BitReader<'a> { + pub fn new(bytes: &'a [u8]) -> BitReader<'a> { + BitReader { bytes, position: 0 } + } + + pub fn read_u8(&mut self, bit_count: u8) -> Option { + debug_assert!(bit_count <= 8); + + if (self.position + usize::from(bit_count)) > self.bytes.len() * 8 { + return None; + } + + let mut value = 0; + for bit in self.take(usize::from(bit_count)) { + value = (value << 1) | u8::from(bit); + } + Some(value) + } + + pub fn read_u16(&mut self, bit_count: u8) -> Option { + debug_assert!(bit_count <= 16); + + if (self.position + usize::from(bit_count)) > self.bytes.len() * 8 { + return None; + } + + let mut value = 0; + for bit in self.take(usize::from(bit_count)) { + value = (value << 1) | u16::from(bit); + } + Some(value) + } + + pub fn read_u32(&mut self, bit_count: u8) -> Option { + debug_assert!(bit_count <= 32); + + if (self.position + usize::from(bit_count)) > self.bytes.len() * 8 { + return None; + } + + let mut value = 0; + for bit in self.take(usize::from(bit_count)) { + value = (value << 1) | u32::from(bit); + } + Some(value) + } + + pub fn read_u64(&mut self, bit_count: u8) -> Option { + debug_assert!(bit_count <= 64); + + if (self.position + usize::from(bit_count)) > self.bytes.len() * 8 { + return None; + } + + let mut value = 0; + for bit in self.take(usize::from(bit_count)) { + value = (value << 1) | u64::from(bit); + } + Some(value) + } + + pub fn read_bool(&mut self) -> Option { + self.next() + } + + /// Skip arbitrary number of bits. However, you can skip at most to the end of the byte slice. + pub fn advance_by(&mut self, bit_count: usize) -> bool { + let end_position = self.position + bit_count; + if end_position > self.bytes.len() * 8 { + return false; + } + self.position = end_position; + true + } + + /// Returns the position of the cursor, or how many bits have been read so far. + pub fn position(&self) -> usize { + self.position + } +} + +impl<'a> Iterator for BitReader<'a> { + type Item = bool; + + fn next(&mut self) -> Option { + let position = self.position; + let byte = *self.bytes.get(position / 8)?; + + self.position += 1; + + let bit = byte << (position % 8); + Some(bit & 0b1000_0000 != 0) + } +} + +/// A bitreader which can read at most 57 bits, but is faster on systems +/// with 64 bit registers. +/// NOTE: a few points are assumed by this bitreader +/// +/// * `::next` will never yield `None`, even after +/// the guaranteed to be valid 57 bits have been exhausted. +/// * The soft limit on 57 bits can be reset by calling `CachedBitReader::refresh` +/// * At least 8 bytes must be available in the `BitReader` passed to `CachedBitReader::refresh` +/// * If `::next` may have been called at least one time, +/// `CachedBitReader::restore` must be called in order to update the `BitReader::position` +/// * `CachedBitReader::restore` must be called before calling `CachedBitReader::refresh`, +/// unless `CachedBitReader::read() == 0` +#[cfg(target_pointer_width = "64")] +pub struct CachedBitReader { + cache: u64, + read: usize, +} + +#[cfg(target_pointer_width = "64")] +impl CachedBitReader { + pub fn new(reader: &BitReader<'_>) -> Option { + let mut this = Self { cache: 0, read: 0 }; + this.refresh(reader)?; + Some(this) + } + + pub fn refresh(&mut self, reader: &BitReader<'_>) -> Option<()> { + let pos = reader.position / 8; + let data = reader.bytes.get(pos..pos + 8)?; + + self.cache = u64::from_be_bytes(data.try_into().unwrap()); + self.cache <<= reader.position % 8; + self.read = 0; + Some(()) + } + + pub fn read(&self) -> usize { + self.read + } + + pub fn overflowed(&self) -> bool { + self.read() > 57 + } + + pub fn restore(&mut self, reader: &mut BitReader<'_>, read: usize) { + reader.position += read; + } +} + +#[cfg(target_pointer_width = "64")] +impl Iterator for CachedBitReader { + type Item = bool; + + fn next(&mut self) -> Option { + // read the left most bit + let bit = self.cache & !(u64::MAX >> 1); + + self.read += 1; + self.cache <<= 1; + + Some(bit != 0) + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/block/bwt.rs b/crates/exav-unpack/src/formats/bzip2_rs/block/bwt.rs new file mode 100644 index 00000000..91b8a335 --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/block/bwt.rs @@ -0,0 +1,17 @@ +use std::mem; + +pub(crate) fn inverse_bwt(tt: &mut [u32], orig_ptr: usize, mut c: [u32; 256]) -> u32 { + let mut sum = 0u32; + + for ci in c.iter_mut() { + sum += mem::replace(ci, sum); + } + + for i in 0..tt.len() { + let b = (tt[i] & 0xff) as usize; + tt[c[b] as usize] |= (i as u32) << 8; + c[b] += 1; + } + + tt[orig_ptr] >> 8 +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/block/error.rs b/crates/exav-unpack/src/formats/bzip2_rs/block/error.rs new file mode 100644 index 00000000..44312694 --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/block/error.rs @@ -0,0 +1,34 @@ +use std::error::Error as StdError; +use std::fmt::{self, Display, Formatter}; +use std::io; + +/// An error returned by the block decoder +/// +/// At the moment it's not possible to find out what +/// error occurred other than through the `Display` +/// implementation, this will change in a future release. +#[derive(Debug, Clone, PartialEq)] +pub struct BlockError { + reason: &'static str, +} + +impl BlockError { + #[inline(always)] + pub(super) fn new(reason: &'static str) -> Self { + Self { reason } + } +} + +impl Display for BlockError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(self.reason) + } +} + +impl StdError for BlockError {} + +impl From for io::Error { + fn from(err: BlockError) -> io::Error { + io::Error::other(err) + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/block/mod.rs b/crates/exav-unpack/src/formats/bzip2_rs/block/mod.rs new file mode 100644 index 00000000..0c7b01e8 --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/block/mod.rs @@ -0,0 +1,452 @@ +//! bzip2 low-level block APIs + +use std::mem; + +use tinyvec::ArrayVec; + +pub use self::error::BlockError; +use super::bitreader::BitReader; +#[cfg(target_pointer_width = "64")] +use super::bitreader::CachedBitReader; +use super::crc::Hasher; +use super::header::Header; +use super::huffman::HuffmanTree; +use super::move_to_front::MoveToFrontDecoder; + +mod bwt; +mod error; + +const BLOCK_MAGIC: u64 = 0x314159265359; +const FINAL_MAGIC: u64 = 0x177245385090; + +pub(crate) struct Block { + header: Header, + + tt: Vec, + pre_rle_used: u32, + t_pos: u32, + last_byte: i16, + byte_repeats: u8, + repeats: u8, + + hasher: Hasher, + expected_crc: u32, + + state: State, +} + +enum State { + ReadyForRead, + Reading, + NotReady, +} + +impl Block { + pub fn new(header: Header) -> Self { + let max_blocksize = header.max_blocksize(); + + Self { + header, + + tt: Vec::with_capacity(max_blocksize as usize), + pre_rle_used: 0, + t_pos: 0, + last_byte: -1, + byte_repeats: 0, + repeats: 0, + + hasher: Hasher::new(), + expected_crc: 0, + + state: State::NotReady, + } + } + + pub fn is_ready_for_read(&self) -> bool { + match self.state { + State::ReadyForRead => true, + State::Reading | State::NotReady => false, + } + } + + pub fn is_reading(&self) -> bool { + match self.state { + State::Reading => true, + State::ReadyForRead | State::NotReady => false, + } + } + + pub fn is_not_ready(&self) -> bool { + match self.state { + State::NotReady => true, + State::ReadyForRead | State::Reading => false, + } + } + + pub fn set_ready_for_read(&mut self) { + self.state = State::ReadyForRead; + } + + pub fn read( + &mut self, + reader: &mut BitReader<'_>, + out: &mut [u8], + ) -> Result { + match &self.state { + State::ReadyForRead => { + let magic = reader + .read_u64(48) + .ok_or_else(|| BlockError::new("next magic truncated"))?; + match magic { + BLOCK_MAGIC => { + self.read_block(reader)?; + self.state = State::Reading; + + self.read(reader, out) + } + FINAL_MAGIC => { + let _crc = reader + .read_u32(32) + .ok_or_else(|| BlockError::new("whole stream crc truncated"))?; + + // TODO: check whole stream crc + + self.state = State::NotReady; + Ok(0) + } + _ => { + self.state = State::NotReady; + Err(BlockError::new("bad magic value found")) + } + } + } + State::Reading => { + let n = self.read_from_block(out)?; + if n == 0 && !out.is_empty() { + self.state = State::NotReady; + } + + Ok(n) + } + State::NotReady => Err(BlockError::new("not ready")), + } + } + + pub fn read_from_block(&mut self, out: &mut [u8]) -> Result { + let mut read = 0; + + while (self.repeats > 0 || self.pre_rle_used < (self.tt.len() as u32)) && read < out.len() { + if self.repeats > 0 { + out[read] = self.last_byte as u8; + read += 1; + + self.repeats -= 1; + if self.repeats == 0 { + self.last_byte = -1; + } + continue; + } + + self.t_pos = self.tt[self.t_pos as usize]; + let b = self.t_pos as u8; + self.t_pos >>= 8; + self.pre_rle_used += 1; + + if self.byte_repeats == 3 { + self.repeats = b; + self.byte_repeats = 0; + continue; + } + + if self.last_byte == i16::from(b) { + self.byte_repeats += 1; + } else { + self.byte_repeats = 0; + } + self.last_byte = i16::from(b); + + out[read] = b; + read += 1; + } + + if read == 0 && !out.is_empty() { + self.state = State::NotReady; + + let crc = self.hasher.finalyze(); + return if self.expected_crc == crc { + Ok(0) + } else { + Err(BlockError::new("bad crc")) + }; + } + + self.hasher.update(&out[..read]); + Ok(read) + } + + pub fn read_block(&mut self, reader: &mut BitReader<'_>) -> Result<(), BlockError> { + self.hasher = Hasher::new(); + self.tt.clear(); + + self.expected_crc = reader + .read_u32(32) + .ok_or_else(|| BlockError::new("crc truncated"))?; + + let randomised = reader + .read_bool() + .ok_or_else(|| BlockError::new("randomised truncated"))?; + if randomised { + return Err(BlockError::new("randomised expected to be 'normal'")); + } + + let orig_ptr = reader + .read_u32(24) + .ok_or_else(|| BlockError::new("orig ptr truncated"))?; + + let mut huffman_used_symbols = ArrayVec::<[u8; 16]>::new(); + let mut huffman_used_bitmaps = ArrayVec::<[u8; 256]>::new(); + + for i in 0..16 { + if reader + .read_bool() + .ok_or_else(|| BlockError::new("symbol range truncated"))? + { + huffman_used_symbols.push(i); + } + } + + for symbol_range in huffman_used_symbols { + for symbol in 0..16 { + if reader + .read_bool() + .ok_or_else(|| BlockError::new("symbol range truncated"))? + { + huffman_used_bitmaps.push(symbol_range * 16 + symbol); + } + } + } + + if huffman_used_bitmaps.is_empty() { + return Err(BlockError::new("huffman no symbols in input")); + } + + let huffman_groups = reader + .read_u8(3) + .ok_or_else(|| BlockError::new("huffmann groups truncated"))?; + if !(2..=6).contains(&huffman_groups) { + return Err(BlockError::new("invalid number of huffman trees")); + } + + let selectors_used = reader + .read_u16(15) + .ok_or_else(|| BlockError::new("selectors used truncated"))?; + + let mut selectors_list = vec![0u8; usize::from(selectors_used)]; + + let mut move_to_front_decoder = MoveToFrontDecoder::new(); + for selector_item in selectors_list.iter_mut().rev() { + let mut trees = 0; + + while reader + .read_bool() + .ok_or_else(|| BlockError::new("selector truncated"))? + { + trees += 1; + + if trees >= huffman_groups { + return Err(BlockError::new("tree index too large")); + } + } + + *selector_item = move_to_front_decoder.decode(trees); + } + + // limited in lenght of huffman_used_symbols + let mut symbols = [0u8; 256]; + symbols[..huffman_used_bitmaps.len()].copy_from_slice(&huffman_used_bitmaps); + let mut move_to_front_decoder_2 = MoveToFrontDecoder::new_from_symbols(symbols); + + let mut huffman_trees = ArrayVec::<[HuffmanTree; 6]>::new(); + + let mut lengths = ArrayVec::<[u8; super::LEN_258]>::new(); + lengths.set_len(huffman_used_bitmaps.len() + 2); + + for _ in 0..huffman_groups { + let mut length = reader + .read_u8(5) + .ok_or_else(|| BlockError::new("huffman group length truncated"))?; + + for length_item in &mut *lengths { + loop { + if !(1..=20).contains(&length) { + return Err(BlockError::new("huffman length out of range")); + } + + if !reader + .read_bool() + .ok_or_else(|| BlockError::new("length bit1 truncated"))? + { + break; + } + + if reader + .read_bool() + .ok_or_else(|| BlockError::new("length bit2 truncated"))? + { + length -= 1; + } else { + length += 1; + } + } + + *length_item = length; + } + + let tree = HuffmanTree::new(&lengths).map_err(BlockError::new)?; + huffman_trees.push(tree); + } + + let selector = selectors_list + .pop() + .ok_or_else(|| BlockError::new("no tree selectors given"))?; + let mut current_huffman_tree = huffman_trees + .get(usize::from(selector)) + .ok_or_else(|| BlockError::new("tree selector out of range"))?; + + let mut repeat = 0u32; + let mut repeat_power = 0u32; + let mut c = [0u32; 256]; + + let mut decoded = 0; + #[cfg(target_pointer_width = "64")] + let mut r = CachedBitReader::new(reader) + .ok_or_else(|| BlockError::new("huffman bitstream truncated"))?; + loop { + if decoded == 50 { + let selector = selectors_list.pop().ok_or_else(|| { + BlockError::new("insufficient selector indices for number of symbols") + })?; + + current_huffman_tree = huffman_trees + .get(usize::from(selector)) + .ok_or_else(|| BlockError::new("tree selector out of range"))?; + decoded = 0; + } + + #[cfg(target_pointer_width = "64")] + let v = { + let read = r.read(); + let v = current_huffman_tree.decode(&mut r); + + if r.overflowed() { + r.restore(reader, read); + r.refresh(reader) + .ok_or_else(|| BlockError::new("huffman bitstream truncated"))?; + + current_huffman_tree.decode(&mut r) + } else { + v + } + }; + #[cfg(not(target_pointer_width = "64"))] + let v = current_huffman_tree.decode(reader); + + let v = v.ok_or_else(|| BlockError::new("huffman bitstream truncated"))?; + decoded += 1; + + if v < 2 { + if repeat == 0 { + repeat_power = 1; + } + repeat += repeat_power << v; + repeat_power <<= 1; + + if repeat > 2 * 1024 * 1024 { + return Err(BlockError::new("repeat count too large")); + } + continue; + } + + let old_repeat = mem::replace(&mut repeat, 0); + if old_repeat > 0 { + if old_repeat > self.header.max_blocksize() - (self.tt.len() as u32) { + return Err(BlockError::new("repeats past end of block")); + } + + let b = move_to_front_decoder_2.first(); + // extend self.tt with `b` repeated `old_repeat` times + let new_len = self.tt.len() + old_repeat as usize; + self.tt.resize(new_len, u32::from(b)); + c[usize::from(b)] += old_repeat; + } + + if usize::from(v) == (huffman_used_bitmaps.len() + 2) - 1 { + break; + } + + let b = move_to_front_decoder_2.decode((v - 1) as u8); + if self.tt.len() >= self.header.max_blocksize() as usize { + return Err(BlockError::new("data exceeds block size")); + } + + self.tt.push(u32::from(b)); + c[usize::from(b)] += 1; + } + #[cfg(target_pointer_width = "64")] + r.restore(reader, r.read()); + + if (orig_ptr as usize) >= self.tt.len() { + return Err(BlockError::new("orig_ptr out of bounds")); + } + + self.pre_rle_used = 0; + self.t_pos = bwt::inverse_bwt(&mut self.tt, orig_ptr as usize, c); + self.last_byte = -1; + self.byte_repeats = 0; + self.repeats = 0; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::io::{Read, Write}; + + use super::super::DecoderReader; + + fn roundtrip(data: &[u8], level: u32) { + let mut enc = ::bzip2::write::BzEncoder::new(Vec::new(), ::bzip2::Compression::new(level)); + enc.write_all(data).unwrap(); + let bz = enc.finish().unwrap(); + let mut out = Vec::new(); + DecoderReader::new(&bz[..]) + .read_to_end(&mut out) + .unwrap_or_else(|e| panic!("level {level}, {} bytes: {e}", data.len())); + assert!( + out == data, + "level {level}, {} bytes: output differs", + data.len() + ); + } + + #[test] + fn decodes_what_the_reference_encoder_writes() { + let mut x: u32 = 1; + let noise: Vec = (0..230_000) + .map(|_| { + x = x.wrapping_mul(1_103_515_245).wrapping_add(12_345); + (x >> 16) as u8 + }) + .collect(); + let text = b"the quick brown fox jumps over the lazy dog\n".repeat(6000); + let runs: Vec = (0..=255u8).flat_map(|b| vec![b; 1000]).collect(); + for level in [1, 5, 9] { + roundtrip(b"", level); + roundtrip(b"a", level); + roundtrip(&text, level); + roundtrip(&runs, level); + roundtrip(&noise, level); + } + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/crc.rs b/crates/exav-unpack/src/formats/bzip2_rs/crc.rs new file mode 100644 index 00000000..e91dac3c --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/crc.rs @@ -0,0 +1,46 @@ +pub struct Hasher { + // CRC32B hasher + val: crc32fast::Hasher, + // reversed bits + bytes: [u8; 512], +} + +impl Hasher { + pub fn new() -> Self { + Self { + val: crc32fast::Hasher::new(), + bytes: [0; 512], + } + } + + pub fn update(&mut self, mut bytes: &[u8]) { + while !bytes.is_empty() { + let len = bytes.len().min(self.bytes.len()); + self.bytes[..len].copy_from_slice(&bytes[..len]); + bytes = &bytes[len..]; + + for byte in self.bytes.iter_mut() { + *byte = byte.reverse_bits(); + } + self.val.update(&self.bytes[..len]); + } + } + + pub fn finalyze(&self) -> u32 { + let reversed = self.val.clone().finalize(); + // CRC32B to CRC32 + reversed.reverse_bits() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn crc() { + let mut hasher = Hasher::new(); + hasher.update(b"123456789"); + assert_eq!(hasher.finalyze(), 0xFC891918); + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/decoder/error.rs b/crates/exav-unpack/src/formats/bzip2_rs/decoder/error.rs new file mode 100644 index 00000000..f466b73c --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/decoder/error.rs @@ -0,0 +1,50 @@ +use std::error::Error as StdError; +use std::fmt::{self, Display, Formatter}; +use std::io; + +use super::super::block::BlockError; +use super::super::header::HeaderError; + +/// An error returned by [`Decoder`] or [`DecoderReader`] +/// +/// [`Decoder`]: super::Decoder +/// [`DecoderReader`]: super::DecoderReader +#[derive(Debug, Clone, PartialEq)] +pub enum DecoderError { + /// An error was returned by the `Header` decoder + Header(HeaderError), + /// An error was returned by the `Block` decoder + Block(BlockError), +} + +impl Display for DecoderError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + DecoderError::Header(err) => write!(f, "header: {}", err), + DecoderError::Block(err) => write!(f, "block: {}", err), + } + } +} + +impl From for DecoderError { + fn from(err: HeaderError) -> Self { + DecoderError::Header(err) + } +} + +impl From for DecoderError { + fn from(err: BlockError) -> Self { + DecoderError::Block(err) + } +} + +impl StdError for DecoderError {} + +impl From for io::Error { + fn from(err: DecoderError) -> io::Error { + match err { + DecoderError::Header(err) => err.into(), + DecoderError::Block(err) => err.into(), + } + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/decoder/mod.rs b/crates/exav-unpack/src/formats/bzip2_rs/decoder/mod.rs new file mode 100644 index 00000000..2eec44da --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/decoder/mod.rs @@ -0,0 +1,188 @@ +//! bzip2 decoding APIs + +pub use self::error::DecoderError; +pub use self::reader::DecoderReader; +use super::bitreader::BitReader; +use super::block::Block; +use super::header::Header; + +mod error; +mod reader; + +/// A low-level decoder implementation +/// +/// This decoder does no IO by itself, instead enough data +/// has to be written to it in order for it to be able +/// to decode the next block. After that the decompressed content +/// for the block can be read until all of the data from the block +/// has been exhausted. +/// Repeating this process for every block in sequence will result +/// into the entire file being decompressed. +pub struct Decoder { + header_block: Option<(Header, Block)>, + + skip_bits: usize, + in_buf: Vec, + + eof: bool, +} + +/// State returned by [`Decoder::write`] +pub enum WriteState { + /// Enough data has already been written to [`Decoder`] + /// in order for it to be able to decode the next block. + /// Now call [`Decoder::read`] to read the decompressed data. + NeedsRead, + /// N. number of bytes have been written. + Written(usize), +} + +/// State returned by [`Decoder::read`] +pub enum ReadState { + /// Not enough data has been written to the underlying [`Decoder`] + /// in order to allow the next block to be decoded. Call + /// [`Decoder::write`] to write more data. If the end of the file + /// has been reached, call [`Decoder::write`] with an empty buffer. + NeedsWrite(usize), + /// N. number of data has been read + Read(usize), + /// The end of the compressed file has been reached and + /// there is no more data to read + Eof, +} + +impl Decoder { + /// Construct a new [`Decoder`], ready to decompress a new bzip2 file + pub fn new() -> Self { + Self { + header_block: None, + + skip_bits: 0, + in_buf: Vec::new(), + + eof: false, + } + } + + fn space(&self) -> usize { + match &self.header_block { + Some((_, block)) if block.is_reading() => 0, + Some((header, _)) => { + let max_length = max_compressed_block(header) + (self.skip_bits / 8) + 1; + max_length - self.in_buf.len() + } + None => { + Header::from_raw_blocksize(1) + .expect("blocksize is valid") + .max_blocksize() as usize + + 4 + } + } + } + + /// Write more compressed data into this [`Decoder`] + /// + /// See the documentation for [`WriteState`] to decide + /// what to do next. + pub fn write(&mut self, buf: &[u8]) -> Result { + let space = self.space(); + + match &mut self.header_block { + Some((_, block)) if block.is_reading() => Ok(WriteState::NeedsRead), + Some((header, block)) => { + let written = space.min(buf.len()); + + self.in_buf.extend_from_slice(&buf[..written]); + + let minimum = (self.skip_bits / 8) + max_compressed_block(header); + if buf.is_empty() || self.in_buf.len() >= minimum { + block.set_ready_for_read(); + } + + Ok(WriteState::Written(written)) + } + None => { + let written = space.min(buf.len()); + self.in_buf.extend_from_slice(&buf[..written]); + + if self.in_buf.len() < 4 { + return Ok(WriteState::Written(buf.len())); + } + + let header = Header::parse(self.in_buf[..4].try_into().unwrap())?; + let block = Block::new(header.clone()); + self.header_block = Some((header, block)); + + self.skip_bits = 4 * 8; + + if written == buf.len() { + return Ok(WriteState::Written(written)); + } + + match self.write(&buf[written..])? { + WriteState::NeedsRead => unreachable!(), + WriteState::Written(n) => Ok(WriteState::Written(n + written)), + } + } + } + } + + /// Read more decompressed data from this [`Decoder`] + /// + /// See the documentation for [`ReadState`] to decide + /// what to do next. + pub fn read(&mut self, buf: &mut [u8]) -> Result { + match &mut self.header_block { + Some(_) if self.eof => Ok(ReadState::Eof), + Some((_, block)) if block.is_not_ready() => Ok(ReadState::NeedsWrite(self.space())), + Some((_, block)) => { + let mut reader = BitReader::new(&self.in_buf); + reader.advance_by(self.skip_bits); + + let ready_for_read = block.is_ready_for_read(); + + let read = block.read(&mut reader, buf)?; + + if read == 0 { + if !buf.is_empty() { + self.eof = ready_for_read; + } + + return Ok(ReadState::NeedsWrite(self.space())); + } + + if read == 0 && !buf.is_empty() { + self.eof = true; + } + + self.skip_bits = reader.position(); + + if block.is_not_ready() { + let bytes = self.skip_bits / 8; + + self.in_buf.drain(..bytes); + + self.skip_bits -= bytes * 8; + } + + Ok(ReadState::Read(read)) + } + None => Ok(ReadState::NeedsWrite(self.space())), + } + } +} + +/// The most bytes one compressed block can take: every symbol (at most one per +/// block byte, plus end-of-block) at the longest code, 20 bits, plus the block +/// header, selectors (up to 32767 at 7 bits) and code tables (under 8 KiB). +/// Upstream buffered only the block size, which a block of incompressible data +/// exceeds. +fn max_compressed_block(header: &Header) -> usize { + (header.max_blocksize() as usize + 1) * 20 / 8 + 64 * 1024 +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/decoder/reader.rs b/crates/exav-unpack/src/formats/bzip2_rs/decoder/reader.rs new file mode 100644 index 00000000..1c23c043 --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/decoder/reader.rs @@ -0,0 +1,52 @@ +use std::io::{self, Read, Result}; + +use super::{Decoder, ReadState, WriteState}; + +/// A high-level decoder that wraps a [`Read`] and implements [`Read`], yielding decompressed bytes +pub struct DecoderReader { + decoder: Decoder, + + reader: R, +} + +impl DecoderReader { + /// Construct a new decoder from something implementing [`Read`] + pub fn new(reader: R) -> Self { + Self { + decoder: Decoder::new(), + + reader, + } + } +} + +impl Read for DecoderReader { + /// Decompress bzip2 data from the underlying reader + fn read(&mut self, buf: &mut [u8]) -> Result { + let mut read_zero = false; + let mut tmp_buf = [0; 1024]; + + loop { + match self.decoder.read(buf)? { + ReadState::NeedsWrite(space) => { + let read = self.reader.read(&mut tmp_buf[..space.min(1024)])?; + + if read_zero && self.decoder.header_block.is_none() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "The reader is empty?", + )); + } + read_zero = read == 0; + + match self.decoder.write(&tmp_buf[..read])? { + WriteState::NeedsRead => unreachable!(), + WriteState::Written(written) => assert_eq!(written, read), + }; + } + ReadState::Read(n) => return Ok(n), + ReadState::Eof => return Ok(0), + } + } + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/header/error.rs b/crates/exav-unpack/src/formats/bzip2_rs/header/error.rs new file mode 100644 index 00000000..9e8f7535 --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/header/error.rs @@ -0,0 +1,34 @@ +use std::error::Error as StdError; +use std::fmt::{self, Display, Formatter}; +use std::io; + +/// An error returned by [`Header`] +/// +/// [`Header`]: super::Header +#[derive(Debug, Clone, PartialEq)] +pub enum HeaderError { + /// The file signature isn't valid (should be `BZ`) + InvalidSignature, + /// The bzip2 version isn't supported (only the huffman version is supported) + UnsupportedVersion, + /// The blocksize isn't `1..=9` + InvalidBlockSize, +} + +impl Display for HeaderError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(match self { + HeaderError::InvalidSignature => "invalid file signature", + HeaderError::UnsupportedVersion => "unsupported bzip2 version", + HeaderError::InvalidBlockSize => "invalid block-size", + }) + } +} + +impl StdError for HeaderError {} + +impl From for io::Error { + fn from(err: HeaderError) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, err) + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/header/mod.rs b/crates/exav-unpack/src/formats/bzip2_rs/header/mod.rs new file mode 100644 index 00000000..ca6ca660 --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/header/mod.rs @@ -0,0 +1,68 @@ +//! bzip2 low-level header APIs + +pub use self::error::HeaderError; + +mod error; + +/// A bzip2 header +#[derive(Clone)] +pub struct Header { + max_blocksize: u32, +} + +impl Header { + /// Parse a bzip2 header + pub fn parse(buf: [u8; 4]) -> Result { + let signature = &buf[..2]; + if signature != b"BZ" { + return Err(HeaderError::InvalidSignature); + } + + let version = buf[2]; + if version != b'h' { + return Err(HeaderError::UnsupportedVersion); + } + + let hundred_k_blocksize = buf[3]; + match hundred_k_blocksize { + b'1'..=b'9' => { + let raw_blocksize = hundred_k_blocksize - b'0'; + Self::from_raw_blocksize(raw_blocksize) + } + _ => Err(HeaderError::InvalidBlockSize), + } + } + + /// Construct `Header` from the raw blocksize + /// + /// # Errors + /// + /// Returns [`HeaderError::InvalidBlockSize`] if `raw_blocksize` + /// isn't `1..=9` + pub fn from_raw_blocksize(raw_blocksize: u8) -> Result { + if !(1..=9).contains(&raw_blocksize) { + return Err(HeaderError::InvalidBlockSize); + } + + let max_blocksize = 100 * 1000 * u32::from(raw_blocksize); + Ok(Self { max_blocksize }) + } + + /// The maximum blocksize + /// + /// The returned value is always `100000..=900000` + pub fn max_blocksize(&self) -> u32 { + self.max_blocksize + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn valid_9k() { + let header = Header::parse(*b"BZh9").unwrap(); + assert_eq!(header.max_blocksize(), 900000); + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/huffman.rs b/crates/exav-unpack/src/formats/bzip2_rs/huffman.rs new file mode 100644 index 00000000..dd9cf9c6 --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/huffman.rs @@ -0,0 +1,167 @@ +use tinyvec::ArrayVec; + +pub struct HuffmanTree { + nodes: [HuffmanNode; 258], +} + +#[derive(Default, Copy, Clone)] +struct HuffmanNode { + // [right, left] + right_left: [HuffmanNodeState; 2], +} + +#[derive(Copy, Clone)] +enum HuffmanNodeState { + Next(u16), + Done(u16), +} + +#[derive(Default)] +struct LengthPair { + value: u16, + length: u8, +} + +#[derive(Default)] +struct HuffmanCode { + code: u32, + value: u16, +} + +impl HuffmanTree { + pub fn new(lengths: &[u8]) -> Result { + assert!(lengths.len() >= 2, "too few symbols"); + + let mut pairs = lengths + .iter() + .enumerate() + .map(|(i, &length)| LengthPair { + value: i as u16, + length, + }) + .collect::>(); + + pairs.sort_unstable_by(|a, b| { + a.length + .cmp(&b.length) + .then_with(|| a.value.cmp(&b.value)) + .reverse() + }); + + let mut code = 0u32; + let mut length = 32u8; + + let mut codes = pairs + .into_iter() + .map(|pair| { + length = length.min(pair.length); + + let c = HuffmanCode { + code, + value: pair.value, + }; + + code = code.wrapping_add(1 << (32 - length)); + c + }) + .collect::>(); + + codes.sort_unstable_by_key(|c| c.code); + + let mut this = HuffmanTree::default(); + Self::build_huffman_node(&mut this.nodes[..codes.len()], &mut 0, &codes, 0)?; + Ok(this) + } + + pub fn decode(&self, bits_iter: &mut I) -> Option + where + I: Iterator, + { + let mut node = &self.nodes[0]; + + for bit in bits_iter { + let val = &node.right_left[usize::from(bit)]; + match val { + HuffmanNodeState::Next(node_index) => node = &self.nodes[usize::from(*node_index)], + HuffmanNodeState::Done(value) => return Some(*value), + }; + } + + None + } + + fn build_huffman_node( + nodes: &mut [HuffmanNode], + next_node: &mut usize, + codes: &[HuffmanCode], + level: u32, + ) -> Result { + let test = 1u32.rotate_right(level); + + let first_right_index = codes + .iter() + .enumerate() + .find(|(_, code)| code.code & test != 0) + .map_or(codes.len(), |(i, _)| i); + + let left = &codes[..first_right_index]; + let right = &codes[first_right_index..]; + + if left.is_empty() || right.is_empty() { + if codes.len() < 2 { + return Err("empty huffman tree"); + } + + if level == 31 { + return Err("equal symbols in huffman tree"); + } + + if left.is_empty() { + Self::build_huffman_node(nodes, next_node, right, level + 1) + } else { + Self::build_huffman_node(nodes, next_node, left, level + 1) + } + } else { + let node_index = *next_node; + // remove downstream bounds checking + assert!(node_index < nodes.len()); + *next_node += 1; + + if left.len() == 1 { + let node = &mut nodes[node_index]; + node.right_left[1] = HuffmanNodeState::Done(left[0].value); + } else { + let val = Self::build_huffman_node(nodes, next_node, left, level + 1)?; + + let node = &mut nodes[node_index]; + node.right_left[1] = HuffmanNodeState::Next(val); + } + + if right.len() == 1 { + let node = &mut nodes[node_index]; + node.right_left[0] = HuffmanNodeState::Done(right[0].value); + } else { + let val = Self::build_huffman_node(nodes, next_node, right, level + 1)?; + + let node = &mut nodes[node_index]; + node.right_left[0] = HuffmanNodeState::Next(val); + } + + Ok(node_index as u16) + } + } +} + +impl Default for HuffmanTree { + fn default() -> Self { + Self { + nodes: [HuffmanNode::default(); 258], + } + } +} + +impl Default for HuffmanNodeState { + fn default() -> Self { + HuffmanNodeState::Next(0) + } +} diff --git a/crates/exav-unpack/src/formats/bzip2_rs/mod.rs b/crates/exav-unpack/src/formats/bzip2_rs/mod.rs new file mode 100644 index 00000000..d8af1d78 --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/mod.rs @@ -0,0 +1,22 @@ +//! bzip2 decoder, vendored from `bzip2-rs` 0.1.2 (Copyright Paolo Barbolini, +//! MIT OR Apache-2.0; attributed in `NOTICE`). Pure Rust, no `unsafe`. +//! +//! Changes from upstream: +//! * the streaming decoder buffers enough input for the largest compressed +//! block, which is larger than the block itself when the data does not +//! compress (upstream issue #13, unfixed in any release); +//! * the `rustc_1_37` code paths are taken unconditionally, the `nightly` +//! ones dropped, and lints brought up to this crate's settings; +//! * paths and visibility adapted to a module of this crate. + +pub(crate) use self::decoder::DecoderReader; + +mod bitreader; +mod block; +mod crc; +mod decoder; +mod header; +mod huffman; +mod move_to_front; + +const LEN_258: usize = 512; diff --git a/crates/exav-unpack/src/formats/bzip2_rs/move_to_front.rs b/crates/exav-unpack/src/formats/bzip2_rs/move_to_front.rs new file mode 100644 index 00000000..06972a6c --- /dev/null +++ b/crates/exav-unpack/src/formats/bzip2_rs/move_to_front.rs @@ -0,0 +1,65 @@ +pub struct MoveToFrontDecoder { + symbols: [u8; 256], +} + +impl MoveToFrontDecoder { + pub fn new() -> Self { + let mut this = Self::new_from_symbols([0u8; 256]); + + for (i, symbol) in this.symbols.iter_mut().enumerate() { + *symbol = i as u8; + } + + this + } + + pub fn new_from_symbols(symbols: [u8; 256]) -> Self { + Self { symbols } + } + + pub fn decode(&mut self, n: u8) -> u8 { + let b = self.symbols[usize::from(n)]; + self.symbols.copy_within(..usize::from(n), 1); + self.symbols[0] = b; + + b + } + + pub fn first(&self) -> u8 { + self.symbols[0] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn move_stuff() { + let mut decoder = MoveToFrontDecoder::new(); + + for i in 0..=255 { + assert_eq!(i, usize::from(decoder.symbols[i])); + } + + let decode = decoder.decode(0); + assert_eq!(decode, 0); + + for i in 0..=255 { + assert_eq!(i, usize::from(decoder.symbols[i])); + } + + let decode = decoder.decode(5); + assert_eq!(decode, 5); + + assert_eq!(decoder.symbols[0], 5); + assert_eq!(decoder.symbols[1], 0); + assert_eq!(decoder.symbols[2], 1); + assert_eq!(decoder.symbols[3], 2); + assert_eq!(decoder.symbols[4], 3); + assert_eq!(decoder.symbols[5], 4); + for i in 6..=255 { + assert_eq!(i, usize::from(decoder.symbols[i])); + } + } +} diff --git a/crates/exav-unpack/src/formats/cab.rs b/crates/exav-unpack/src/formats/cab.rs index 3a691c4f..0a77a2e2 100644 --- a/crates/exav-unpack/src/formats/cab.rs +++ b/crates/exav-unpack/src/formats/cab.rs @@ -2,21 +2,30 @@ use crate::*; use std::io::{BufReader, Cursor, Read, Seek, Write}; +/// Walk a cabinet off its source. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + stream_cab(&mut crate::source::Reader::new(src), budget, visit) +} + /// Streaming cab extraction (pattern A): each CFFOLDER is decoded as a /// forward-only [`FolderReader`] and each file is handed the visitor as a -/// `take(uncompressed_size)` window — the decompressed folder is never buffered. -/// Files are emitted folder-by-folder in offset order. +/// `take(uncompressed_size)` window; the decompressed folder is never buffered. +/// Files are emitted folder-by-folder in offset order, and a file that starts +/// before the reader restarts the folder. pub(crate) fn stream_cab( source: &mut R, budget: &mut Budget, - visit: crate::stream::StreamVisit, + visit: crate::stream::Visit, ) -> Result, LimitHit> { use crate::formats::cab_parse::cabinet::Cabinet; use crate::formats::cab_parse::folder::FolderReader; - use crate::stream::{visit_member, MemberMeta}; + use crate::stream::{emit_stream, MemberMeta}; let (folder_metas, files) = Cabinet::layout(source).map_err(|e| LimitHit::new(format!("cab: {e}")))?; - let max_buffer = budget.limits.max_buffer_bytes; for (folder_idx, meta) in folder_metas.iter().enumerate() { let mut folder_files: Vec<&crate::formats::cab_parse::file::FileEntry> = files .iter() @@ -31,7 +40,6 @@ pub(crate) fn stream_cab( meta.first_data_offset, meta.num_data_blocks, meta.compression_type, - max_buffer, ) .map_err(|e| LimitHit::new(format!("cab folder: {e}")))?; let mut pos = 0u64; @@ -45,6 +53,7 @@ pub(crate) fn stream_cab( let m = MemberMeta { name: f.name().to_string(), comp_size: f.uncompressed_size as u64, + size: Some(f.uncompressed_size as u64), encrypted: false, unsupported: Some("CAB folder could not be read up to this member"), }; @@ -53,25 +62,26 @@ pub(crate) fn stream_cab( } continue; } - // Both of these are members the cabinet's own directory names, so - // they exist; this reader just cannot reach them. Reporting is the - // whole difference between "no malware here" and "did not look". + // A member starting before the reader shares bytes with the one + // before it: MSI cabinets list a file installed under two names + // twice at one offset. Decode the folder again from its start. The + // bytes decoded twice are charged to the scan budget, which bounds + // a cabinet built to force a restart per member. if want < pos { - budget.count_entry()?; - let m = MemberMeta { - name: f.name().to_string(), - comp_size: f.uncompressed_size as u64, - encrypted: false, - unsupported: Some( - "CAB member lies before the current position in its folder \ - (this walker reads forward only)", - ), - }; - if let Some(t) = visit(&m, None, budget) { - return Ok(Some(t)); - } - continue; + budget.charge_scan(want)?; + drop(reader); + reader = FolderReader::new( + &mut *source, + meta.first_data_offset, + meta.num_data_blocks, + meta.compression_type, + ) + .map_err(|e| LimitHit::new(format!("cab folder: {e}")))?; + pos = 0; } + // A member the cabinet's own directory names exists; when this + // reader cannot reach it, reporting is the whole difference between + // "no malware here" and "did not look". let (skipped, failed) = skip_forward(&mut reader, want - pos); pos += skipped; stalled = failed; @@ -80,6 +90,7 @@ pub(crate) fn stream_cab( let m = MemberMeta { name: f.name().to_string(), comp_size: f.uncompressed_size as u64, + size: Some(f.uncompressed_size as u64), encrypted: false, unsupported: Some(if failed { "CAB folder could not be read up to this member" @@ -96,12 +107,13 @@ pub(crate) fn stream_cab( let meta_m = MemberMeta { name: f.name().to_string(), comp_size: f.uncompressed_size as u64, + size: Some(f.uncompressed_size as u64), encrypted: false, unsupported: None, }; let r = { let mut window = (&mut reader).take(f.uncompressed_size as u64); - let out = visit_member(&meta_m, &mut window, budget, visit)?; + let out = emit_stream(&meta_m, &mut window, budget, visit)?; // Drain any bytes the visitor left so `pos` advances by the full // size and the next file lands at the right offset. stalled = std::io::copy(&mut window, &mut std::io::sink()).is_err(); @@ -132,82 +144,3 @@ fn skip_forward(r: &mut R, n: u64) -> (u64, bool) { } (skipped, false) } - -pub(crate) fn extract_cab( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let patched = repair_cab_size(data); - let src: &[u8] = patched.as_deref().unwrap_or(data); - let mut cursor = Cursor::new(src); - let cabinet = crate::formats::cab_parse::cabinet::Cabinet::new( - &mut cursor, - budget.limits.max_buffer_bytes, - ) - .map_err(|e| LimitHit::new(format!("cab: {e}")))?; - - for file_entry in cabinet.file_entries() { - budget.count_entry()?; - let cap = budget.reserve()?; - let name = file_entry.name().to_string(); - // By entry, not by name: two members can share a name, and a name lookup - // would hand back the first one's bytes twice and never the second's. - let reader = match cabinet.read_entry(file_entry) { - Ok(r) => r, - Err(_) => { - if let Some(r) = visit( - Entry::unsupported(name, 0, false, "corrupt CAB member"), - budget, - ) { - return Ok(Some(r)); - } - continue; - } - }; - let (buf, failed) = tolerant_read(reader, cap); - budget.commit(buf.len() as u64); - let mut e = Entry::new(name, buf); - if failed { - e.unsupported = Some( - "CAB member data failed to decode part way; the bytes before \ - the failure were scanned", - ); - } - if let Some(r) = visit(e, budget) { - return Ok(Some(r)); - } - } - Ok(None) -} - -/// Up to `cap` bytes, and whether the reader failed before it ended. -fn tolerant_read(mut r: R, cap: u64) -> (Vec, bool) { - let mut buf = Vec::new(); - let mut chunk = [0u8; 8192]; - while (buf.len() as u64) < cap { - match r.read(&mut chunk) { - Ok(0) => break, - Ok(n) => { - let room = (cap - buf.len() as u64) as usize; - buf.extend_from_slice(&chunk[..n.min(room)]); - } - Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} - Err(_) => return (buf, true), - } - } - (buf, false) -} - -fn repair_cab_size(data: &[u8]) -> Option> { - if data.len() < 12 || &data[0..4] != b"MSCF" { - return None; - } - let declared = u32::from_le_bytes([data[8], data[9], data[10], data[11]]) as u64; - if declared <= data.len() as u64 { - return None; - } - let mut out = data.to_vec(); - out[8..12].copy_from_slice(&(data.len() as u32).to_le_bytes()); - Some(out) -} diff --git a/crates/exav-unpack/src/formats/cab_parse/cabinet.rs b/crates/exav-unpack/src/formats/cab_parse/cabinet.rs index 6dccf5a1..b634cb06 100644 --- a/crates/exav-unpack/src/formats/cab_parse/cabinet.rs +++ b/crates/exav-unpack/src/formats/cab_parse/cabinet.rs @@ -5,16 +5,11 @@ use byteorder::{LittleEndian, ReadBytesExt}; use crate::formats::cab_parse::consts; use crate::formats::cab_parse::ctype::CompressionType; use crate::formats::cab_parse::file::FileEntry; -use crate::formats::cab_parse::folder::Folder; use crate::formats::cab_parse::string::read_null_terminated_string; -pub(crate) struct Cabinet { - file_entries: Vec, - decompressed_data: Vec, -} +pub(crate) struct Cabinet; -/// A CFFOLDER's location and codec, parsed without decompressing its data — used -/// by both the buffered [`Cabinet::new`] and the streaming extractor. +/// A CFFOLDER's location and codec, parsed without decompressing its data. pub(crate) struct FolderMeta { pub first_data_offset: u32, pub num_data_blocks: u16, @@ -23,9 +18,8 @@ pub(crate) struct FolderMeta { impl Cabinet { /// Parse the CFHEADER, CFFOLDER table, and CFFILE table **without** - /// decompressing any folder data. The streaming extractor uses this to walk - /// files and decode each folder lazily (via [`super::folder::FolderReader`]); - /// `new` uses it and then decompresses. + /// decompressing any folder data. The extractor walks the files and decodes + /// each folder lazily (via [`super::folder::FolderReader`]). pub(crate) fn layout( reader: &mut R, ) -> io::Result<(Vec, Vec)> { @@ -115,87 +109,4 @@ impl Cabinet { } Ok((folder_metas, file_entries)) } - - pub(crate) fn new(reader: &mut R, max_buffer: u64) -> io::Result { - let (folder_metas, mut file_entries) = Self::layout(reader)?; - - // === Decompress each folder === - let mut folders: Vec = Vec::with_capacity(folder_metas.len()); - for m in &folder_metas { - folders.push(Folder::new( - reader, - m.first_data_offset, - m.num_data_blocks, - m.compression_type, - max_buffer, - )?); - } - - // === Build combined decompressed buffer === - let mut decompressed_data = Vec::new(); - let mut folder_decomp_offsets: Vec = Vec::new(); - for folder in &folders { - folder_decomp_offsets.push(decompressed_data.len() as u32); - decompressed_data.extend_from_slice(folder.decompressed_data()); - // The combined buffer across all folders must also stay within the - // global peak-buffer limit. - if decompressed_data.len() as u64 > max_buffer { - invalid_data!("Cabinet decompressed size exceeds max-buffer"); - } - } - - // Map file entries to absolute offsets in the combined buffer - for fe in &mut file_entries { - let base = folder_decomp_offsets - .get(fe.folder_index as usize) - .copied() - .unwrap_or(0); - fe.data_offset += base; - } - - Ok(Cabinet { - file_entries, - decompressed_data, - }) - } - - pub(crate) fn file_entries(&self) -> &[FileEntry] { - &self.file_entries - } - - /// Read the member at a specific entry. - /// - /// A cabinet may hold two members under one name. Looking a member up by - /// name then yields the first one's bytes twice and never the second's, so a - /// walk over [`Self::file_entries`] reads each entry directly. - pub(crate) fn read_entry(&self, file_entry: &FileEntry) -> io::Result> { - let start = file_entry.data_offset as usize; - let end = start + file_entry.uncompressed_size as usize; - if end > self.decompressed_data.len() { - invalid_data!( - "File extends past end of decompressed data (offset {}, size {})", - start, - file_entry.uncompressed_size - ); - } - Ok(FileReader { - data: &self.decompressed_data[start..end], - pos: 0, - }) - } -} - -pub(crate) struct FileReader<'a> { - data: &'a [u8], - pos: usize, -} - -impl<'a> Read for FileReader<'a> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let remaining = &self.data[self.pos..]; - let n = remaining.len().min(buf.len()); - buf[..n].copy_from_slice(&remaining[..n]); - self.pos += n; - Ok(n) - } } diff --git a/crates/exav-unpack/src/formats/cab_parse/folder.rs b/crates/exav-unpack/src/formats/cab_parse/folder.rs index f4d4ed83..3627c4c2 100644 --- a/crates/exav-unpack/src/formats/cab_parse/folder.rs +++ b/crates/exav-unpack/src/formats/cab_parse/folder.rs @@ -4,24 +4,17 @@ use byteorder::{LittleEndian, ReadBytesExt}; use crate::formats::cab_parse::ctype::{CompressionType, Decompressor}; -pub(crate) struct Folder { - decompressed_data: Vec, -} - /// A cab folder decoded as a **forward-only `Read`**: CFDATA blocks are read and /// decompressed one at a time — the LZX/MSZIP decompressor retains only its -/// window/dictionary — so the whole decompressed folder is never buffered. Used -/// by the streaming extractor to hand each file a `take(size)` window. Mirrors -/// [`Folder::new`]'s block loop; total output is bounded by `max_buffer` for -/// parity with the buffered path. +/// window/dictionary — so the whole decompressed folder is never buffered. The +/// extractor hands each file a `take(size)` window of it; what is read is +/// bounded by the scan budget, as any streamed member is. pub(crate) struct FolderReader<'a, R: Read + Seek> { reader: &'a mut R, decompressor: Decompressor, blocks_left: u16, out: Vec, out_pos: usize, - max_buffer: u64, - produced: u64, } impl<'a, R: Read + Seek> FolderReader<'a, R> { @@ -30,7 +23,6 @@ impl<'a, R: Read + Seek> FolderReader<'a, R> { first_data_offset: u32, num_data_blocks: u16, compression_type: CompressionType, - max_buffer: u64, ) -> io::Result { let decompressor = compression_type.into_decompressor()?; reader.seek(SeekFrom::Start(first_data_offset as u64))?; @@ -40,8 +32,6 @@ impl<'a, R: Read + Seek> FolderReader<'a, R> { blocks_left: num_data_blocks, out: Vec::new(), out_pos: 0, - max_buffer, - produced: 0, }) } @@ -56,17 +46,9 @@ impl<'a, R: Read + Seek> FolderReader<'a, R> { let uncompressed_size = self.reader.read_u16::()? as usize; let mut compressed = vec![0u8; compressed_size]; self.reader.read_exact(&mut compressed)?; - let decompressed = self + self.out = self .decompressor .decompress(compressed, uncompressed_size)?; - self.produced = self.produced.saturating_add(decompressed.len() as u64); - if self.produced > self.max_buffer { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "cab folder exceeds max-buffer", - )); - } - self.out = decompressed; self.out_pos = 0; self.blocks_left -= 1; Ok(true) @@ -86,46 +68,3 @@ impl Read for FolderReader<'_, R> { Ok(n) } } - -impl Folder { - pub(crate) fn new( - reader: &mut R, - first_data_offset: u32, - num_data_blocks: u16, - compression_type: CompressionType, - max_buffer: u64, - ) -> io::Result { - let mut decompressor = compression_type.into_decompressor()?; - let mut decompressed_data = Vec::new(); - - if num_data_blocks == 0 { - return Ok(Folder { decompressed_data }); - } - - reader.seek(SeekFrom::Start(first_data_offset as u64))?; - - for _ in 0..num_data_blocks { - let _checksum = reader.read_u32::()?; - let compressed_size = reader.read_u16::()? as usize; - let uncompressed_size = reader.read_u16::()? as usize; - let mut compressed_data = vec![0u8; compressed_size]; - reader.read_exact(&mut compressed_data)?; - let decompressed = decompressor.decompress(compressed_data, uncompressed_size)?; - decompressed_data.extend_from_slice(&decompressed); - // Bound the decompressed folder by the global peak-buffer limit: a - // CFDATA run can amplify far beyond the compressed input. - if decompressed_data.len() as u64 > max_buffer { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "cab folder exceeds max-buffer", - )); - } - } - - Ok(Folder { decompressed_data }) - } - - pub(crate) fn decompressed_data(&self) -> &[u8] { - &self.decompressed_data - } -} diff --git a/crates/exav-unpack/src/formats/cpio.rs b/crates/exav-unpack/src/formats/cpio.rs index 2e5ad8ca..584f051d 100644 --- a/crates/exav-unpack/src/formats/cpio.rs +++ b/crates/exav-unpack/src/formats/cpio.rs @@ -24,6 +24,17 @@ fn read_name(source: &mut R, off: u64, namesize: u64) -> Result< .to_string()) } +/// Walk a cpio archive, each member streamed from where it lies. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let mut source = crate::source::Reader::new(src); + let members = stream_offsets(&mut source)?; + crate::stream::stream_stored(&mut source, budget, visit, members) +} + /// Parse member offsets from a seekable source (reader-based streaming): walk the /// cpio headers (newc/odc/old-binary) and return each file member as /// `(name, data_offset, size)`. Mirrors [`extract_cpio`]'s boundaries but never @@ -133,186 +144,6 @@ fn oct(field: &[u8]) -> usize { usize::from_str_radix(std::str::from_utf8(field).unwrap_or("0").trim(), 8).unwrap_or(0) } -fn align4(n: usize) -> usize { - (n + 3) & !3 -} - -pub(crate) fn extract_cpio( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - if data.starts_with(b"070701") || data.starts_with(b"070702") { - extract_newc(data, budget, visit) - } else if data.starts_with(b"070707") { - extract_odc(data, budget, visit) - } else if data.starts_with(&[0xc7, 0x71]) { - extract_bin(data, budget, false, visit) // little-endian (0o070707 LE) - } else if data.starts_with(&[0x71, 0xc7]) { - extract_bin(data, budget, true, visit) // byte-swapped (big-endian writer) - } else { - Err(LimitHit::new("cpio: unknown format".to_string())) - } -} - -/// Old binary format: 26-byte header of 16-bit fields; 32-bit fields (mtime, -/// filesize) are stored most-significant-half first. Name and data are each -/// padded to an even byte boundary. -fn extract_bin( - data: &[u8], - budget: &mut Budget, - swap: bool, - visit: Sink, -) -> Result, LimitHit> { - let u16at = |p: usize, d: &[u8]| -> usize { - let v = u16::from_le_bytes([d[p], d[p + 1]]); - (if swap { v.swap_bytes() } else { v }) as usize - }; - let mut pos = 0usize; - while pos + 26 <= data.len() { - let magic = u16at(pos, data); - if magic != 0o070707 { - break; - } - let namesize = u16at(pos + 20, data); - let filesize = (u16at(pos + 22, data) << 16) | u16at(pos + 24, data); - let name_start = pos + 26; - let name_end = name_start.saturating_add(namesize).min(data.len()); - let name = String::from_utf8_lossy(&data[name_start..name_end]) - .trim_end_matches('\0') - .to_string(); - if name == TRAILER { - break; - } - // Clamp to EOF: `namesize` is attacker-controlled, so an oversized value - // must not let `data_start` slice past the buffer (start > end panics). - let data_start = name_start - .saturating_add(namesize) - .saturating_add(namesize & 1) // pad name to even - .min(data.len()); - let data_end = data_start.saturating_add(filesize).min(data.len()); - if filesize > 0 { - budget.count_entry()?; - // The clamp above keeps the slice in bounds. A member whose declared - // size runs past EOF is simply truncated: every byte that EXISTS is - // scanned, and the missing tail is absent rather than hidden — so a - // clean result here is a real clean. exav scans for malware, it is - // not a file-integrity validator. See docs/QUIRKS.md. - let cap = budget.reserve()?; - if filesize as u64 > cap { - return Err(LimitHit::new(format!( - "cpio member '{name}' exceeds budget" - ))); - } - let member = data[data_start..data_end].to_vec(); - budget.commit(member.len() as u64); - if let Some(r) = visit(Entry::new(name, member), budget) { - return Ok(Some(r)); - } - } - pos = data_end + (filesize & 1); // pad data to even - } - Ok(None) -} - -/// SVR4 "new ASCII": 110-byte header of 8-hex-digit fields, then name -/// (`namesize` bytes) and data (`filesize` bytes), each padded so the header+name -/// and the data each end on a 4-byte boundary. -fn extract_newc( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let mut pos = 0usize; - while pos + 110 <= data.len() { - let h = &data[pos..pos + 110]; - if &h[0..6] != b"070701" && &h[0..6] != b"070702" { - break; - } - let filesize = hex(&h[54..62]); - let namesize = hex(&h[94..102]); - let name_start = pos + 110; - let name_end = name_start.saturating_add(namesize).min(data.len()); - let name = String::from_utf8_lossy(&data[name_start..name_end]) - .trim_end_matches('\0') - .to_string(); - if name == TRAILER { - break; - } - // Clamp to EOF: an oversized `namesize` must not push `data_start` past - // the buffer (start > end panics). - let data_start = align4(name_start.saturating_add(namesize)).min(data.len()); - let data_end = data_start.saturating_add(filesize).min(data.len()); - if filesize > 0 { - budget.count_entry()?; - // The clamp above keeps the slice in bounds. A member whose declared - // size runs past EOF is simply truncated: every byte that EXISTS is - // scanned, and the missing tail is absent rather than hidden — so a - // clean result here is a real clean. exav scans for malware, it is - // not a file-integrity validator. See docs/QUIRKS.md. - let cap = budget.reserve()?; - if filesize as u64 > cap { - return Err(LimitHit::new(format!( - "cpio member '{name}' exceeds budget" - ))); - } - let member = data[data_start..data_end].to_vec(); - budget.commit(member.len() as u64); - if let Some(r) = visit(Entry::new(name, member), budget) { - return Ok(Some(r)); - } - } - pos = align4(data_end); - } - Ok(None) -} - -/// Old POSIX "portable": 76-byte header of octal fields, no padding. -fn extract_odc(data: &[u8], budget: &mut Budget, visit: Sink) -> Result, LimitHit> { - let mut pos = 0usize; - while pos + 76 <= data.len() { - let h = &data[pos..pos + 76]; - if &h[0..6] != b"070707" { - break; - } - let namesize = oct(&h[59..65]); - let filesize = oct(&h[65..76]); - let name_start = pos + 76; - let name_end = name_start.saturating_add(namesize).min(data.len()); - let name = String::from_utf8_lossy(&data[name_start..name_end]) - .trim_end_matches('\0') - .to_string(); - if name == TRAILER { - break; - } - // Clamp to EOF: an oversized `namesize` must not push `data_start` past - // the buffer (start > end panics). - let data_start = name_start.saturating_add(namesize).min(data.len()); - let data_end = data_start.saturating_add(filesize).min(data.len()); - if filesize > 0 { - budget.count_entry()?; - // The clamp above keeps the slice in bounds. A member whose declared - // size runs past EOF is simply truncated: every byte that EXISTS is - // scanned, and the missing tail is absent rather than hidden — so a - // clean result here is a real clean. exav scans for malware, it is - // not a file-integrity validator. See docs/QUIRKS.md. - let cap = budget.reserve()?; - if filesize as u64 > cap { - return Err(LimitHit::new(format!( - "cpio member '{name}' exceeds budget" - ))); - } - let member = data[data_start..data_end].to_vec(); - budget.commit(member.len() as u64); - if let Some(r) = visit(Entry::new(name, member), budget) { - return Ok(Some(r)); - } - } - pos = data_end; - } - Ok(None) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/exav-unpack/src/formats/dmg.rs b/crates/exav-unpack/src/formats/dmg.rs index 50adb490..ccbf9e6c 100644 --- a/crates/exav-unpack/src/formats/dmg.rs +++ b/crates/exav-unpack/src/formats/dmg.rs @@ -2,13 +2,14 @@ not(feature = "decrypt"), allow(dead_code, unused_mut, unused_imports, unreachable_code) )] +use crate::source::{ByteSource, Reader}; use crate::*; -use std::io::Cursor; +use std::io::{Cursor, Read, Seek, SeekFrom}; -/// "koly" — the UDIF trailer signature (last 512 bytes of the file). +/// "koly": the UDIF trailer signature (last 512 bytes of the file). const KOLY_SIG: &[u8; 4] = b"koly"; -/// "encrcdsa" — encrypted DMG header signature. +/// "encrcdsa": encrypted DMG header signature. const ENCRCDSA_SIG: &[u8; 8] = b"encrcdsa"; /// HFS+ volume header signature (big-endian 0x482B). @@ -19,11 +20,12 @@ const APFS_SIG: &[u8; 4] = b"NXSB"; // ─── Detection ─────────────────────────────────────────────────────────────── -pub(crate) fn is_dmg(data: &[u8]) -> bool { - if data.is_empty() { +pub(crate) fn is_dmg(p: &Probe) -> bool { + let data = p.head; + if p.len == 0 { return false; } - if data.len() >= 512 && &data[data.len() - 512..data.len() - 508] == KOLY_SIG { + if p.len >= 512 && p.window(p.len - 512, 4)[..] == KOLY_SIG[..] { return true; } if data.len() >= 8 && &data[0..8] == ENCRCDSA_SIG { @@ -41,7 +43,7 @@ const HFSX_SIG: [u8; 2] = [0x48, 0x58]; /// /// The signature alone is **two bytes**. Scanning 64 KiB at 16-byte steps gives /// about four thousand chances for it to appear by accident, so on arbitrary -/// data — a compressed archive, say — it hits perhaps one time in twenty. That +/// data (a compressed archive, say) it hits perhaps one time in twenty. That /// is not a theoretical worry: it costs the file its real format, because /// whatever it actually was is never tried once `detect` has answered `Dmg`. /// @@ -74,7 +76,7 @@ fn find_hfs_offset(data: &[u8]) -> Option { /// The APFS container superblock's `nx_block_size`, which corroborates the /// four-byte signature the same way the HFS+ check does. Four bytes is a much /// stronger start than two, but the object header before it is free to check. -/// `off` is where `NXSB` itself sits, which is 32 bytes into the superblock — +/// `off` is where `NXSB` itself sits, which is 32 bytes into the superblock: /// the object header (checksum, oid, xid, type, subtype) comes first. /// `nx_block_size` is the field immediately after the magic. fn plausible_apfs_header(data: &[u8], off: usize) -> bool { @@ -95,11 +97,6 @@ fn find_apfs_offset(data: &[u8]) -> Option { .find(|&off| plausible_apfs_header(data, off)) } -#[cfg(feature = "decrypt")] -fn is_encrypted(data: &[u8]) -> bool { - data.len() >= 8 && &data[0..8] == ENCRCDSA_SIG -} - // ─── Encrypted DMG header parsing ──────────────────────────────────────────── /// Apple encrypted DMG header, matching dmgwiz's `EncryptedDmgHeader`. @@ -189,7 +186,7 @@ fn decrypt_keyblob( header: &EncryptedDmgHeader, derived_key: &[u8; 24], ) -> Result, LimitHit> { - use cbc::cipher::{block_padding::NoPadding, BlockDecryptMut, KeyIvInit}; + use cbc::cipher::{block_padding::NoPadding, BlockModeDecrypt, KeyIvInit}; let iv_len = header.blob_enc_iv_size as usize; let mut iv = [0u8; 8]; @@ -213,7 +210,7 @@ fn decrypt_keyblob( let decryptor = TdesCbc::new_from_slices(derived_key, &iv) .map_err(|_| LimitHit::corrupt("invalid 3DES key/iv length".into()))?; let plaintext = decryptor - .decrypt_padded_mut::(&mut buf) + .decrypt_padded::(&mut buf) .map_err(|_| { LimitHit::corrupt("3DES keyblob decryption failed (wrong password?)".into()) })?; @@ -223,7 +220,7 @@ fn decrypt_keyblob( /// Compute per-chunk IV using HMAC-SHA1(hmac_key, chunk_no_be32)[0..16]. #[cfg(feature = "decrypt")] fn compute_chunk_iv(hmac_key: &[u8], chunk_no: u32) -> [u8; 16] { - use hmac::{Hmac, Mac}; + use hmac::{Hmac, KeyInit, Mac}; type HmacSha1 = Hmac; let mut mac = HmacSha1::new_from_slice(hmac_key).expect("HMAC accepts any key size"); @@ -301,14 +298,14 @@ fn try_decrypt_dmg(data: &[u8], password: &str) -> Result, LimitHit> { let src_len = src_end - src_start; chunk_buf[..src_len].copy_from_slice(&data[src_start..src_end]); if src_len < chunk_size { - // Last chunk may be short — pad with zeros for decryption. + // Last chunk may be short; pad with zeros for decryption. chunk_buf[src_len..chunk_size].fill(0); } let iv = compute_chunk_iv(hmac_key, chunk_no as u32); // AES-CBC decrypt. - use cbc::cipher::{block_padding::NoPadding, BlockDecryptMut, KeyIvInit}; + use cbc::cipher::{block_padding::NoPadding, BlockModeDecrypt, KeyIvInit}; type Aes128Cbc = cbc::Decryptor; type Aes256Cbc = cbc::Decryptor; @@ -319,7 +316,7 @@ fn try_decrypt_dmg(data: &[u8], password: &str) -> Result, LimitHit> { let decryptor = Aes128Cbc::new_from_slices(aes_key, &iv) .map_err(|_| LimitHit::corrupt("invalid AES-128 key/iv".into()))?; let mut buf = chunk_buf[..chunk_size].to_vec(); - match decryptor.decrypt_padded_mut::(&mut buf) { + match decryptor.decrypt_padded::(&mut buf) { Ok(pt) => plaintext.extend_from_slice(&pt[..to_write]), Err(_) => { return Err(LimitHit::corrupt( @@ -331,7 +328,7 @@ fn try_decrypt_dmg(data: &[u8], password: &str) -> Result, LimitHit> { let decryptor = Aes256Cbc::new_from_slices(aes_key, &iv) .map_err(|_| LimitHit::corrupt("invalid AES-256 key/iv".into()))?; let mut buf = chunk_buf[..chunk_size].to_vec(); - match decryptor.decrypt_padded_mut::(&mut buf) { + match decryptor.decrypt_padded::(&mut buf) { Ok(pt) => plaintext.extend_from_slice(&pt[..to_write]), Err(_) => { return Err(LimitHit::corrupt( @@ -347,184 +344,214 @@ fn try_decrypt_dmg(data: &[u8], password: &str) -> Result, LimitHit> { // ─── Extraction ────────────────────────────────────────────────────────────── -pub(crate) fn extract_dmg( - data: &[u8], +/// Walk a DMG off its source, each file handed to `visit`. The disk image is +/// decompressed a run at a time as the filesystem reaches it, never whole. +pub(crate) fn walk( + src: &dyn ByteSource, budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { + visit: crate::stream::Visit, +) -> Result, LimitHit> { budget.count_entry()?; + let mut emit = + |entry: Entry, budget: &mut Budget| crate::stream::emit_entry(entry, budget, &mut *visit); + if src.window(0, 8)[..] != ENCRCDSA_SIG[..] { + return walk_disk(Reader::new(src), budget, &mut emit); + } + match decrypt_image(src, budget)? { + Ok(disk) => walk_disk(Cursor::new(disk), budget, &mut emit), + Err(reason) => emit( + Entry::unsupported("encrypted.dmg".to_string(), src.len() as u64, true, reason), + budget, + ), + } +} - // Encrypted DMG — try each password from the budget. +type Emit<'a, T> = &'a mut dyn FnMut(Entry, &mut Budget) -> Result, LimitHit>; + +/// The decrypted image of an encrypted DMG, or why it was not decrypted. +/// Decryption takes the whole image. +fn decrypt_image( + src: &dyn ByteSource, + budget: &Budget, +) -> Result, &'static str>, LimitHit> { #[cfg(not(feature = "decrypt"))] - if data.len() >= 8 && &data[0..8] == ENCRCDSA_SIG { - let entry = Entry::unsupported( - "encrypted.dmg".to_string(), - data.len() as u64, - true, - "encrypted DMG (decrypt feature disabled)", - ); - return Ok(visit(entry, budget)); + { + let _ = (src, budget); + Ok(Err("encrypted DMG (decrypt feature disabled)")) } #[cfg(feature = "decrypt")] - if is_encrypted(data) { + { if budget.passwords.is_empty() { - let entry = Entry::unsupported( - "encrypted.dmg".to_string(), - data.len() as u64, - true, - "encrypted DMG (no password provided)", - ); - if let Some(r) = visit(entry, budget) { - return Ok(Some(r)); + return Ok(Err("encrypted DMG (no password provided)")); + } + let data = crate::stream::read_whole(Format::Dmg, src, budget)?; + for pw in &budget.passwords { + let Ok(decrypted) = try_decrypt_dmg(&data, pw) else { + continue; + }; + // A right password gives a koly trailer or a filesystem. + let has_koly = decrypted.len() >= 512 + && &decrypted[decrypted.len() - 512..decrypted.len() - 508] == KOLY_SIG; + let has_fs = + find_hfs_offset(&decrypted).is_some() || find_apfs_offset(&decrypted).is_some(); + if has_koly || has_fs { + return Ok(Ok(decrypted)); } - return Ok(None); } + // The reason is `&'static str`, one of a fixed set rather than a + // formatted string: leaking a fresh allocation per attempt would let + // anyone grow a long-running daemon without bound by resubmitting + // encrypted images. Which decryption step objected does not change what + // the operator does about it: supply the password. + Ok(Err("encrypted DMG (wrong password, decryption failed)")) + } +} - let mut last_err = None; - for pw in &budget.passwords { - match try_decrypt_dmg(data, pw) { - Ok(decrypted) => { - // Validate: decrypted payload must contain a koly trailer or filesystem. - let has_koly = decrypted.len() >= 512 - && &decrypted[decrypted.len() - 512..decrypted.len() - 508] == KOLY_SIG; - let has_fs = find_hfs_offset(&decrypted).is_some() - || find_apfs_offset(&decrypted).is_some(); - if !has_koly && !has_fs { - last_err = Some(LimitHit::corrupt( - "wrong password (no koly/filesystem in decrypted data)".into(), - )); - continue; - } - let raw = decompress_udif(&decrypted, budget.limits.max_buffer_bytes)?; - return extract_from_raw(&raw, budget, visit); - } - Err(e) => { - last_err = Some(e); - } +/// Walk the filesystem on a (possibly UDIF-compressed) disk image. +fn walk_disk( + disk: D, + budget: &mut Budget, + emit: Emit, +) -> Result, LimitHit> { + let mut disk = super::udif::disk(disk)?; + // Both filesystems are looked for in the first 64 KiB, as `is_dmg` does. + let mut head = Vec::new(); + disk.seek(SeekFrom::Start(0)) + .and_then(|_| (&mut disk).take(64 * 1024 + 44).read_to_end(&mut head)) + .map_err(|e| LimitHit::corrupt(format!("UDIF decompress: {e}")))?; + + if let Some(nxsb_off) = find_apfs_offset(&head) { + let part = Part::new(disk, nxsb_off.saturating_sub(32) as u64)?; + let mut vol = apfs::ApfsVolume::open(part) + .map_err(|e| LimitHit::corrupt(format!("apfs open: {e}")))?; + let walk = vol + .walk() + .map_err(|e| LimitHit::corrupt(format!("apfs walk: {e}")))?; + for we in walk { + if we.entry.kind != apfs::EntryKind::File { + continue; + } + let hit = read_file(we.path, budget, emit, |path, out| { + vol.read_file_to(path, out).map(|_| ()) + })?; + if hit.is_some() { + return Ok(hit); } } + return Ok(None); + } - // All passwords failed. The reason is `&'static str`, so it is one of a - // fixed set rather than a formatted string: leaking a fresh allocation - // per attempt would let anyone grow a long-running daemon without bound - // by resubmitting encrypted images. Which decryption step objected does - // not change what the operator does about it — supply the password. - let reason = if last_err.is_some() { - "encrypted DMG (wrong password, decryption failed)" - } else { - "encrypted DMG (wrong password)" - }; - let entry = - Entry::unsupported("encrypted.dmg".to_string(), data.len() as u64, true, reason); - if let Some(r) = visit(entry, budget) { - return Ok(Some(r)); + if let Some(hfs_off) = find_hfs_offset(&head) { + let part = Part::new(disk, hfs_off.saturating_sub(1024) as u64)?; + let mut vol = hfsplus::HfsVolume::open(part) + .map_err(|e| LimitHit::corrupt(format!("hfs+ open: {e}")))?; + let walk = vol + .walk() + .map_err(|e| LimitHit::corrupt(format!("hfs+ walk: {e}")))?; + for we in walk { + if we.entry.kind != hfsplus::EntryKind::File { + continue; + } + let hit = read_file(we.path, budget, emit, |path, out| { + vol.read_file_to(path, out).map(|_| ()) + })?; + if hit.is_some() { + return Ok(hit); + } } return Ok(None); } - let raw = decompress_udif(data, budget.limits.max_buffer_bytes)?; - extract_from_raw(&raw, budget, visit) + Err(LimitHit::corrupt( + "no HFS+ or APFS filesystem found in DMG".into(), + )) } -/// Extract filesystem from a raw (already decrypted/decompressed) disk image. -fn extract_from_raw( - raw: &[u8], +/// Read one file, bounded by the peak-buffer limit, and emit it. The +/// filesystem crates hand a file over whole. +fn read_file( + path: String, budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - if let Some(nxsb_off) = find_apfs_offset(raw) { - let partition_start = nxsb_off.saturating_sub(32); - return extract_apfs(raw, partition_start, budget, visit); + emit: Emit, + read: impl FnOnce(&str, &mut Capped) -> Result<(), E>, +) -> Result, LimitHit> { + budget.count_entry()?; + let mut out = Capped { + data: Vec::new(), + cap: budget.reserve()?, + over: false, + }; + let read = read(&path, &mut out); + if out.over { + return Err(LimitHit::new(format!("DMG file '{path}' exceeds budget"))); } + if read.is_err() { + // Its bytes are in the image and were not read: reported, not dropped. + return emit( + Entry::unsupported(path, 0, false, "DMG file could not be read"), + budget, + ); + } + budget.commit(out.data.len() as u64); + emit(Entry::new(path, out.data), budget) +} + +/// A `Write` that refuses to grow past `cap`. +struct Capped { + data: Vec, + cap: u64, + over: bool, +} - if let Some(hfs_off) = find_hfs_offset(raw) { - let partition_start = hfs_off.saturating_sub(1024); - return extract_hfs_with_crate(raw, partition_start, budget, visit); +impl std::io::Write for Capped { + fn write(&mut self, b: &[u8]) -> std::io::Result { + if (self.data.len() + b.len()) as u64 > self.cap { + self.over = true; + return Err(std::io::Error::other("over the buffer limit")); + } + self.data.extend_from_slice(b); + Ok(b.len()) } - Err(LimitHit::corrupt( - "no HFS+ or APFS filesystem found in DMG".into(), - )) + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } } -fn decompress_udif(data: &[u8], max_buffer: u64) -> Result, LimitHit> { - super::udif::decompress_udif(data, max_buffer) +/// The disk from `start` on, where a filesystem begins. +struct Part { + disk: D, + start: u64, } -fn extract_hfs_with_crate( - data: &[u8], - partition_start: usize, - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let reader = &data[partition_start..]; - let mut vol = hfsplus::HfsVolume::open(Cursor::new(reader)) - .map_err(|e| LimitHit::corrupt(format!("hfs+ open: {e}")))?; - - let walk = vol - .walk() - .map_err(|e| LimitHit::corrupt(format!("hfs+ walk: {e}")))?; - - for we in walk { - if we.entry.kind != hfsplus::EntryKind::File { - continue; - } - if let Ok(file_data) = vol.read_file(&we.path) { - budget.count_entry()?; - // The FS crate returns each file whole; bound it by the peak-buffer - // limit (a member larger than the cap is reported, not buffered). - let cap = budget.reserve()?; - if file_data.len() as u64 > cap { - return Err(LimitHit::new(format!( - "hfs+ file '{}' exceeds budget", - we.path - ))); - } - budget.commit(file_data.len() as u64); - let entry = Entry::new(we.path, file_data); - if let Some(r) = visit(entry, budget) { - return Ok(Some(r)); - } - } +impl Part { + fn new(mut disk: D, start: u64) -> Result { + disk.seek(SeekFrom::Start(start)) + .map_err(|e| LimitHit::corrupt(format!("DMG: {e}")))?; + Ok(Part { disk, start }) } - Ok(None) } -fn extract_apfs( - data: &[u8], - partition_start: usize, - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let reader = &data[partition_start..]; - let mut vol = apfs::ApfsVolume::open(Cursor::new(reader)) - .map_err(|e| LimitHit::corrupt(format!("apfs open: {e}")))?; - - let walk = vol - .walk() - .map_err(|e| LimitHit::corrupt(format!("apfs walk: {e}")))?; - - for we in walk { - if we.entry.kind != apfs::EntryKind::File { - continue; - } - if let Ok(file_data) = vol.read_file(&we.path) { - budget.count_entry()?; - let cap = budget.reserve()?; - if file_data.len() as u64 > cap { - return Err(LimitHit::new(format!( - "apfs file '{}' exceeds budget", - we.path - ))); - } - budget.commit(file_data.len() as u64); - let entry = Entry::new(we.path, file_data); - if let Some(r) = visit(entry, budget) { - return Ok(Some(r)); - } - } +impl Read for Part { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.disk.read(buf) + } +} + +impl Seek for Part { + fn seek(&mut self, to: SeekFrom) -> std::io::Result { + let to = match to { + SeekFrom::Start(p) => SeekFrom::Start(self.start.saturating_add(p)), + other => other, + }; + let at = self.disk.seek(to)?; + at.checked_sub(self.start).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "seek before the start of the filesystem", + ) + }) } - Ok(None) } // ─── Tests ─────────────────────────────────────────────────────────────────── @@ -533,6 +560,10 @@ fn extract_apfs( mod tests { use super::*; + fn is_dmg(data: &[u8]) -> bool { + super::is_dmg(&Probe::whole(data)) + } + #[test] fn detect_koly_trailer() { let mut buf = vec![0u8; 1024]; @@ -585,7 +616,7 @@ mod tests { #[test] fn a_bare_signature_without_a_plausible_header_is_not_a_dmg() { // `H+` is two bytes. Across 64 KiB scanned at 16-byte steps it turns up - // by chance in roughly one arbitrary buffer in twenty — and a false hit + // by chance in roughly one arbitrary buffer in twenty, and a false hit // is not a wasted check, it is a stolen file: `detect` answers `Dmg`, so // whatever the buffer really was never gets tried. // @@ -623,4 +654,24 @@ mod tests { fn no_false_positive_too_short() { assert!(!is_dmg(&[0u8; 4])); } + + /// The disk is read as the filesystem needs it, so one larger than the + /// buffer limit still gives up its files. + #[test] + fn a_disk_over_the_buffer_limit_is_walked() { + for name in ["hfs_plus_udzo.dmg", "apfs_udrw.dmg"] { + let path = format!("{}/tests/fixtures/dmg/{name}", env!("CARGO_MANIFEST_DIR")); + let blob = std::fs::read(path).unwrap(); + let mut budget = Budget::new(Limits { + max_buffer_bytes: 64 * 1024, + ..Limits::default() + }); + let files: Vec<(String, Vec)> = crate::extract(Format::Dmg, &blob, &mut budget) + .unwrap_or_else(|e| panic!("{name}: {e:?}")) + .into_iter() + .map(|e| (e.name, e.data)) + .collect(); + assert_eq!(files, [("/test.txt".to_string(), b"hello hfs+\n".to_vec())]); + } + } } diff --git a/crates/exav-unpack/src/formats/egg.rs b/crates/exav-unpack/src/formats/egg.rs index 03c753ea..41fdea11 100644 --- a/crates/exav-unpack/src/formats/egg.rs +++ b/crates/exav-unpack/src/formats/egg.rs @@ -13,8 +13,9 @@ //! The archive is a chain of self-describing records. Each extension field is //! `magic(4) | flags(1) | size(2 or 4) | data`, and a run of them ends at a stop //! marker — so an unknown field is skipped by length rather than guessed at, and -//! a future EGG version stays readable. AZO is ESTsoft's own algorithm and is -//! not documented; blocks using it are reported. +//! a future EGG version stays readable. AZO is ESTsoft's own undocumented +//! algorithm, decoded by a port of the one permissively licensed implementation +//! (`formats::azo`). use crate::{Budget, Entry, Format, LimitHit, Sink}; @@ -222,9 +223,9 @@ fn decode_block(algo: u8, raw: &[u8], uncomp: u64, cap: u64) -> Option<(Vec, DEFLATE => { crate::bounded_read(flate2::read::DeflateDecoder::new(Cursor::new(raw)), cap).ok() } - #[cfg(feature = "bzip2")] - BZIP2 => crate::bounded_read(bzip2_rs::DecoderReader::new(Cursor::new(raw)), cap).ok(), - #[cfg(feature = "lzip")] + BZIP2 => { + crate::bounded_read(super::bzip2_rs::DecoderReader::new(Cursor::new(raw)), cap).ok() + } LZMA => { // An LZMA block is not a bare stream. It opens with a 4-byte codec // record — algorithm, flags, then a u16 property length — followed by diff --git a/crates/exav-unpack/src/formats/gzip.rs b/crates/exav-unpack/src/formats/gzip.rs index 6db59ecb..e6667a61 100644 --- a/crates/exav-unpack/src/formats/gzip.rs +++ b/crates/exav-unpack/src/formats/gzip.rs @@ -1,52 +1,17 @@ -#![allow(unused_imports)] -use crate::*; -use std::io::{BufReader, Cursor, Read, Seek, Write}; +//! gzip: one member, every gzip member of the file decoded in turn as it is +//! read. +use std::io::{BufReader, Read}; -pub(crate) fn extract_gzip( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - budget.count_entry()?; - let cap = budget.reserve()?; - let s = gunzip(data, cap, budget).map_err(|e| LimitHit::new(format!("gzip: {e}")))?; - if s.over_cap { - return Err(LimitHit::new("gzip member exceeds budget".to_string())); - } - ratio_guard(data.len() as u64, s.data.len() as u64, budget)?; - budget.commit(s.data.len() as u64); - Ok(visit(gzip_entry(s), budget)) -} +use crate::source::{ByteSource, Reader}; +use crate::stream::{stream_single, Visit}; +use crate::{Budget, LimitHit}; -/// Decode every member of a gzip file, up to `cap` bytes. Every member, not -/// the first: a gzip file may be several concatenated members (RFC 1952 -/// section 2.2) and the payload can live in a later one (observed: a 2-member -/// gz whose first member is a 1 KiB header and whose second holds the -/// malware). gzip, zcat and ClamAV all read them all. -/// -/// Unless checksum verification is on, a decode error keeps what was decoded -/// before it, so a damaged or mis-summed file still has its content scanned. -pub(crate) fn gunzip( - src: R, - cap: u64, - budget: &Budget, -) -> std::io::Result { - bounded_read_salvage( - crate::inflate::Gunzip::new(src), - cap, - !budget.should_verify_checksums(), - ) -} - -/// The entry for decoded gzip content, flagged when damage left part of it -/// undecoded. -pub(crate) fn gzip_entry(s: Salvaged) -> Entry { - let mut e = Entry::new("gzip-content".to_string(), s.data); - if s.undecoded { - e.unsupported = Some( - "gzip data failed to decode part way; the bytes before the failure \ - were scanned", - ); - } - e +pub(crate) fn walk( + src: &dyn ByteSource, + budget: &mut Budget, + visit: Visit, +) -> Result, LimitHit> { + stream_single(&mut Reader::new(src), budget, visit, "gzip-content", |r| { + Ok(Box::new(crate::inflate::Gunzip::new(BufReader::new(r))) as Box) + }) } diff --git a/crates/exav-unpack/src/formats/iso.rs b/crates/exav-unpack/src/formats/iso.rs index a3f2bb28..d6dfe768 100644 --- a/crates/exav-unpack/src/formats/iso.rs +++ b/crates/exav-unpack/src/formats/iso.rs @@ -65,16 +65,31 @@ fn vd_roots(read: &mut dyn FnMut(u64, usize) -> Vec) -> Vec<(bool, u64, u64) joliet.into_iter().chain(primary).collect() } -/// The ISO 9660 members as `(name, offset, size)`, and the sectors they start at. -type StreamedTree = (Vec<(String, u64, u64)>, HashSet); +/// Walk a disc image: the ISO 9660 trees, then UDF, skipping the files the +/// first already emitted (the order `extract_iso` uses). Each file is streamed +/// from where it lies. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let mut source = crate::source::Reader::new(src); + let (members, mut seen) = stream_offsets(&mut source, budget.limits().max_buffer_bytes)?; + if let Some(t) = crate::stream::stream_stored(&mut source, budget, visit, members)? { + return Ok(Some(t)); + } + crate::formats::udf::stream_udf(&mut source, budget, visit, &mut seen) +} + +/// The ISO 9660 regions, and the sectors the files start at. +type StreamedTree = (Vec, HashSet); -/// Reader-based streaming: walk the ISO 9660 directory tree(s) via targeted reads -/// (directory records are small) and return each file as `(name, offset, size)`. -/// File extents — the bulk of a disc image — stream via seek+take. Mirrors -/// [`extract_iso`]'s tree walk and emission order; `max_buffer` bounds each -/// directory read. Both the primary and Joliet trees are walked; a file present -/// in both (by extent) is emitted once. The extents emitted come back too, for -/// the UDF walk that follows ([`udf::stream_udf`]) to skip. +/// Walk the ISO 9660 directory tree(s) via targeted reads (directory records +/// are small) and return each file as a region; file extents, the bulk of a +/// disc image, are streamed by the caller. `max_buffer` bounds each directory +/// read. Both the primary and Joliet trees are walked; a file present in both +/// (by extent) is emitted once. The extents emitted come back too, for the UDF +/// walk that follows to skip. pub(crate) fn stream_offsets( source: &mut R, max_buffer: u64, @@ -111,12 +126,12 @@ pub(crate) fn stream_offsets( while let Some((joliet, lba, len)) = dirs.pop() { visited += 1; if visited > MAX_DIRS { - // A zero-length marker region, the same channel the partition walker - // uses for its own caps. Breaking out silently would leave every - // remaining directory in the image unenumerated while the file could - // still be reported clean — and ISO is streamable, so this is the - // path a top-level scan takes. - out.push((format!(""), 0, 0)); + // Breaking out silently would leave every remaining directory + // unenumerated while the image could still be reported clean. + out.push(Region::Unwalked( + format!(""), + "too many ISO directories to walk them all", + )); break; } let start = lba.saturating_mul(SECTOR); @@ -138,9 +153,11 @@ pub(crate) fn stream_offsets( continue; } if rec_len < 33 || p + rec_len > dir.len() { - // Everything after a corrupt record is unreachable; the marker - // reports it, as `extract_iso` does. - out.push((format!(""), 0, 0)); + // Everything after a corrupt record is unreachable. + out.push(Region::Unwalked( + format!(""), + "corrupt ISO directory record; remaining entries unreadable", + )); break; } let rec = &dir[p..p + rec_len]; @@ -161,7 +178,7 @@ pub(crate) fn stream_offsets( let fstart = child_lba.saturating_mul(SECTOR); let fend = fstart.saturating_add(child_len).min(total_len); if fend > fstart { - out.push((name, fstart, fend - fstart)); + out.push(Region::Member(name, fstart, fend - fstart)); } } p += rec_len; @@ -169,135 +186,3 @@ pub(crate) fn stream_offsets( } Ok((out, seen_files)) } - -/// ISO 9660: a minimal, dependency-free reader that walks the volume descriptors' -/// directory trees (primary + Joliet) and emits each file's extent. Bounded by -/// the budget and a fixed directory-count guard. A file listed in more than one -/// tree (by extent) is emitted once. Rock Ridge extensions are ignored (the raw -/// payload is what matters for scanning); Joliet is honored for both reach and -/// the real long file names. -pub(crate) fn extract_iso( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let mut read_at = |off: u64, len: usize| -> Vec { - let o = off as usize; - data.get(o..o.saturating_add(len)) - .or_else(|| data.get(o..)) - .unwrap_or(&[]) - .to_vec() - }; - let roots = vd_roots(&mut read_at); - if roots.is_empty() && !udf::has_udf(data) { - return Ok(None); - } - let sector = SECTOR as usize; - let mut seen_files: HashSet = HashSet::new(); - let mut dirs: Vec<(bool, u64, u64)> = roots; - let mut visited = 0usize; - while let Some((joliet, lba, len)) = dirs.pop() { - visited += 1; - if visited > MAX_DIRS { - // Say so. Breaking here silently would leave every remaining - // directory in the image unenumerated while the file could still be - // reported clean. - budget.count_entry()?; - if let Some(r) = visit( - Entry::unsupported( - format!(""), - 0, - false, - "too many ISO directories to walk them all", - ), - budget, - ) { - return Ok(Some(r)); - } - break; - } - let start = lba as usize * sector; - let end = start.saturating_add(len as usize).min(data.len()); - let Some(dir) = data.get(start..end) else { - // A directory extent lying outside the image: the image is truncated, - // so those bytes are absent rather than hidden. Nothing to report. - continue; - }; - let mut p = 0usize; - while p < dir.len() { - let rec_len = dir[p] as usize; - if rec_len == 0 { - // Records don't span sector boundaries; advance to the next. - let next = (p / sector + 1) * sector; - if next <= p { - break; - } - p = next; - continue; - } - // `rec_len` is the record's own declared length; it must cover the - // 33-byte fixed area we index below. A short (but non-zero) length is - // corrupt — bail rather than slice `rec` too short and panic. - if rec_len < 33 || p + rec_len > dir.len() { - // Corrupt record. Everything after it in this directory is - // unreachable, so report rather than stop quietly. - budget.count_entry()?; - if let Some(r) = visit( - Entry::unsupported( - format!(""), - 0, - false, - "corrupt ISO directory record; remaining entries unreadable", - ), - budget, - ) { - return Ok(Some(r)); - } - break; - } - let rec = &dir[p..p + rec_len]; - let child_lba = u32::from_le_bytes([rec[2], rec[3], rec[4], rec[5]]) as u64; - let child_len = u32::from_le_bytes([rec[10], rec[11], rec[12], rec[13]]) as u64; - let name_len = rec[32] as usize; - let flags = rec[25]; - // First two records are "." and ".." (name_len 1, names 0x00/0x01). - let is_self_or_parent = name_len == 1 && matches!(rec.get(33), Some(0) | Some(1)); - if is_self_or_parent { - p += rec_len; - continue; - } - if flags & 0x02 != 0 { - // Subdirectory: queue it (depth bounded by the count guard). - dirs.push((joliet, child_lba, child_len)); - } else if seen_files.insert(child_lba) { - budget.count_entry()?; - let raw = rec.get(33..33 + name_len.min(rec_len - 33)).unwrap_or(&[]); - let name = decode_name(raw, joliet); - let fstart = child_lba as usize * sector; - let fend = fstart.saturating_add(child_len as usize).min(data.len()); - // A short read means the image is truncated: the declared bytes - // are absent from the file rather than hidden in it, so scanning - // what exists is the honest answer. exav scans for malware, it is - // not a file-integrity validator. See docs/QUIRKS.md. - let content = data.get(fstart..fend).unwrap_or(&[]); - let cap = budget.reserve()?; - if content.len() as u64 > cap { - return Err(LimitHit::new(format!("iso member '{name}' exceeds budget"))); - } - budget.commit(content.len() as u64); - if let Some(r) = visit(Entry::new(name, content.to_vec()), budget) { - return Ok(Some(r)); - } - } - p += rec_len; - } - } - // Most `.iso` files a modern tool writes carry UDF as well, over the same - // extents ("UDF bridge"), and a UDF-only image has no ISO 9660 tree at all. - // `seen_files` is keyed on the extent's first block, so a file both trees - // name is emitted once. - if udf::has_udf(data) { - return udf::extract_udf(data, budget, visit, &mut seen_files); - } - Ok(None) -} diff --git a/crates/exav-unpack/src/formats/lha.rs b/crates/exav-unpack/src/formats/lha.rs index fae34823..90748274 100644 --- a/crates/exav-unpack/src/formats/lha.rs +++ b/crates/exav-unpack/src/formats/lha.rs @@ -41,20 +41,20 @@ const MAX_PLAUSIBLE_HEADER: u64 = 1 << 20; /// have *and* claim a size no archiver ever writes. That is the region where /// the reader cannot succeed either — it would allocate, read short, and fail — /// which keeps this check aligned with the answer the decoder would have given. -fn header_worth_reading(data: &[u8]) -> bool { +fn header_worth_reading(head: &[u8], len: u64) -> bool { // Too short to carry a level byte. The reader reports that truncation // itself, and does so without over-reserving. - if data.len() < 21 { + if head.len() < 21 { return true; } - let declared = match data[20] { + let declared = match head[20] { // Levels 0 and 1 size the header with a single byte, so the worst // over-declaration is a couple of hundred bytes. 0 | 1 => return true, - 2 => u64::from(u16::from_le_bytes([data[0], data[1]])), + 2 => u64::from(u16::from_le_bytes([head[0], head[1]])), // Level 3 is the only one that states its total in 32 bits, and so the // only one that can name a size no machine will satisfy. - 3 => match data.get(24..28) { + 3 => match head.get(24..28) { Some(b) => u64::from(u32::from_le_bytes([b[0], b[1], b[2], b[3]])), None => return true, }, @@ -62,37 +62,44 @@ fn header_worth_reading(data: &[u8]) -> bool { // judgement to the reader rather than guess against it. _ => return true, }; - declared <= data.len() as u64 || declared <= MAX_PLAUSIBLE_HEADER + declared <= len || declared <= MAX_PLAUSIBLE_HEADER } -/// LHA/LZH: decode each member with the pure-Rust `delharc` reader. -pub(crate) fn extract_lha( - data: &[u8], +/// Walk an LHA/LZH archive: `delharc`'s per-member decoder is already a +/// `Read`, so each member is decoded as it is read. A member with an +/// unsupported compression method is reported, not skipped; directories are +/// ignored. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - if !header_worth_reading(data) { - return Err(LimitHit::corrupt( - "lha: header declares a size no archive carries".into(), - )); - } - let mut dec = delharc::LhaDecodeReader::new(Cursor::new(data)) + visit: crate::stream::Visit, +) -> Result, LimitHit> { + use crate::stream::{emit_stream, MemberMeta}; + check_first_header(src)?; + let mut dec = delharc::LhaDecodeReader::new(crate::source::Reader::new(src)) .map_err(|e| LimitHit::new(format!("lha: {e}")))?; loop { - let header = dec.header(); - let is_dir = header.is_directory(); - let name = header.parse_pathname_to_str(); - if !is_dir && dec.is_decoder_supported() { + if !dec.header().is_directory() { budget.count_entry()?; - let cap = budget.reserve()?; - let (buf, truncated) = - bounded_read(&mut dec, cap).map_err(|e| LimitHit::new(format!("lha read: {e}")))?; - if truncated { - return Err(LimitHit::new(format!("lha member '{name}' exceeds budget"))); - } - budget.commit(buf.len() as u64); - if let Some(r) = visit(Entry::new(name, buf), budget) { - return Ok(Some(r)); + let header = dec.header(); + let meta = MemberMeta { + name: header.parse_pathname_to_str(), + comp_size: header.compressed_size, + size: Some(header.original_size), + encrypted: false, + unsupported: None, + }; + let visited = if dec.is_decoder_supported() { + emit_stream(&meta, &mut dec, budget, visit)? + } else { + let meta = MemberMeta { + unsupported: Some("unsupported LHA compression method"), + ..meta + }; + visit(&meta, None, budget) + }; + if let Some(t) = visited { + return Ok(Some(t)); } } match dec.next_file() { @@ -103,3 +110,14 @@ pub(crate) fn extract_lha( } Ok(None) } + +/// Refuse an archive whose first header would make the reader reserve more +/// than any archive carries (see [`header_worth_reading`]). +pub(crate) fn check_first_header(src: &dyn crate::source::ByteSource) -> Result<(), LimitHit> { + if header_worth_reading(&src.window(0, 28), src.len() as u64) { + return Ok(()); + } + Err(LimitHit::corrupt( + "lha: header declares a size no archive carries".into(), + )) +} diff --git a/crates/exav-unpack/src/formats/lz4.rs b/crates/exav-unpack/src/formats/lz4.rs index 48ee82c6..5c0a054f 100644 --- a/crates/exav-unpack/src/formats/lz4.rs +++ b/crates/exav-unpack/src/formats/lz4.rs @@ -19,11 +19,7 @@ //! the previous block, so every block decodes into one continuous buffer rather //! than being decompressed on its own. -use crate::{Budget, Entry, Format, LimitHit, Sink}; - -fn is_lz4(d: &[u8]) -> bool { - super::sniff::is(d, Format::Lz4) -} +use crate::source::ByteSource; /// Skippable frames are `0x184D2A50`..`0x184D2A5F`: a length and payload a /// reader steps over. Needed here to find the next frame when several are @@ -149,148 +145,246 @@ fn header(data: &[u8]) -> Result<(usize, bool, bool), &'static str> { )) } -pub(crate) fn extract_lz4( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - if !is_lz4(data) { - return Ok(None); - } +/// Output kept for matches to reach back into: an offset is 16 bits. +const HISTORY: usize = 64 * 1024; + +/// The largest block the reader takes in. A frame block is at most 4 MiB and +/// a legacy one at most 8 MiB compressed plus its worst-case growth. +const MAX_BLOCK_READ: usize = 16 * 1024 * 1024; + +/// Walk an `.lz4` file: one member, decoded as it is read. The decoder holds +/// one block and the 64 KiB a match can reach back into. +pub(crate) fn walk( + src: &dyn ByteSource, + budget: &mut crate::Budget, + visit: crate::stream::Visit, +) -> Result, crate::LimitHit> { + use crate::stream::{emit_stream, single_meta}; budget.count_entry()?; - let cap = budget.reserve()? as usize; + match content_reader(src) { + Ok(mut dec) => emit_stream( + &single_meta("lz4-content", src, None), + &mut dec, + budget, + visit, + ), + Err(reason) => Ok(visit( + &single_meta("lz4-content", src, Some(reason)), + None, + budget, + )), + } +} - let mut out: Vec = Vec::new(); - let mut decoded_any = false; - let mut pos = 0usize; +/// The content of every frame in `src`, concatenated, as a `Read` decoding a +/// block at a time. Frames concatenate: `lz4 -dc` on a file holding several +/// emits all of them, and stopping at the first end mark would leave a payload +/// behind it unscanned. `Err` when not one block decodes, with the reason. +pub(crate) fn content_reader(src: &dyn ByteSource) -> Result, &'static str> { + let mut r = Lz4Reader { + src, + pos: 0, + blocks: None, + history: Vec::new(), + handed: 0, + done: false, + failed: None, + }; + if r.next_block()? { + Ok(r) + } else { + Err("LZ4 frame holds no readable block") + } +} - // Frames concatenate: `lz4 -dc` on a file holding several emits all of them, - // one after another. Stopping at the first end mark would leave everything - // behind it unscanned, which is a one-command way to hide a payload. - while pos + 8 <= data.len() { - let legacy = data[pos..].starts_with(&LEGACY_MAGIC); - let (mut p, block_checksum, content_checksum) = if legacy { - (pos + 4, false, false) - } else if data[pos..].starts_with(&MAGIC) { - match header(&data[pos..]) { - Ok((off, bc, cc)) => (pos + off, bc, cc), - Err(reason) => { - return finish(out, decoded_any, reason, data.len() as u64, budget, visit) - } - } - } else if data[pos] & 0xF0 == 0x50 && data[pos + 1..pos + 4] == SKIPPABLE_MASK { - // A skippable frame carries no compressed data of its own; step over - // it and carry on with whatever follows. - let len = le_u32(data, pos + 4).unwrap_or(0) as usize; - pos = pos.saturating_add(8).saturating_add(len); - continue; - } else { - // Trailing bytes that are not a frame. If nothing decoded at all the - // file was never really LZ4; otherwise this is junk after the data. - break; - }; +pub(crate) struct Lz4Reader<'a> { + src: &'a dyn ByteSource, + /// Where the frame being read starts, or the next one. + pos: usize, + /// Inside a frame: where its next block is, and how its blocks are laid + /// out. + blocks: Option, + /// Decoded bytes: the last [`HISTORY`] handed out, then those not yet. + history: Vec, + handed: usize, + done: bool, + failed: Option<&'static str>, +} + +#[derive(Clone, Copy)] +struct Blocks { + p: usize, + legacy: bool, + block_checksum: bool, + content_checksum: bool, +} + +impl<'a> Lz4Reader<'a> { + /// Up to `n` bytes at `at`, as many as the input holds there. `Err` when + /// the source fails to give them. + fn window(&self, at: usize, n: usize) -> Result, &'static str> { + let w = self.src.window(at, n); + if w.len() < n.min(self.src.len().saturating_sub(at)) { + return Err("LZ4 input could not be read"); + } + Ok(w) + } - while let Some(size) = le_u32(data, p) { - p += 4; - if !legacy && size == 0 { + /// Leave the frame, the next one starting at `p`. `false` when that makes + /// no progress. + fn end_frame(&mut self, p: usize) -> bool { + self.blocks = None; + if p <= self.pos { + return false; + } + self.pos = p; + true + } + + /// Decode the next block onto `history`. `Ok(false)` when no block is + /// left, `Err` with the reason when one fails. + fn next_block(&mut self) -> Result { + let len = self.src.len(); + loop { + let Some(mut b) = self.blocks else { + // Frames concatenate: `lz4 -dc` on a file holding several emits + // all of them, one after another. + let pos = self.pos; + if pos + 8 > len { + return Ok(false); + } + let head = self.window(pos, 32)?; + let blocks = if head.starts_with(&LEGACY_MAGIC) { + Blocks { + p: pos + 4, + legacy: true, + block_checksum: false, + content_checksum: false, + } + } else if head.starts_with(&MAGIC) { + let (off, block_checksum, content_checksum) = header(&head)?; + Blocks { + p: pos + off, + legacy: false, + block_checksum, + content_checksum, + } + } else if head.len() >= 8 && head[0] & 0xF0 == 0x50 && head[1..4] == SKIPPABLE_MASK + { + // A skippable frame carries no compressed data of its own. + let skip = le_u32(&head, 4).unwrap_or(0) as usize; + self.pos = pos.saturating_add(8).saturating_add(skip); + continue; + } else { + // Trailing bytes that are not a frame. + return Ok(false); + }; + self.blocks = Some(blocks); + continue; + }; + let Some(size) = le_u32(&self.window(b.p, 4)?, 0) else { + if !self.end_frame(b.p) { + return Ok(false); + } + continue; + }; + b.p += 4; + if !b.legacy && size == 0 { // End mark, then the optional whole-frame checksum. - if content_checksum { - p += 4; + if b.content_checksum { + b.p += 4; + } + if !self.end_frame(b.p) { + return Ok(false); } - break; + continue; } - let stored = !legacy && size & BLOCK_UNCOMPRESSED != 0; + let stored = !b.legacy && size & BLOCK_UNCOMPRESSED != 0; let n = (size & !BLOCK_UNCOMPRESSED) as usize; - // A block extending past the end means the file is truncated: those - // bytes are absent rather than hidden, and what came before is kept. - let Some(body) = data.get(p..p + n) else { - p = data.len(); - break; - }; - p += n; - let block_cap = if legacy { - cap.min(out.len() + LEGACY_BLOCK) - } else { - cap - }; - if stored { - if out.len() + body.len() > block_cap { - return finish( - out, - decoded_any, - "LZ4 frame exceeds the size budget", - data.len() as u64, - budget, - visit, - ); + if b.p.saturating_add(n) > len { + // Truncated: those bytes are absent rather than hidden. + if !self.end_frame(len) { + return Ok(false); } - out.extend_from_slice(body); - } else if let Err(reason) = block(body, &mut out, block_cap) { - return finish(out, decoded_any, reason, data.len() as u64, budget, visit); + continue; } - decoded_any = true; - if block_checksum { - p += 4; + if n > MAX_BLOCK_READ { + return Err("LZ4 block is larger than the format allows"); } - if p >= data.len() { - break; + let body = self.window(b.p, n)?; + b.p += n; + // A frame block is held to the legacy size too, which is twice the + // largest the format allows, so that one block cannot fill memory. + let block_cap = self.history.len() + LEGACY_BLOCK; + if stored { + if self.history.len() + body.len() > block_cap { + return Err("LZ4 frame exceeds the size budget"); + } + self.history.extend_from_slice(&body); + } else { + block(&body, &mut self.history, block_cap)?; + } + if b.block_checksum { + b.p += 4; } + self.blocks = Some(b); // A legacy frame has no end mark: it ends where the next frame's // magic begins, or at the end of the file. - if legacy && (data[p..].starts_with(&LEGACY_MAGIC) || data[p..].starts_with(&MAGIC)) { - break; + let next = self.window(b.p, 4)?; + if b.p >= len + || (b.legacy && (next.starts_with(&LEGACY_MAGIC) || next.starts_with(&MAGIC))) + { + self.end_frame(b.p); } + return Ok(true); } - if p <= pos { - break; // no progress — malformed - } - pos = p; - } - - if !decoded_any { - return report( - "LZ4 frame holds no readable block", - data.len() as u64, - budget, - visit, - ); } - budget.commit(out.len() as u64); - Ok(visit(Entry::new("lz4-content".to_string(), out), budget)) } -/// A frame that failed partway has still produced real bytes. Emitting them and -/// reporting the failure beats discarding content that was successfully decoded. -fn finish( - out: Vec, - decoded_any: bool, - reason: &'static str, - size: u64, - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - if !decoded_any { - return report(reason, size, budget, visit); - } - budget.count_entry()?; - if let Some(r) = visit( - Entry::unsupported("lz4-content".to_string(), size, false, reason), - budget, - ) { - return Ok(Some(r)); +impl std::io::Read for Lz4Reader<'_> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + loop { + if self.handed < self.history.len() { + let n = (self.history.len() - self.handed).min(out.len()); + out[..n].copy_from_slice(&self.history[self.handed..self.handed + n]); + self.handed += n; + return Ok(n); + } + if let Some(reason) = self.failed.take() { + self.done = true; + return Err(std::io::Error::other(reason)); + } + if self.done { + return Ok(0); + } + // Keep only what a match can still reach. + let drop = self.handed.saturating_sub(HISTORY); + self.history.drain(..drop); + self.handed -= drop; + match self.next_block() { + Ok(true) => {} + Ok(false) => self.done = true, + Err(reason) => self.failed = Some(reason), + } + } } - budget.commit(out.len() as u64); - Ok(visit(Entry::new("lz4-content".to_string(), out), budget)) } -fn report( - reason: &'static str, - size: u64, - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - Ok(visit( - Entry::unsupported("lz4-content".to_string(), size, false, reason), - budget, - )) +#[cfg(test)] +mod tests { + /// Input the source fails to deliver is a read error, not the frame's end. + #[test] + fn a_source_that_fails_is_not_taken_for_the_end() { + let blob = crate::read_fixture(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/lz4/one.lz4" + )) + .unwrap(); + assert!(super::content_reader(&blob).is_ok()); + let src = crate::source::short_source(&blob[..100], blob.len() as u64); + assert_eq!( + super::content_reader(&src).err(), + Some("LZ4 input could not be read") + ); + } } diff --git a/crates/exav-unpack/src/formats/lzip.rs b/crates/exav-unpack/src/formats/lzip.rs index 955a2808..32944b26 100644 --- a/crates/exav-unpack/src/formats/lzip.rs +++ b/crates/exav-unpack/src/formats/lzip.rs @@ -1,20 +1,50 @@ -use crate::*; -use std::io::Cursor; +//! lzip: one member, every lzip member of the file decoded in turn as it is +//! read. +use std::io::{self, Read, Seek}; -pub(crate) fn extract_lzip( - data: &[u8], +use crate::source::{ByteSource, Reader}; +use crate::stream::{stream_single, Visit}; +use crate::{Budget, LimitHit}; + +pub(crate) fn walk( + src: &dyn ByteSource, budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - budget.count_entry()?; - let cap = budget.reserve()?; - let reader = lzma_rust2::LzipReader::new(Cursor::new(data)); - let (out, truncated) = - bounded_read(reader, cap).map_err(|e| LimitHit::new(format!("lzip: {e}")))?; - if truncated { - return Err(LimitHit::new("lzip member exceeds budget".to_string())); + visit: Visit, +) -> Result, LimitHit> { + // `LzipReader` takes any failure to read a member header, an I/O error + // included, for the end of the stream, so errors are caught underneath. + let mut source = ErrorLatch { + inner: Reader::new(src), + failed: None, + }; + let out = stream_single(&mut source, budget, visit, "lzip-content", |r| { + Ok(Box::new(lzma_rust2::LzipReader::new(r)) as Box) + })?; + match (out, source.failed) { + (None, Some(e)) => Err(LimitHit::corrupt(format!("lzip: read failed: {e}"))), + (out, _) => Ok(out), + } +} + +/// Passes reads and seeks through, keeping the first read error for a caller +/// whose decoder does not report it. +struct ErrorLatch { + inner: R, + failed: Option, +} + +impl Read for ErrorLatch { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf).inspect_err(|e| { + if e.kind() != io::ErrorKind::Interrupted && self.failed.is_none() { + self.failed = Some(io::Error::new(e.kind(), e.to_string())); + } + }) + } +} + +impl Seek for ErrorLatch { + fn seek(&mut self, to: io::SeekFrom) -> io::Result { + self.inner.seek(to) } - ratio_guard(data.len() as u64, out.len() as u64, budget)?; - budget.commit(out.len() as u64); - Ok(visit(Entry::new("lzip-content".to_string(), out), budget)) } diff --git a/crates/exav-unpack/src/formats/lzw.rs b/crates/exav-unpack/src/formats/lzw.rs index 00f2ec84..7600b6e7 100644 --- a/crates/exav-unpack/src/formats/lzw.rs +++ b/crates/exav-unpack/src/formats/lzw.rs @@ -31,174 +31,233 @@ //! which is why this is checked against real `compress` output rather than a //! round-trip against a matching encoder. -use crate::{Budget, Entry, LimitHit, Sink}; +use std::io::{self, Read}; + +use crate::source::{ByteSource, Bytes, Stepper}; const INIT_WIDTH: u32 = 9; const MAX_WIDTH_LIMIT: u32 = 16; const CLEAR: u16 = 256; const FIRST_FREE: u16 = 257; -pub(crate) fn extract_lzw( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - if !super::sniff::is(data, crate::Format::Lzw) { - return Ok(None); - } - budget.count_entry()?; - let cap = budget.reserve()?; - let (out, truncated) = match decompress(data, cap) { - Some(v) => v, - // A stream we cannot decode at all: the bytes are here and unread, so - // report rather than drop. - None => { - return Ok(visit( - Entry::unsupported( - "lzw-content".to_string(), - data.len() as u64, - false, - "malformed Unix compress (.Z) stream", - ), - budget, - )) +/// A decoder over a whole `.Z` file, header included, one code at a time. +struct Lzw { + data: B, + max_width: u32, + block_mode: bool, + /// `prefix`/`suffix` are the classic parallel arrays: entry `c` extends + /// entry `prefix[c]` by the byte `suffix[c]`. + prefix: Vec, + suffix: Vec, + next: u16, + width: u32, + prev: Option, + stack: Vec, + /// Bit cursor over the code stream (after the 3-byte header). + bitpos: u64, + total_bits: u64, + /// Codes are written in groups of eight, and on any width change the + /// writer pads to the next group boundary at the OLD width. The boundary is + /// measured from a base that MOVES to each padding point, not from the + /// start of the file: the reference restarts its input buffer there, so a + /// decoder that aligns absolutely drifts after the first width change. + base: u64, +} + +impl Lzw { + /// `None` when the header is unusable. + fn new(mut data: B) -> Option { + if data.len() < 3 { + return None; } - }; - if truncated { - return Ok(visit( - Entry::unsupported( - "lzw-content".to_string(), - out.len() as u64, - false, - "Unix compress (.Z) stream exceeds the per-member size budget", - ), - budget, - )); - } - if out.is_empty() { - return Ok(None); + let flags = data.at(2); + let max_width = (flags & 0x1f) as u32; + let block_mode = flags & 0x80 != 0; + if !(INIT_WIDTH..=MAX_WIDTH_LIMIT).contains(&max_width) { + return None; + } + let table_cap = 1usize << max_width; + let total_bits = (data.len() as u64 - 3) * 8; + Some(Lzw { + data, + max_width, + block_mode, + prefix: vec![0u16; table_cap], + suffix: vec![0u8; table_cap], + next: if block_mode { FIRST_FREE } else { CLEAR }, + width: INIT_WIDTH, + prev: None, + stack: Vec::new(), + bitpos: 0, + total_bits, + base: 0, + }) } - budget.commit(out.len() as u64); - Ok(visit(Entry::new("lzw-content".to_string(), out), budget)) -} -/// Decode the LZW stream. Returns `(bytes, hit_cap)`; `None` only when the -/// header itself is unusable. A stream that ends mid-code is *truncated*, not -/// corrupt: everything decoded so far is returned, matching the salvage rule for -/// sequential streams (docs/QUIRKS.md). -fn decompress(data: &[u8], cap: u64) -> Option<(Vec, bool)> { - let max_width = (data[2] & 0x1f) as u32; - let block_mode = data[2] & 0x80 != 0; - if !(INIT_WIDTH..=MAX_WIDTH_LIMIT).contains(&max_width) { - return None; + fn align_to_group(&mut self) { + let group = (self.width as u64) * 8; + self.bitpos = self.base + (self.bitpos - self.base).div_ceil(group) * group; + self.base = self.bitpos; } - // `prefix`/`suffix` are the classic parallel arrays: entry `c` extends - // entry `prefix[c]` by the byte `suffix[c]`. - let table_cap = 1usize << max_width; - let mut prefix = vec![0u16; table_cap]; - let mut suffix = vec![0u8; table_cap]; - let mut next: u16 = if block_mode { FIRST_FREE } else { CLEAR }; - let mut width = INIT_WIDTH; - - let mut out: Vec = Vec::new(); - let mut stack: Vec = Vec::new(); - let mut prev: Option = None; - - // Bit cursor over the code stream (after the 3-byte header). - let bits = &data[3..]; - let total_bits = (bits.len() as u64) * 8; - let mut bitpos: u64 = 0; - - // Codes are written in groups of eight, and on any width change the writer - // pads to the next group boundary at the OLD width. The boundary is measured - // from a base that MOVES to each padding point, not from the start of the - // file: the reference restarts its input buffer there, so a decoder that - // aligns absolutely drifts after the first width change. - let mut base: u64 = 0; - let align_to_group = |pos: u64, base: u64, width: u32| -> u64 { - let group = (width as u64) * 8; - base + (pos - base).div_ceil(group) * group - }; - - let read_code = |bitpos: &mut u64, width: u32| -> Option { - if *bitpos + width as u64 > total_bits { + fn read_code(&mut self) -> Option { + if self.bitpos + self.width as u64 > self.total_bits { return None; } let mut v: u32 = 0; - for i in 0..width { - let b = *bitpos + i as u64; - let byte = bits[(b / 8) as usize]; + for i in 0..self.width { + let b = self.bitpos + i as u64; + let byte = self.data.at(3 + (b / 8) as usize); let bit = (byte >> (b % 8)) & 1; v |= (bit as u32) << i; // LSB-first } - *bitpos += width as u64; + self.bitpos += self.width as u64; Some(v as u16) - }; - - loop { - // Widen before reading, mirroring the writer: it grows the code width as - // soon as the table reaches `(1 << width) - 1` entries, and pads to the - // next group boundary at the old width as it does so. - if width < max_width && next as u32 >= (1u32 << width) { - bitpos = align_to_group(bitpos, base, width); - base = bitpos; - width += 1; - } - let Some(code) = read_code(&mut bitpos, width) else { - break; // ran out of input: truncated, keep what we have - }; - - if block_mode && code == CLEAR { - // A reset is also a width change: pad at the width in force, then - // start over at the initial width. - bitpos = align_to_group(bitpos, base, width); - base = bitpos; - next = FIRST_FREE; - width = INIT_WIDTH; - prev = None; - continue; - } + } - // Rebuild the string for `code` by walking the prefix chain. - stack.clear(); - let mut cur = code; - if cur >= next { - // KwKwK: the code refers to the entry being defined right now. - let p = prev?; - stack.push(first_byte(&prefix, &suffix, p)); - cur = p; - } - let mut guard = 0usize; - while cur >= 256 { - if cur as usize >= table_cap || guard > table_cap { - return None; // cyclic or out-of-range chain: unusable header state + /// Append the string the next code expands to. `Some(false)` at the end of + /// the input (a stream that ends mid-code is truncated, not corrupt), + /// `None` for a code the table cannot expand. + fn step(&mut self, out: &mut Vec) -> Option { + let table_cap = self.prefix.len(); + loop { + // Widen before reading, mirroring the writer: it grows the code + // width as soon as the table reaches `(1 << width) - 1` entries, + // and pads to the next group boundary at the old width as it does + // so. + if self.width < self.max_width && self.next as u32 >= (1u32 << self.width) { + self.align_to_group(); + self.width += 1; } - stack.push(suffix[cur as usize]); - cur = prefix[cur as usize]; - guard += 1; - } - stack.push(cur as u8); - for &b in stack.iter().rev() { - out.push(b); + let Some(code) = self.read_code() else { + return Some(false); + }; + + if self.block_mode && code == CLEAR { + // A reset is also a width change: pad at the width in force, + // then start over at the initial width. + self.align_to_group(); + self.next = FIRST_FREE; + self.width = INIT_WIDTH; + self.prev = None; + continue; + } + + // Rebuild the string for `code` by walking the prefix chain. + self.stack.clear(); + let mut cur = code; + if cur >= self.next { + // KwKwK: the code refers to the entry being defined right now. + let p = self.prev?; + self.stack.push(first_byte(&self.prefix, &self.suffix, p)); + cur = p; + } + let mut guard = 0usize; + while cur >= 256 { + if cur as usize >= table_cap || guard > table_cap { + return None; // cyclic or out-of-range chain + } + self.stack.push(self.suffix[cur as usize]); + cur = self.prefix[cur as usize]; + guard += 1; + } + self.stack.push(cur as u8); + out.extend(self.stack.iter().rev()); + + // Define the next table entry from the previous code plus this + // string's first byte. + if let Some(p) = self.prev { + if (self.next as usize) < table_cap { + self.prefix[self.next as usize] = p; + self.suffix[self.next as usize] = *self.stack.last().unwrap_or(&0); + self.next += 1; + } + } + self.prev = Some(code); + return Some(true); } - if out.len() as u64 > cap { - out.truncate(cap as usize); - return Some((out, true)); + } +} + +/// Walk a `.Z` file: one member, decoded as it is read. +pub(crate) fn walk( + src: &dyn ByteSource, + budget: &mut crate::Budget, + visit: crate::stream::Visit, +) -> Result, crate::LimitHit> { + use crate::stream::{emit_stream, single_meta}; + budget.count_entry()?; + match content_reader(src) { + Some(mut dec) => emit_stream( + &single_meta("lzw-content", src, None), + &mut dec, + budget, + visit, + ), + None => { + let meta = single_meta( + "lzw-content", + src, + Some("malformed Unix compress (.Z) stream"), + ); + Ok(visit(&meta, None, budget)) } + } +} + +/// The content of a `.Z` file as a `Read`, decoded as it is read, for the +/// streaming walk. `None` when the header is unusable. +pub(crate) fn content_reader(data: &dyn ByteSource) -> Option> { + Some(LzwReader { + src: data, + lzw: Lzw::new(Stepper::new(data))?, + staged: Vec::new(), + taken: 0, + done: false, + }) +} + +pub(crate) struct LzwReader<'a> { + src: &'a dyn ByteSource, + lzw: Lzw>, + /// Decoded bytes not yet handed out. + staged: Vec, + taken: usize, + done: bool, +} - // Define the next table entry from the previous code plus this string's - // first byte. - if let Some(p) = prev { - if (next as usize) < table_cap { - prefix[next as usize] = p; - suffix[next as usize] = *stack.last().unwrap_or(&0); - next += 1; +impl Read for LzwReader<'_> { + fn read(&mut self, out: &mut [u8]) -> io::Result { + while self.taken == self.staged.len() && !self.done { + self.staged.clear(); + self.taken = 0; + while self.staged.len() < 4096 { + match self.lzw.step(&mut self.staged) { + Some(true) => {} + Some(false) => { + self.done = true; + break; + } + None => { + return Err(io::Error::other(self.src.read_error().unwrap_or_else( + || "malformed Unix compress (.Z) stream".to_string(), + ))); + } + } + } + // A source that failed reads as zeros: what came from them is not + // the content. + if let Some(e) = self.src.read_error() { + self.done = true; + self.staged.clear(); + return Err(io::Error::other(e)); } } - prev = Some(code); + let n = (self.staged.len() - self.taken).min(out.len()); + out[..n].copy_from_slice(&self.staged[self.taken..self.taken + n]); + self.taken += n; + Ok(n) } - Some((out, false)) } /// First byte of the string a code expands to. @@ -214,3 +273,29 @@ fn first_byte(prefix: &[u16], suffix: &[u8], mut c: u16) -> u8 { } c as u8 } + +#[cfg(test)] +mod tests { + use std::io::Read; + + /// Input the source fails to deliver is a read error, not the end of the + /// stream. + #[test] + fn a_source_that_fails_is_not_taken_for_the_end() { + let blob = crate::read_fixture(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/lzw/bigreset.b16.Z" + )) + .unwrap(); + let mut all = Vec::new(); + super::content_reader(&blob) + .unwrap() + .read_to_end(&mut all) + .unwrap(); + let src = crate::source::short_source(&blob[..1000], blob.len() as u64); + let mut part = Vec::new(); + let read = super::content_reader(&src).unwrap().read_to_end(&mut part); + assert!(read.is_err()); + assert!(part.len() < all.len() && all.starts_with(&part)); + } +} diff --git a/crates/exav-unpack/src/formats/machofat.rs b/crates/exav-unpack/src/formats/machofat.rs index 50c86458..ba4af102 100644 --- a/crates/exav-unpack/src/formats/machofat.rs +++ b/crates/exav-unpack/src/formats/machofat.rs @@ -61,18 +61,12 @@ fn be_u64(data: &[u8], p: usize) -> Option { ])) } -/// Parse and fully validate the fat header. Returns the arch slices only when -/// the input is a *plausible* universal binary (see the module docs for the -/// strictness rationale); returns `None` for anything else, including a Java -/// `.class` file that merely shares the `0xCAFEBABE` magic. -fn parse_archs(data: &[u8]) -> Option> { - parse_archs_len(data, data.len() as u64) -} - -/// As [`parse_archs`], but validating slice ranges against an explicit -/// `total_len` rather than `data.len()`. The reader-based path passes only the -/// header region as `data` (which must still contain the whole arch table) and -/// the true file length as `total_len`. +/// Parse and fully validate the fat header of a file of `total_len` bytes, +/// whose start is `data` (which must contain the whole arch table). Returns +/// the arch slices only when the input is a *plausible* universal binary (see +/// the module docs for the strictness rationale); returns `None` for anything +/// else, including a Java `.class` file that merely shares the `0xCAFEBABE` +/// magic. fn parse_archs_len(data: &[u8], total_len: u64) -> Option> { let magic = be_u32(data, 0)?; let is64 = match magic { @@ -119,13 +113,25 @@ fn parse_archs_len(data: &[u8], total_len: u64) -> Option> { } /// True when `data` is a plausible Mach-O universal binary (used by `detect`). -pub(crate) fn looks_like_machofat(data: &[u8]) -> bool { - parse_archs(data).is_some() +pub(crate) fn looks_like_machofat(p: &crate::Probe) -> bool { + parse_archs_len(p.head, p.len as u64).is_some() } -/// Parse arch slice offsets from a seekable source (reader-based streaming). The -/// arch table is a bounded header at the start; each arch image streams via -/// seek+take. Returns `(name, offset, size)` per arch, matching [`extract_machofat`]. +/// Walk a universal binary, each arch image streamed from where it lies. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let mut source = crate::source::Reader::new(src); + let members = stream_offsets(&mut source)?; + crate::stream::stream_stored(&mut source, budget, visit, members) +} + +/// Parse arch slice offsets from a seekable source. The arch table is a bounded +/// header at the start; each arch image streams via seek+take. Returns +/// `(name, offset, size)` per arch, and none for an implausible fat binary +/// (a Java `.class` sharing the magic), which is then scanned as itself. pub(crate) fn stream_offsets( source: &mut R, ) -> Result, LimitHit> { @@ -149,38 +155,6 @@ pub(crate) fn stream_offsets( .collect()) } -pub(crate) fn extract_machofat( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - // Not a plausible fat binary (e.g. a Java `.class` sharing the magic): emit - // nothing so the engine falls back to normal handling. - let Some(archs) = parse_archs(data) else { - return Ok(None); - }; - - for (i, a) in archs.iter().enumerate() { - // `parse_archs` already validated the range; clamp again defensively so a - // future change can never produce `start > end` (which would panic). - let start = a.offset.min(data.len()); - let end = a.offset.saturating_add(a.size).min(data.len()); - let slice = &data[start..end]; - - budget.count_entry()?; - let cap = budget.reserve()?; - if slice.len() as u64 > cap { - return Err(LimitHit::new(format!("macho fat arch {i} exceeds budget"))); - } - let bytes = slice.to_vec(); - budget.commit(bytes.len() as u64); - if let Some(r) = visit(Entry::new(format!("macho-arch-{i}"), bytes), budget) { - return Ok(Some(r)); - } - } - Ok(None) -} - #[cfg(test)] mod tests { use super::*; @@ -233,7 +207,7 @@ mod tests { // arch-table / offset-within-file checks must still reject it. let mut blob = vec![0xCA, 0xFE, 0xBA, 0xBE, 0x00, 0x00, 0x00, 0x34]; blob.extend_from_slice(&[0xABu8; 200]); // constant pool: not valid arch records - assert!(!looks_like_machofat(&blob)); + assert!(!looks_like_machofat(&crate::Probe::whole(&blob))); let mut budget = Budget::new(Limits::default()); assert!(extract(Format::Machofat, &blob, &mut budget) .unwrap() diff --git a/crates/exav-unpack/src/formats/mod.rs b/crates/exav-unpack/src/formats/mod.rs index e759e80e..91b19e1d 100644 --- a/crates/exav-unpack/src/formats/mod.rs +++ b/crates/exav-unpack/src/formats/mod.rs @@ -10,8 +10,18 @@ mod arj; mod arj_parse; #[cfg(feature = "bzip2")] pub(crate) mod bzip2; +// Every feature that enables `bzip2-decoder`; the feature alone decodes nothing. +#[cfg(any( + feature = "bzip2", + feature = "sevenz", + feature = "nsis", + feature = "alz", + feature = "egg", + feature = "dmg" +))] +mod bzip2_rs; #[cfg(feature = "cab")] -mod cab; +pub(crate) mod cab; #[cfg(feature = "cab")] mod cab_parse; #[cfg(feature = "chm")] @@ -19,19 +29,19 @@ mod chm; #[cfg(feature = "cpio")] pub(crate) mod cpio; #[cfg(feature = "dmg")] -mod dmg; +pub(crate) mod dmg; #[cfg(feature = "egg")] pub(crate) mod egg; #[cfg(feature = "email")] mod email; #[cfg(feature = "gzip")] -mod gzip; +pub(crate) mod gzip; #[cfg(feature = "iso")] pub(crate) mod iso; #[cfg(feature = "lha")] -mod lha; +pub(crate) mod lha; #[cfg(feature = "lzip")] -mod lzip; +pub(crate) mod lzip; #[cfg(feature = "ole")] mod ole; #[cfg(all(feature = "ole", feature = "decrypt"))] @@ -44,6 +54,8 @@ mod pdf_parse; mod xlm; #[cfg(feature = "ole")] mod xlm_functions; +#[cfg(feature = "zstd")] +pub(crate) mod zstd; // Shared PPMd7 model: used by both RAR (RAR3 PPMd blocks) and 7-Zip. #[cfg(feature = "ace")] pub(crate) mod ace; @@ -53,6 +65,8 @@ pub(crate) mod alz; pub(crate) mod arc; #[cfg(feature = "autoit")] mod autoit; +#[cfg(feature = "autoit")] +mod autoit_data; #[cfg(feature = "egg")] pub(crate) mod azo; #[cfg(feature = "egg")] @@ -136,8 +150,6 @@ mod zip_crypto; pub(crate) mod zoo; #[cfg(feature = "zoo")] mod zoo_parse; -#[cfg(feature = "zstd")] -mod zstd; // aPLib codec: the compression used by the Petite/FSG2/NsPack PE packers. #[cfg(feature = "aimodel")] mod aimodel; @@ -156,9 +168,9 @@ mod screnc; #[cfg(feature = "sfx")] pub(crate) mod sfx; #[cfg(feature = "swf")] -mod swf; +pub(crate) mod swf; #[cfg(feature = "szdd")] -mod szdd; +pub(crate) mod szdd; #[cfg(feature = "tnef")] pub(crate) mod tnef; #[cfg(feature = "uuencode")] @@ -169,48 +181,32 @@ mod xdp; // Re-export the extractor entry points used by the dispatch in `lib.rs`. #[cfg(feature = "aimodel")] pub(crate) use aimodel::{extract_aimodel, is_aimodel}; -#[cfg(feature = "ar")] -pub(crate) use ar::extract_ar; #[cfg(feature = "arj")] pub(crate) use arj::extract_arj; #[cfg(feature = "autoit")] -pub(crate) use autoit::{extract_autoit, is_autoit}; +pub(crate) use autoit::{extract_autoit, is_autoit, MARKER_EA05, MARKER_EA06}; #[cfg(feature = "binhex")] pub(crate) use binhex::{extract_binhex, looks_like_binhex}; -#[cfg(feature = "bzip2")] -pub(crate) use bzip2::extract_bzip2; -#[cfg(feature = "cab")] -pub(crate) use cab::{extract_cab, stream_cab}; #[cfg(feature = "chm")] pub(crate) use chm::extract_chm; -#[cfg(feature = "cpio")] -pub(crate) use cpio::extract_cpio; #[cfg(feature = "dmg")] -pub(crate) use dmg::{extract_dmg, is_dmg}; +pub(crate) use dmg::is_dmg; #[cfg(feature = "email")] pub(crate) use email::extract_email; -#[cfg(feature = "gzip")] -pub(crate) use gzip::{extract_gzip, gunzip, gzip_entry}; -#[cfg(feature = "iso")] -pub(crate) use iso::extract_iso; #[cfg(feature = "javaclass")] pub(crate) use javaclass::extract_javaclass; -#[cfg(feature = "lha")] -pub(crate) use lha::extract_lha; #[cfg(feature = "lnk")] pub(crate) use lnk::extract_lnk; -#[cfg(feature = "lzip")] -pub(crate) use lzip::extract_lzip; #[cfg(feature = "machofat")] -pub(crate) use machofat::{extract_machofat, looks_like_machofat}; +pub(crate) use machofat::looks_like_machofat; #[cfg(feature = "nsis")] -pub(crate) use nsis::{extract_nsis, is_nsis}; +pub(crate) use nsis::{extract_nsis, is_nsis, NSIS_SIG}; #[cfg(feature = "ole")] pub(crate) use ole::extract_ole; #[cfg(feature = "onenote")] -pub(crate) use onenote::{extract_onenote, is_onenote}; +pub(crate) use onenote::is_onenote; #[cfg(feature = "partition")] -pub(crate) use partition::{extract_partition, is_partition}; +pub(crate) use partition::is_partition; #[cfg(feature = "pdf")] pub(crate) use pdf::extract_pdf; #[cfg(feature = "pdf")] @@ -219,26 +215,16 @@ pub use pdf::has_obfuscated_name_object; pub(crate) use pepack::emulate_pe; #[cfg(feature = "pepack")] pub(crate) use pepack::{extract_pepack, is_pepack}; -#[cfg(feature = "pyc")] -pub(crate) use pyc::extract_pyc; #[cfg(feature = "rar")] pub(crate) use rar::extract_rar; #[cfg(feature = "rtf")] pub(crate) use rtf::extract_rtf; #[cfg(feature = "screnc")] -pub(crate) use screnc::{extract_screnc, looks_like_screnc}; -#[cfg(feature = "sevenz")] -pub(crate) use sevenz::extract_sevenz; +pub(crate) use screnc::{extract_screnc, looks_like_screnc, MARKER as SCRENC_MARKER}; #[cfg(feature = "sfx")] -pub(crate) use sfx::{extract_sfx, looks_like_sfx}; -#[cfg(feature = "swf")] -pub(crate) use swf::extract_swf; +pub(crate) use sfx::looks_like_sfx; #[cfg(feature = "szdd")] pub(crate) use szdd::{extract_szdd, is_szdd}; -#[cfg(feature = "tar")] -pub(crate) use tar::extract_tar; -#[cfg(feature = "tnef")] -pub(crate) use tnef::extract_tnef; #[cfg(feature = "upx")] pub(crate) use upx::{extract_upx, find_packheader, has_packheader_layout}; #[cfg(feature = "uuencode")] @@ -247,14 +233,8 @@ pub(crate) use uuencode::{extract_uuencode, looks_like_uuencode}; pub(crate) use xar::extract_xar; #[cfg(feature = "xdp")] pub(crate) use xdp::{extract_xdp, looks_like_xdp}; -#[cfg(feature = "xz")] -pub(crate) use xz::extract_xz; #[cfg(feature = "zip")] pub(crate) use zip::extract_zip; -#[cfg(feature = "zip")] -pub use zip::ZipMembers; -#[cfg(feature = "zstd")] -pub(crate) use zstd::extract_zstd; // Exposed for the rar5_check example / integration tests. #[cfg(feature = "rar")] pub use rar5_unpack::{unpack50, window_size_from_comp_info}; diff --git a/crates/exav-unpack/src/formats/nsis.rs b/crates/exav-unpack/src/formats/nsis.rs index ba9bd4ac..5d0a6360 100644 --- a/crates/exav-unpack/src/formats/nsis.rs +++ b/crates/exav-unpack/src/formats/nsis.rs @@ -28,7 +28,7 @@ use crate::*; use std::io::Cursor; /// 16-byte firstheader signature: LE `0xDEADBEEF` then ASCII `"NullsoftInst"`. -const NSIS_SIG: [u8; 16] = [ +pub(crate) const NSIS_SIG: [u8; 16] = [ 0xEF, 0xBE, 0xAD, 0xDE, b'N', b'u', b'l', b'l', b's', b'o', b'f', b't', b'I', b'n', b's', b't', ]; @@ -47,8 +47,9 @@ fn find_firstheader(data: &[u8]) -> Option { } /// True if `data` is a PE stub carrying the NSIS firstheader signature. -pub(crate) fn is_nsis(data: &[u8]) -> bool { - data.starts_with(b"MZ") && find_firstheader(data).is_some() +pub(crate) fn is_nsis(p: &crate::Probe) -> bool { + // As `find_firstheader` finds it. + p.head.starts_with(b"MZ") && p.find(&NSIS_SIG).and_then(|at| at.checked_sub(4)).is_some() } pub(crate) fn extract_nsis( @@ -129,7 +130,7 @@ pub(crate) fn extract_nsis( } /// Decode one NSIS compressed stream, selecting the codec from the leading byte: -/// `'1'` ⇒ NSIS bzip2, `0x5d` ⇒ LZMA props, else raw DEFLATE — with a fallback. +/// `'1'` ⇒ NSIS bzip2, `0x5d` ⇒ LZMA props, else raw DEFLATE, each with a fallback. fn decode_stream(block: &[u8], cap: u64) -> Option> { match block.first().copied()? { b'1' => decode_bzip2(block, cap).or_else(|| decode_deflate(block, cap)), @@ -157,15 +158,15 @@ fn decode_lzma(block: &[u8], cap: u64) -> Option> { return None; } // The dictionary size comes straight out of the file and the decoder - // allocates it UP FRONT, before a byte is decompressed — so an attacker sets + // allocates it UP FRONT, before a byte is decompressed, so an attacker sets // it to whatever they like and exav allocates that much. Measured: a 766 KB // installer declaring a 1.5 GB dictionary, which aborted the process under // the daemon's per-job RLIMIT_AS. Under the daemon that abort closes the // client connection with no reply at all, which reads as a clean scan. // // Clamp it to the caller's budget. A dictionary bigger than the output it is - // used to produce cannot help — LZMA only ever looks back into bytes it has - // already emitted — so capping at `cap` costs nothing on a real stream while + // used to produce cannot help (LZMA only ever looks back into bytes it has + // already emitted), so capping at `cap` costs nothing on a real stream while // making the allocation bounded by the same limit as everything else. let dict = crate::bounded_dict(u32le(block, 1), cap); let reader = lzma_rust2::LzmaReader::new_with_props( @@ -183,7 +184,7 @@ fn decode_lzma(block: &[u8], cap: u64) -> Option> { /// in which case the block is reported unsupported. fn decode_bzip2(block: &[u8], cap: u64) -> Option> { nonempty(bounded_read( - bzip2_rs::DecoderReader::new(Cursor::new(block)), + super::bzip2_rs::DecoderReader::new(Cursor::new(block)), cap, )) } @@ -191,12 +192,16 @@ fn decode_bzip2(block: &[u8], cap: u64) -> Option> { #[cfg(test)] mod tests { use super::*; + + fn is_nsis(data: &[u8]) -> bool { + super::is_nsis(&crate::Probe::whole(data)) + } use flate2::{write::DeflateEncoder, Compression}; use std::io::Write; /// A declared dictionary size is attacker input and is allocated UP FRONT. - /// Clamping it is what stops a small file committing gigabytes — measured at - /// 1.5 GB from a 766 KB installer, which aborted the process. + /// Clamping it is what stops a small file committing gigabytes (measured at + /// 1.5 GB from a 766 KB installer, which aborted the process). #[test] fn a_huge_declared_dictionary_is_clamped_to_the_budget() { // 4 GiB-1 declared, 1 MiB of budget. diff --git a/crates/exav-unpack/src/formats/ole_crypto.rs b/crates/exav-unpack/src/formats/ole_crypto.rs index dfec28f3..5413c946 100644 --- a/crates/exav-unpack/src/formats/ole_crypto.rs +++ b/crates/exav-unpack/src/formats/ole_crypto.rs @@ -13,7 +13,7 @@ //! project) are not. //! //! This module also implements the legacy **XOR obfuscation** scheme -//! (`wEncryptionType == 0`, [MS-OFFCRYPTO] §2.3.7) and both OOXML schemes — +//! (`wEncryptionType == 0`, [MS-OFFCRYPTO] §2.3.7) and both OOXML schemes: //! **standard** (AES-ECB, SHA-1 spun 50000×) and **agile** (AES-CBC with a //! per-blob KDF), §2.3.4. `ole.rs` routes an `EncryptionInfo` + //! `EncryptedPackage` compound file here and scans the recovered `.zip`; a @@ -24,7 +24,7 @@ use sha1::Sha1; /// Passwords exav tries automatically on an encrypted Office document before /// giving up and reporting it password-protected: the Excel default (opens with -/// no prompt — the common malware trick) and the empty password. +/// no prompt, the common malware trick) and the empty password. pub(crate) const DEFAULT_OFFICE_PASSWORDS: &[&str] = &["VelvetSweatshop", ""]; fn md5(data: &[u8]) -> [u8; 16] { @@ -230,7 +230,7 @@ const BLOCK: usize = 1024; /// Decrypt a whole `Workbook`/`Book` stream in place following the BIFF record /// structure: record headers stay cleartext, exempt records' data stays -/// cleartext, and every other record's data is RC4-decrypted — while the cipher +/// cleartext, and every other record's data is RC4-decrypted, while the cipher /// position advances continuously over the cleartext regions (re-keyed each /// 1024-byte block). See [MS-OFFCRYPTO] §2.3.6.1. fn decrypt_workbook(stream: &[u8], deriver: &KeyDeriver) -> Vec { @@ -290,7 +290,7 @@ fn decrypt_workbook(stream: &[u8], deriver: &KeyDeriver) -> Vec { /// (basic / CryptoAPI) and the legacy XOR obfuscation (`wEncryptionType == 0`). /// Returns the decrypted stream on the first password that verifies, or `None` /// if the scheme is unhandled or no password matched (the caller then reports -/// password-protected — never a silent clean). +/// password-protected, never a silent clean). pub(crate) fn try_decrypt_workbook(stream: &[u8], passwords: &[String]) -> Option> { let candidates: Vec<&str> = passwords .iter() @@ -350,7 +350,7 @@ const XOR_MATRIX: [u16; 105] = [ 0x4084, 0x8108, 0x1231, 0x2462, 0x48C4, ]; -/// `ROR(b1 ^ b2, 1)` over 8 bits — the per-element mixing of the XOR array. +/// `ROR(b1 ^ b2, 1)` over 8 bits: the per-element mixing of the XOR array. fn xor_ror(b1: u8, b2: u8) -> u8 { (b1 ^ b2).rotate_right(1) } @@ -492,7 +492,7 @@ fn decrypt_workbook_xor(stream: &[u8], password: &str) -> Vec { // ─────────────────────────── OOXML (.docx/.xlsx) AES ─────────────────────────── -use aes::cipher::{BlockDecrypt, KeyInit}; +use aes::cipher::{BlockCipherDecrypt, KeyInit}; /// AES-ECB decrypt `data` in place-copy (used by OOXML STANDARD encryption, which /// is ECB with no IV). Trailing partial block is left as-is. @@ -503,8 +503,7 @@ fn aes_ecb_decrypt(key: &[u8], data: &[u8]) -> Vec { if let Ok(c) = <$ty>::new_from_slice(key) { for chunk in out.chunks_mut(16) { if chunk.len() == 16 { - let mut b = - aes::cipher::generic_array::GenericArray::clone_from_slice(chunk); + let mut b = aes::Block::try_from(&*chunk).expect("a whole block"); c.decrypt_block(&mut b); chunk.copy_from_slice(&b); } @@ -702,7 +701,7 @@ impl HashAlg { /// trailing partial block untouched. All AES key lengths (128/192/256) supported. fn aes_cbc_decrypt(key: &[u8], iv: &[u8], data: &[u8]) -> Vec { use aes::cipher::block_padding::NoPadding; - use aes::cipher::{BlockDecryptMut, KeyIvInit}; + use aes::cipher::{BlockModeDecrypt, KeyIvInit}; let mut out = data.to_vec(); let n = out.len() - out.len() % 16; if iv.len() < 16 || n == 0 { @@ -711,7 +710,7 @@ fn aes_cbc_decrypt(key: &[u8], iv: &[u8], data: &[u8]) -> Vec { macro_rules! run { ($ty:ty) => {{ if let Ok(c) = >::new_from_slices(key, &iv[..16]) { - let _ = c.decrypt_padded_mut::(&mut out[..n]); + let _ = c.decrypt_padded::(&mut out[..n]); } }}; } @@ -1032,11 +1031,11 @@ mod tests { } fn aes_ecb_encrypt(key: &[u8], data: &[u8]) -> Vec { - use aes::cipher::{BlockEncrypt, KeyInit}; + use aes::cipher::{BlockCipherEncrypt, KeyInit}; let cipher = aes::Aes128::new_from_slice(key).unwrap(); let mut out = data.to_vec(); for chunk in out.chunks_mut(16) { - let mut b = aes::cipher::generic_array::GenericArray::clone_from_slice(chunk); + let mut b = aes::Block::try_from(&*chunk).unwrap(); cipher.encrypt_block(&mut b); chunk.copy_from_slice(&b); } @@ -1159,11 +1158,11 @@ mod tests { } fn aes256_cbc_encrypt(key: &[u8], iv: &[u8], data: &[u8]) -> Vec { - use aes::cipher::{block_padding::NoPadding, BlockEncryptMut, KeyIvInit}; + use aes::cipher::{block_padding::NoPadding, BlockModeEncrypt, KeyIvInit}; let mut out = data.to_vec(); let n = out.len(); let enc = >::new_from_slices(key, &iv[..16]).unwrap(); - enc.encrypt_padded_mut::(&mut out, n) + enc.encrypt_padded::(&mut out, n) .unwrap() .to_vec() } @@ -1305,7 +1304,7 @@ encryptedVerifierHashInput=\"{}\" encryptedVerifierHashValue=\"{}\" encryptedKey /// End-to-end: obfuscate a BIFF `Workbook` (BOF, XOR `FilePass`, an EICAR /// record, EOF) with the XOR scheme, then confirm the default-password path - /// recovers the cleartext EICAR — exercising the per-record index math. + /// recovers the cleartext EICAR, exercising the per-record index math. #[test] fn xor_obfuscation_recovers_eicar() { let pw = "VelvetSweatshop"; diff --git a/crates/exav-unpack/src/formats/onenote.rs b/crates/exav-unpack/src/formats/onenote.rs index 270467c3..9c3f6527 100644 --- a/crates/exav-unpack/src/formats/onenote.rs +++ b/crates/exav-unpack/src/formats/onenote.rs @@ -39,79 +39,39 @@ pub(crate) fn is_onenote(data: &[u8]) -> bool { data.starts_with(&ONENOTE_HEADER_GUID) } -/// Read the `cbLength` (u64) that follows the GUID at `guid_at`, returning the -/// data start offset and the length clamped to `end` (the bytes available). -fn record_at(guid_at: usize, cb_source: &[u8], data_start: u64, end: u64) -> Option<(u64, u64)> { - let cb_at = guid_at + FILE_DATA_STORE_GUID.len(); - let cb = cb_source.get(cb_at..cb_at + 8)?; - if data_start > end { - return None; - } - let declared = u64::from_le_bytes(cb.try_into().ok()?); - let avail = end - data_start; - let take = declared.min(avail); - (take > 0).then_some((data_start, take)) +/// Walk a OneNote section, each embedded file streamed from where it lies. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let members = stream_offsets(src); + crate::stream::stream_stored(&mut crate::source::Reader::new(src), budget, visit, members) } -/// Streaming variant: scan a bounded prefix for embedded-file records and return -/// each as `(name, offset, size)`; the (possibly large) payload is streamed by -/// the caller via seek+take. -pub(crate) fn stream_offsets( - source: &mut R, - max_buffer: u64, -) -> Result, LimitHit> { - use std::io::SeekFrom; - let file_len = source - .seek(SeekFrom::End(0)) - .map_err(|e| LimitHit::corrupt(format!("onenote: {e}")))?; - let scan = max_buffer.min(file_len) as usize; - let prefix = crate::read_at(source, 0, scan)?; - +/// Every embedded-file record as `(name, offset, size)`, the payloads left in +/// place for the caller to stream. `cbLength` is clamped to the bytes present. +pub(crate) fn stream_offsets(src: &dyn crate::source::ByteSource) -> Vec<(String, u64, u64)> { + let len = src.len(); let mut out = Vec::new(); - for (idx, guid_at) in memchr::memmem::Finder::new(&FILE_DATA_STORE_GUID) - .find_iter(&prefix) - .enumerate() - { - let data_start = (guid_at + FILE_DATA_STORE_GUID.len() + POST_GUID_HEADER) as u64; - if let Some((off, len)) = record_at(guid_at, &prefix, data_start, file_len) { - out.push((format!("onenote-embedded-{idx}"), off, len)); - } - } - Ok(out) -} - -pub(crate) fn extract_onenote( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - for (idx, guid_at) in memchr::memmem::Finder::new(&FILE_DATA_STORE_GUID) - .find_iter(data) - .enumerate() - { - let data_start = (guid_at + FILE_DATA_STORE_GUID.len() + POST_GUID_HEADER) as u64; - let Some((off, len)) = record_at(guid_at, data, data_start, data.len() as u64) else { - continue; - }; - let (off, len) = (off as usize, len as usize); - - budget.count_entry()?; - let cap = budget.reserve()?; - if len as u64 > cap { - return Err(LimitHit::new( - "onenote embedded file exceeds budget".to_string(), - )); - } - let member = data[off..off + len].to_vec(); - budget.commit(member.len() as u64); - if let Some(r) = visit( - Entry::new(format!("onenote-embedded-{idx}"), member), - budget, - ) { - return Ok(Some(r)); + let mut from = 0; + let mut idx = 0; + while let Some(guid_at) = src.find(&FILE_DATA_STORE_GUID, from, len) { + from = guid_at + 1; + let cb_at = guid_at + FILE_DATA_STORE_GUID.len(); + let data_start = cb_at + POST_GUID_HEADER; + let cb = src.window(cb_at, 8); + if cb.len() == 8 && data_start <= len { + let declared = + u64::from_le_bytes([cb[0], cb[1], cb[2], cb[3], cb[4], cb[5], cb[6], cb[7]]); + let take = declared.min((len - data_start) as u64); + if take > 0 { + out.push((format!("onenote-embedded-{idx}"), data_start as u64, take)); + } } + idx += 1; } - Ok(None) + out } #[cfg(test)] diff --git a/crates/exav-unpack/src/formats/partition.rs b/crates/exav-unpack/src/formats/partition.rs index 38b337aa..8de7baab 100644 --- a/crates/exav-unpack/src/formats/partition.rs +++ b/crates/exav-unpack/src/formats/partition.rs @@ -59,24 +59,19 @@ fn be_u32(d: &[u8], off: usize) -> u32 { /// conservatively-validated MBR). Used by `detect()` — placed last there so the /// weak MBR boot signature never shadows a more specific format. #[cfg(feature = "partition")] -pub(crate) fn is_partition(data: &[u8]) -> bool { - is_gpt(data) || is_apm(data) || is_mbr(data) +pub(crate) fn is_partition(p: &crate::Probe) -> bool { + is_gpt(p.head) || is_apm(p.head) || is_mbr_in(p.head, p.len as u64) } /// One partition as a byte range `[first_lba*512, end_sector*512)` clamped to /// `total_len`; `None` for a degenerate/out-of-range slice. -fn lba_range( - name: String, - first_lba: u64, - end_sector: u64, - total_len: u64, -) -> Option<(String, u64, u64)> { +fn lba_range(name: String, first_lba: u64, end_sector: u64, total_len: u64) -> Option { let start = first_lba.saturating_mul(SECTOR as u64).min(total_len); let end = end_sector.saturating_mul(SECTOR as u64).min(total_len); if end <= start { return None; } - Some((name, start, end - start)) + Some(Region::Member(name, start, end - start)) } /// The ClamAV alert name for an overlapping partition table in `data`, or @@ -90,15 +85,15 @@ fn lba_range( /// The three names are ClamAV's exactly, **including the doubled `n` in /// `MBRPartitionnIntersect`**. That is a typo upstream, but the name is the API: /// a gateway filtering on it would not match a corrected spelling. -pub fn intersection_alert(data: &[u8]) -> Option<&'static str> { - let mut cur = std::io::Cursor::new(data); - let regions = stream_offsets(&mut cur).ok()?; - // Only real partitions count; `stream_offsets` also emits zero-length marker - // regions for truncated tables, which are not partitions and cannot overlap. +pub fn intersection_alert(data: &dyn crate::source::ByteSource) -> Option<&'static str> { + let regions = stream_offsets(&mut crate::source::Reader::new(data)).ok()?; + // Only real partitions count, not the parts of a table left unwalked. let mut ranges: Vec<(u64, u64)> = regions .iter() - .filter(|(_, _start, len)| *len > 0) - .map(|(_, start, len)| (*start, start.saturating_add(*len))) + .filter_map(|r| match r { + Region::Member(_, start, len) if *len > 0 => Some((*start, start.saturating_add(*len))), + _ => None, + }) .collect(); if ranges.len() < 2 { return None; @@ -116,7 +111,8 @@ pub fn intersection_alert(data: &[u8]) -> Option<&'static str> { if !intersects { return None; } - let head = &data[..data.len().min(2 * SECTOR)]; + let head = data.window(0, 2 * SECTOR); + let head = &head[..]; if is_gpt(head) { Some("Heuristics.GPTPartitionIntersection") } else if is_apm(head) { @@ -126,14 +122,27 @@ pub fn intersection_alert(data: &[u8]) -> Option<&'static str> { } } -/// Reader-based streaming: parse the GPT/APM/MBR table (tiny, near the start) via -/// targeted reads and return each partition as `(name, offset, size)`. The -/// partition data itself — which is the bulk of a disk image — streams via -/// seek+take. Mirrors `extract_partition`'s ranges, validated against the true -/// file length rather than an in-memory buffer. +/// Walk a disk image, each partition streamed from where it lies. +#[cfg(feature = "partition")] +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let mut source = crate::source::Reader::new(src); + let members = stream_offsets(&mut source)?; + crate::stream::stream_stored(&mut source, budget, visit, members) +} + +/// Parse the GPT/APM/MBR table (tiny, near the start) via targeted reads and +/// return each partition as a region; the partition data, the bulk of a disk +/// image, is streamed by the caller. Ranges are validated against the file +/// length. A part of the table that was not walked is a region of its own, +/// which the caller reports. pub(crate) fn stream_offsets( source: &mut R, -) -> Result, LimitHit> { +) -> Result, LimitHit> { + const TOO_MANY_GPT: &str = "too many GPT partitions to walk them all"; let total_len = source .seek(std::io::SeekFrom::End(0)) .map_err(|e| LimitHit::corrupt(format!("partition: {e}")))?; @@ -141,17 +150,24 @@ pub(crate) fn stream_offsets( let mut out = Vec::new(); if is_gpt(&head) { let entries_lba = le_u64(&head, SECTOR + 72); - // Same truncation as the buffered path: clamping a parsed count silently - // discards the entries past the cap. `stream_offsets` has no reporting - // channel, so it emits a zero-length marker region that the caller - // surfaces rather than dropping the fact on the floor. + // Clamping a parsed count discards the entries past the cap, so the + // cap is reported. let declared = le_u32(&head, SECTOR + 80); let num_entries = declared.min(GPT_MAX_ENTRIES); if declared > GPT_MAX_ENTRIES { - out.push((format!(""), 0, 0)); + out.push(Region::Unwalked( + format!(""), + TOO_MANY_GPT, + )); } + // An implausible stride is not walked, but the partitions are still + // there. let entry_size = le_u32(&head, SECTOR + 84); if !(GPT_MIN_ENTRY_SIZE..=GPT_MAX_ENTRY_SIZE).contains(&entry_size) { + out.push(Region::Unwalked( + "".to_string(), + "implausible GPT entry size; partition table not walked", + )); return Ok(out); } let base = entries_lba.saturating_mul(SECTOR as u64); @@ -163,10 +179,20 @@ pub(crate) fn stream_offsets( let mut emitted = 0usize; for i in 0..num_entries as usize { if emitted >= MAX_PARTS { + out.push(Region::Unwalked( + format!(""), + TOO_MANY_GPT, + )); break; } let e = i.saturating_mul(entry_size as usize); + // An entry table running past the end: the partitions it describes + // exist in the layout but cannot be read here. let Some(entry) = table.get(e..e.saturating_add(56)) else { + out.push(Region::Unwalked( + "".to_string(), + "GPT partition entry table extends past the end of the image", + )); break; }; if entry[..16].iter().all(|&b| b == 0) { @@ -188,7 +214,10 @@ pub(crate) fn stream_offsets( let declared_map = be_u32(&head, SECTOR + 4); let map_entries = declared_map.min(MAX_PARTS as u32); if declared_map > MAX_PARTS as u32 { - out.push((format!(""), 0, 0)); + out.push(Region::Unwalked( + format!(""), + "too many APM partitions to walk them all", + )); } let mut emitted = 0usize; for i in 0..map_entries as usize { @@ -214,26 +243,7 @@ pub(crate) fn stream_offsets( out.push(m); } } - } else if head.get(510..512) == Some(&[0x55, 0xAA][..]) - && !is_volume_boot_record(&head) - && (0..4).any(|i| { - // is_mbr, but ranges validated against the true file length rather - // than the 1 KiB head, so a real (large) MBR image still qualifies. - // The guards must match `is_mbr` exactly: a check that lives in only - // one of the two paths leaves the other one wrong. - let e = 0x1BE + i * 16; - let status = head.get(e).copied().unwrap_or(0xFF); - let ptype = head.get(e + 4).copied().unwrap_or(0); - let lba_first = le_u32(&head, e + 8) as u64; - let sectors = le_u32(&head, e + 12); - (status == 0x00 || status == 0x80) - && ptype != 0x00 - && ptype != 0xEE - && sectors > 0 - && lba_first > 0 - && lba_first.saturating_mul(SECTOR as u64) < total_len - }) - { + } else if is_mbr_in(&head, total_len) { let mut emitted = 0usize; for i in 0..4 { let e = 0x1BE + i * 16; @@ -304,8 +314,9 @@ fn is_volume_boot_record(data: &[u8]) -> bool { && sectors_per_cluster.is_power_of_two() } -#[cfg(feature = "partition")] -fn is_mbr(data: &[u8]) -> bool { +/// Whether an image `len` bytes long, whose first sector is in `data`, opens +/// with a plausible MBR partition table. +fn is_mbr_in(data: &[u8], len: u64) -> bool { if data.get(510..512) != Some(&[0x55, 0xAA][..]) { return false; } @@ -329,242 +340,10 @@ fn is_mbr(data: &[u8]) -> bool { && ptype != 0xEE && sectors > 0 && lba_first > 0 - && (lba_first as usize).saturating_mul(SECTOR) < data.len() + && (lba_first as u64).saturating_mul(SECTOR as u64) < len }) } -/// Carve `[start_sector*512 .. end_sector*512]` (both clamped to the image), -/// charge it against the budget, and hand it to the visitor. Returns -/// `Ok(Some(r))` if the visitor stopped early. Empty/degenerate ranges are -/// skipped without consuming a file-count slot. -#[cfg(feature = "partition")] -fn emit_region( - data: &[u8], - name: String, - start_sector: u64, - end_sector: u64, - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let start = (start_sector as usize) - .saturating_mul(SECTOR) - .min(data.len()); - let end = (end_sector as usize).saturating_mul(SECTOR).min(data.len()); - let slice = data.get(start..end).unwrap_or(&[]); - if slice.is_empty() { - return Ok(None); - } - budget.count_entry()?; - let cap = budget.reserve()?; - if slice.len() as u64 > cap { - return Err(LimitHit::new(format!("partition '{name}' exceeds budget"))); - } - budget.commit(slice.len() as u64); - Ok(visit(Entry::new(name, slice.to_vec()), budget)) -} - -#[cfg(feature = "partition")] -pub(crate) fn extract_partition( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - // Prefer GPT/APM (self-describing, strong magic); fall back to the - // conservatively-validated MBR table (which also covers the hybrid case - // where a real MBR precedes a GPT but the GPT magic is absent/damaged). - if is_gpt(data) { - extract_gpt(data, budget, visit) - } else if is_apm(data) { - extract_apm(data, budget, visit) - } else if is_mbr(data) { - extract_mbr(data, budget, visit) - } else { - Ok(None) - } -} - -/// GPT: header at offset 512 gives the partition-entry array location, count, -/// and stride. Each used entry (non-zero type GUID) carves its LBA range. -#[cfg(feature = "partition")] -fn extract_gpt(data: &[u8], budget: &mut Budget, visit: Sink) -> Result, LimitHit> { - let hdr = SECTOR; // GPT header lives in LBA1. - // Real GPT header field offsets (relative to the header start): - // +72 partition_entries_lba (u64) - // +80 num_entries (u32) - // +84 entry_size (u32) - let entries_lba = le_u64(data, hdr + 72); - let declared_entries = le_u32(data, hdr + 80); - // Clamping here is the guard, but it is also a truncation: entries past the - // cap describe real regions of the image that will not be walked. Report it - // rather than let `.min()` quietly discard them. (The `emitted >= MAX_PARTS` - // check further down can then never fire — this clamp is what bounds the - // loop — but it is kept as a belt-and-braces bound.) - let num_entries = declared_entries.min(GPT_MAX_ENTRIES); - if declared_entries > GPT_MAX_ENTRIES { - budget.count_entry()?; - if let Some(r) = visit( - Entry::unsupported( - format!(""), - 0, - false, - "too many GPT partitions to walk them all", - ), - budget, - ) { - return Ok(Some(r)); - } - } - let entry_size = le_u32(data, hdr + 84); - if !(GPT_MIN_ENTRY_SIZE..=GPT_MAX_ENTRY_SIZE).contains(&entry_size) { - // Refusing to walk an implausible stride is right, but the partitions - // are still there — say so instead of returning "nothing found". - budget.count_entry()?; - if let Some(r) = visit( - Entry::unsupported( - "".to_string(), - 0, - false, - "implausible GPT entry size; partition table not walked", - ), - budget, - ) { - return Ok(Some(r)); - } - return Ok(None); - } - let base = (entries_lba as usize).saturating_mul(SECTOR); - let mut emitted = 0usize; - for i in 0..num_entries as usize { - if emitted >= MAX_PARTS { - // Report the cap. Breaking quietly leaves the remaining partitions - // unscanned while the image could still be called clean. - budget.count_entry()?; - if let Some(r) = visit( - Entry::unsupported( - format!(""), - 0, - false, - "too many GPT partitions to walk them all", - ), - budget, - ) { - return Ok(Some(r)); - } - break; - } - let e = base.saturating_add(i.saturating_mul(entry_size as usize)); - // An entry table running past EOF: the partitions it describes exist in - // the layout but cannot be read here. - let Some(entry) = data.get(e..e.saturating_add(56)) else { - budget.count_entry()?; - if let Some(r) = visit( - Entry::unsupported( - "".to_string(), - 0, - false, - "GPT partition entry table extends past the end of the image", - ), - budget, - ) { - return Ok(Some(r)); - } - break; - }; - // All-zero type GUID marks an unused slot. - if entry[..16].iter().all(|&b| b == 0) { - continue; - } - let first_lba = le_u64(entry, 32); - let last_lba = le_u64(entry, 40); - // Range is inclusive of last_lba: [first_lba*512 .. (last_lba+1)*512]. - let end_sector = last_lba.saturating_add(1); - emitted += 1; - let name = format!("gpt-part-{emitted}"); - if let Some(r) = emit_region(data, name, first_lba, end_sector, budget, visit)? { - return Ok(Some(r)); - } - } - Ok(None) -} - -/// APM (big-endian): sector 0 is Block0 (`ER`); sectors 1.. each hold one -/// partition-map entry (`PM`). The first entry's `mapEntries` bounds the count. -#[cfg(feature = "partition")] -fn extract_apm(data: &[u8], budget: &mut Budget, visit: Sink) -> Result, LimitHit> { - // mapEntries from the first entry (sector 1), capped. The cap is a real - // truncation — entries past it describe regions of the image that will not - // be walked — so report it rather than let `.min()` discard them quietly. - let declared = be_u32(data, SECTOR + 4); - let map_entries = declared.min(MAX_PARTS as u32); - if declared > MAX_PARTS as u32 { - budget.count_entry()?; - if let Some(r) = visit( - Entry::unsupported( - format!(""), - 0, - false, - "too many APM partitions to walk them all", - ), - budget, - ) { - return Ok(Some(r)); - } - } - let mut emitted = 0usize; - for i in 0..map_entries as usize { - if emitted >= MAX_PARTS { - break; - } - let base = (i + 1).saturating_mul(SECTOR); // entries start at sector 1 - // Each entry must begin with the `PM` signature; stop at the first miss. - if data.get(base..base + 2) != Some(b"PM".as_slice()) { - break; - } - let pblock_start = be_u32(data, base + 8) as u64; - let pblock_count = be_u32(data, base + 12) as u64; - if pblock_count == 0 { - continue; - } - let end_sector = pblock_start.saturating_add(pblock_count); - emitted += 1; - let name = format!("apm-part-{emitted}"); - if let Some(r) = emit_region(data, name, pblock_start, end_sector, budget, visit)? { - return Ok(Some(r)); - } - } - Ok(None) -} - -/// MBR: four 16-byte entries at offset 0x1BE. Empty (`type==0`) and protective -/// (`type==0xEE`, which defers to GPT) entries are skipped. -#[cfg(feature = "partition")] -fn extract_mbr(data: &[u8], budget: &mut Budget, visit: Sink) -> Result, LimitHit> { - let mut emitted = 0usize; - for i in 0..4 { - let e = 0x1BE + i * 16; - let ptype = data.get(e + 4).copied().unwrap_or(0); - if ptype == 0x00 || ptype == 0xEE { - continue; // empty slot, or protective MBR (defer to GPT) - } - let lba_first = le_u32(data, e + 8) as u64; - let sectors = le_u32(data, e + 12) as u64; - if sectors == 0 { - continue; - } - // Only carve entries whose start actually lands inside the image. - if (lba_first as usize).saturating_mul(SECTOR) >= data.len() { - continue; - } - let end_sector = lba_first.saturating_add(sectors); - emitted += 1; - let name = format!("mbr-part-{emitted}"); - if let Some(r) = emit_region(data, name, lba_first, end_sector, budget, visit)? { - return Ok(Some(r)); - } - } - Ok(None) -} - // The module compiles unconditionally (for the intersection heuristic), but // these drive `extract(Format::Partition, …)`, which needs the walker. #[cfg(all(test, feature = "partition"))] @@ -598,7 +377,7 @@ mod tests { // contents never get reached: a FAT image holding a zipped payload came // back LIMITS-EXCEEDED instead of infected. assert!( - !is_partition(&fat_boot_sector()), + !is_partition(&crate::Probe::whole(&fat_boot_sector())), "a FAT boot sector must not be taken for a partition table" ); } @@ -614,11 +393,11 @@ mod tests { b[446 + 4] = 0x0C; b[446 + 8..446 + 12].copy_from_slice(&0u32.to_le_bytes()); b[446 + 12..446 + 16].copy_from_slice(&2u32.to_le_bytes()); - assert!(!is_mbr(&b)); + assert!(!is_mbr_in(&b, b.len() as u64)); // The same entry one sector in is a normal partition. b[446 + 8..446 + 12].copy_from_slice(&1u32.to_le_bytes()); - assert!(is_mbr(&b)); + assert!(is_mbr_in(&b, b.len() as u64)); } use super::*; @@ -733,12 +512,10 @@ mod tests { #[test] fn garbage_and_truncated_do_not_panic() { - // Call the extractor directly (no catch_unwind) so a genuine panic would + // Call the table walk directly (no catch_unwind) so a genuine panic would // fail the test rather than being swallowed by the containment boundary. let run = |bytes: &[u8]| { - let mut budget = Budget::new(Limits::default()); - let _ = - extract_partition::(bytes, &mut budget, &mut |_, _| None); + let _ = stream_offsets(&mut std::io::Cursor::new(bytes)); }; // Random-ish bytes with the boot signature at 510. let mut g = vec![0u8; 1024]; diff --git a/crates/exav-unpack/src/formats/pdf.rs b/crates/exav-unpack/src/formats/pdf.rs index b9adf122..cd21f62a 100644 --- a/crates/exav-unpack/src/formats/pdf.rs +++ b/crates/exav-unpack/src/formats/pdf.rs @@ -838,7 +838,7 @@ trailer\n<< /Root 1 0 R >>\n%%EOF"; } body.push_str(" >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF"); let mut budget = Budget::new(Limits::default()); - let _ = extract(Format::Pdf, body.as_bytes(), &mut budget).unwrap(); + let _ = extract(Format::Pdf, &body.as_bytes(), &mut budget).unwrap(); } /// A PDF with no active content emits no JS/URI members (no false members). diff --git a/crates/exav-unpack/src/formats/pdf_parse/crypt.rs b/crates/exav-unpack/src/formats/pdf_parse/crypt.rs index 0936e266..55f572a2 100644 --- a/crates/exav-unpack/src/formats/pdf_parse/crypt.rs +++ b/crates/exav-unpack/src/formats/pdf_parse/crypt.rs @@ -1,7 +1,7 @@ use crate::formats::pdf_parse::types::Primitive; use std::collections::HashMap; -use aes::cipher::{BlockDecryptMut, BlockEncryptMut, KeyIvInit}; +use aes::cipher::{BlockModeDecrypt, BlockModeEncrypt, KeyIvInit}; use digest::Digest; use md5::Md5; @@ -164,7 +164,7 @@ impl Decoder { Err(PdfCryptError::UnsupportedRevision(level)) } - pub(crate) fn decrypt_stream(&self, obj_id: u32, data: &mut [u8]) { + pub(crate) fn decrypt_stream(&self, obj_id: u32, data: &mut Vec) { if data.is_empty() || self.method == CryptMethod::None { return; } @@ -174,22 +174,12 @@ impl Decoder { rc4_apply(&key, data); } CryptMethod::AESV2 => { - if data.len() < 16 { - return; - } - let (iv, ciphertext) = data.split_at_mut(16); - let iv = iv.to_vec(); let key = self.per_object_key(obj_id, Some(b"sAlT")); - aes_128_cbc_decrypt(&key, &iv, ciphertext); + aes_cbc_decrypt(&key, data); } CryptMethod::AESV3 => { - if data.len() < 16 { - return; - } - let (iv, ciphertext) = data.split_at_mut(16); - let iv = iv.to_vec(); let key = self.key[..32.min(self.key.len())].to_vec(); - aes_256_cbc_decrypt(&key, &iv, ciphertext); + aes_cbc_decrypt(&key, data); } CryptMethod::None => {} } @@ -233,7 +223,7 @@ fn resolve_crypt_method(dict: &CryptDict) -> Result<(u32, CryptMethod), PdfCrypt 1 => Ok((40, CryptMethod::V2)), // V==3 is the (undocumented) variable-length RC4 variant; in practice its // key derivation matches the published V==2 algorithm, so treat it the - // same — best effort. If the derived key doesn't verify against /U|/O we + // same, best effort. If the derived key doesn't verify against /U|/O we // report the PDF encrypted anyway, so a mismatch is never a silent clean. 2 | 3 => { if !dict.bits.is_multiple_of(8) { @@ -279,7 +269,7 @@ fn resolve_crypt_method(dict: &CryptDict) -> Result<(u32, CryptMethod), PdfCrypt } } -// ---- RC4 (RFC 6229) — kept as hand-rolled because the `rc4` crate requires +// ---- RC4 (RFC 6229), kept hand-rolled because the `rc4` crate requires // compile-time typenum key sizes, while PDF uses variable-length keys (40-2048 // bits). This is a well-understood 17-line cipher with no unsafe code. @@ -524,11 +514,9 @@ fn derive_key_v5_v6( } }; - let zero_iv = [0u8; 16]; let mut key_data = wrapped_key.clone(); - aes_256_cbc_decrypt( + unwrap_file_key( &intermediate_key[..32.min(intermediate_key.len())], - &zero_iv, &mut key_data, ); key_data.truncate(32); @@ -595,10 +583,8 @@ fn revision_6_kdf(password: &[u8], salt: &[u8], u: &[u8]) -> [u8; 32] { block_data.push(0); } let padded_len = block_data.len(); - let _ = encryptor.encrypt_padded_mut::( - &mut block_data, - padded_len, - ); + let _ = encryptor + .encrypt_padded::(&mut block_data, padded_len); let encrypted = &block_data[..total]; let sum: usize = encrypted[..16].iter().map(|&b| b as usize).sum(); @@ -643,30 +629,93 @@ fn revision_6_kdf(password: &[u8], salt: &[u8], u: &[u8]) -> [u8; 32] { type Aes128CbcDec = cbc::Decryptor; type Aes256CbcDec = cbc::Decryptor; -fn aes_128_cbc_decrypt(key: &[u8], iv: &[u8], data: &mut [u8]) { - if key.len() < 16 || iv.len() < 16 || data.is_empty() { - return; - } - let cipher = Aes128CbcDec::new_from_slices(&key[..16], &iv[..16]); - if let Ok(cipher) = cipher { - let _ = cipher.decrypt_padded_mut::(data); +/// Decrypt the `UE`/`OE` wrapped file key in place: AES-256, CBC with a zero +/// IV and no padding (ISO 32000-2). +fn unwrap_file_key(key: &[u8], data: &mut [u8]) { + use aes::cipher::block_padding::NoPadding; + let n = data.len() - data.len() % 16; + if let Ok(c) = Aes256CbcDec::new_from_slices(key, &[0u8; 16]) { + let _ = c.decrypt_padded::(&mut data[..n]); } } -fn aes_256_cbc_decrypt(key: &[u8], iv: &[u8], data: &mut [u8]) { - if key.len() < 32 || iv.len() < 16 { +/// Decrypt an AES stream or string in place: the 16-byte IV, then the CBC +/// ciphertext. `data` is left holding the plaintext alone, the IV removed. +/// +/// Stream extraction can include an end-of-line after the ciphertext, so only +/// the block-aligned part is decrypted. PKCS#7 padding is removed only when it +/// is valid: a crafted stream with bad padding still yields all its content. +/// Too short to hold an IV, or with a key of the wrong size, `data` is left as +/// it was. +fn aes_cbc_decrypt(key: &[u8], data: &mut Vec) { + use aes::cipher::block_padding::NoPadding; + if data.len() < 16 { return; } - // CBC needs whole 16-byte blocks, but PDF stream extraction can include a - // trailing EOL after the ciphertext — decrypt only the block-aligned prefix. - // NoPadding keeps the full plaintext (incl. any PKCS#7 padding bytes) so a - // signature near the tail stays visible and a crafted/edited stream with bad - // padding still yields its content. - let n = data.len() - data.len() % 16; - if n == 0 { + let (iv, body) = data.split_at_mut(16); + let n = body.len() - body.len() % 16; + let ok = match key.len() { + 16 => Aes128CbcDec::new_from_slices(key, iv) + .map(|c| c.decrypt_padded::(&mut body[..n]).is_ok()), + 32 => Aes256CbcDec::new_from_slices(key, iv) + .map(|c| c.decrypt_padded::(&mut body[..n]).is_ok()), + _ => return, + }; + if !matches!(ok, Ok(true)) { return; } - if let Ok(cipher) = Aes256CbcDec::new_from_slices(&key[..32], &iv[..16]) { - let _ = cipher.decrypt_padded_mut::(&mut data[..n]); + data.drain(..16); + data.truncate(n); + let pad = data.last().copied().unwrap_or(0) as usize; + if (1..=16).contains(&pad) + && pad <= data.len() + && data[data.len() - pad..].iter().all(|&b| b as usize == pad) + { + data.truncate(data.len() - pad); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PLAIN: &[u8] = b"stream plaintext, 33 bytes long!!"; + + fn unhex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + /// An AES stream or string is the IV, then the ciphertext of the padded + /// plaintext; what comes out is the plaintext alone. With the IV left in + /// front, a FlateDecode stream no longer starts with its zlib header and + /// is reported undecodable. Vectors from Python's `cryptography`. + #[test] + fn aes_decryption_yields_the_plaintext_alone() { + let v3 = Decoder { + key: vec![0x11; 32], + key_size: 32, + method: CryptMethod::AESV3, + }; + let v2 = Decoder { + key: vec![0x33; 16], + key_size: 16, + method: CryptMethod::AESV2, + }; + let cases = [ + (&v3, "222222222222222222222222222222221667316af8df34c59a9cfbb282f3f15390b837b680e02362584c9426a765787eba74d5ee5529da8aad289df5c742541b"), + (&v2, "22222222222222222222222222222222c0f2686b1de9823bb0b732f174856be9b2bde49ef0768d91635c34a59383f9e6064367ffe1064032238b380d4224a448"), + ]; + for (dec, hex) in cases { + // As extracted, and with the end-of-line a stream body can carry. + for tail in [&b""[..], b"\r\n"] { + let mut data = unhex(hex); + data.extend_from_slice(tail); + dec.decrypt_stream(7, &mut data); + assert_eq!(data, PLAIN, "{:?} with tail {tail:?}", dec.method); + } + } } } diff --git a/crates/exav-unpack/src/formats/pepack.rs b/crates/exav-unpack/src/formats/pepack.rs index fc77e7ca..a658e133 100644 --- a/crates/exav-unpack/src/formats/pepack.rs +++ b/crates/exav-unpack/src/formats/pepack.rs @@ -937,7 +937,7 @@ pub(crate) fn emulated_unpack( fn packer_name(p: Packer) -> &'static str { // The vendor's own capitalisation, and the single source of truth for it. // These strings reach the user twice — in the member name and, with - // `--alert-packed`, inside `Heuristics.Packed.` — and a gateway + // `--detect packed`, inside `Heuristics.Packed.` — and a gateway // filtering on an exact string needs the name the vendor uses. Spelling them // once here means the scanner never has to re-derive a display name, so the // two can't drift apart and a packer added below is reportable immediately. @@ -1277,7 +1277,7 @@ mod tests { for junk in [&b"MZ"[..], b"MZ\x00\x00", b"MZ\xff\xff\xff\xff garbage"] { let _ = is_pepack(junk); let mut b = Budget::new(Limits::default()); - let _ = extract(Format::PePacked, junk, &mut b); + let _ = extract(Format::PePacked, &junk, &mut b); } } diff --git a/crates/exav-unpack/src/formats/ppmd7/mod.rs b/crates/exav-unpack/src/formats/ppmd7/mod.rs index 16a9f086..8ed048a7 100644 --- a/crates/exav-unpack/src/formats/ppmd7/mod.rs +++ b/crates/exav-unpack/src/formats/ppmd7/mod.rs @@ -26,7 +26,9 @@ mod rangedec; mod tagged_offset; pub(crate) use model::Ppmd7; +#[cfg(feature = "rar")] pub(crate) use rangedec::RarRangeDecoder; +#[cfg(any(feature = "sevenz", test))] pub(crate) use rangedec::SevenZRangeDecoder; /// A range decoder symbol decode returned this end-of-stream marker. @@ -34,9 +36,13 @@ pub(crate) const SYM_END: i32 = -1; /// A range decoder symbol decode hit a model/range inconsistency. pub(crate) const SYM_ERROR: i32 = -2; +#[cfg_attr(not(feature = "rar"), allow(dead_code))] pub(crate) const PPMD7_MIN_ORDER: u32 = 2; +#[cfg_attr(not(feature = "rar"), allow(dead_code))] pub(crate) const PPMD7_MAX_ORDER: u32 = 64; +#[cfg_attr(not(feature = "rar"), allow(dead_code))] pub(crate) const PPMD7_MIN_MEM_SIZE: u32 = 2048; +#[cfg_attr(not(feature = "rar"), allow(dead_code))] pub(crate) const PPMD7_MAX_MEM_SIZE: u32 = u32::MAX - 12 * 3; /// Decode a complete 7z-framed PPMd7 stream of exactly `out_len` bytes. Used by @@ -78,6 +84,7 @@ pub(crate) trait RangeDec { fn out_of_data(&self) -> bool; /// Number of input bytes consumed so far (for resuming a shared cursor after /// a RAR3 PPMd→LZSS conversion). + #[cfg_attr(not(feature = "rar"), allow(dead_code))] fn bytes_consumed(&self) -> usize; } diff --git a/crates/exav-unpack/src/formats/ppmd7/model.rs b/crates/exav-unpack/src/formats/ppmd7/model.rs index a5501380..94295477 100644 --- a/crates/exav-unpack/src/formats/ppmd7/model.rs +++ b/crates/exav-unpack/src/formats/ppmd7/model.rs @@ -1224,11 +1224,13 @@ impl Ppmd7 { /// 0x20 reset flag): `PpmdRAR_RangeDec_Init` re-initialises the range coder /// over the bytes that follow, but the context tree / SEE / suballocator are /// carried over unchanged. + #[cfg_attr(not(feature = "rar"), allow(dead_code))] pub(crate) fn replace_rc(&mut self, rc: RC) { self.rc = rc; } /// Input bytes consumed by the range decoder so far. + #[cfg_attr(not(feature = "rar"), allow(dead_code))] pub(crate) fn rc_bytes_consumed(&self) -> usize { self.rc.bytes_consumed() } diff --git a/crates/exav-unpack/src/formats/ppmd7/rangedec.rs b/crates/exav-unpack/src/formats/ppmd7/rangedec.rs index 1fb3faf0..875d8f52 100644 --- a/crates/exav-unpack/src/formats/ppmd7/rangedec.rs +++ b/crates/exav-unpack/src/formats/ppmd7/rangedec.rs @@ -2,9 +2,8 @@ //! //! Two coders share the var.H model: //! -//! * [`SevenZRangeDecoder`] — the 7-Zip range coder, vendored from `ppmd-rust` -//! 1.4.0 (CC0-1.0 OR MIT-0). Used to validate the vendored model against -//! real `.7z` PPMd streams produced by the `7z` CLI. +//! * [`SevenZRangeDecoder`]: the 7-Zip range coder, vendored from `ppmd-rust` +//! 1.4.0 (CC0-1.0 OR MIT-0), for 7z and ZIP PPMd members. //! * [`RarRangeDecoder`] — RAR's Subbotin carry-less range coder. Its //! arithmetic (`Low`/`Code`/`Range`/`Bottom`, the `(Low ^ (Low+Range))` //! normalisation, the no-leading-byte init) is reproduced from libarchive's @@ -15,6 +14,8 @@ //! stays allocation-free w.r.t. the input. `out_of_data` records truncation: //! once the source is exhausted, reads yield 0 and the flag is set, letting the //! framing layer stop cleanly instead of looping. +// Each coder is dead in a build without the format that uses it. +#![cfg_attr(not(all(feature = "rar", feature = "sevenz")), allow(dead_code))] use super::RangeDec; @@ -23,8 +24,8 @@ const K_TOP_VALUE: u32 = 1 << 24; const PPMD_BIN_SCALE: u32 = 1 << 14; /// A byte source over an in-memory slice that yields 0 past the end and records -/// that it ran out (so the model can detect truncation). Only the test-only -/// `SevenZRangeDecoder` uses it; `RarRangeDecoder` owns its buffer directly. +/// that it ran out (so the model can detect truncation). `SevenZRangeDecoder` +/// uses it; `RarRangeDecoder` owns its buffer directly. struct ByteIn<'a> { data: &'a [u8], pos: usize, diff --git a/crates/exav-unpack/src/formats/pyc.rs b/crates/exav-unpack/src/formats/pyc.rs index ee433328..ccb3cad0 100644 --- a/crates/exav-unpack/src/formats/pyc.rs +++ b/crates/exav-unpack/src/formats/pyc.rs @@ -30,6 +30,17 @@ use crate::*; /// 16-byte header (`magic`, `bit_field`, `mtime`, `source_size`). const PYC_HEADER_LEN: usize = 16; +/// Walk a `.pyc`: its code object, streamed from where it lies. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let mut source = crate::source::Reader::new(src); + let members = stream_offsets(&mut source)?; + crate::stream::stream_stored(&mut source, budget, visit, members) +} + /// Reader-based streaming: the single `pyc-code` member is the whole file past /// the fixed header — streamed via seek+take with no buffering. pub(crate) fn stream_offsets( @@ -48,35 +59,6 @@ pub(crate) fn stream_offsets( )]) } -/// Emit the marshalled code body of a `.pyc` file (the whole file minus the -/// fixed 16-byte header) as a single `pyc-code` member. Bounds-checked; a file -/// shorter than the header (or with an empty body) yields no members. -pub(crate) fn extract_pyc( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - // Too short to hold a header + body: emit nothing (no panic). - if data.len() < PYC_HEADER_LEN { - return Ok(None); - } - let body = &data[PYC_HEADER_LEN..]; - if body.is_empty() { - return Ok(None); - } - budget.count_entry()?; - let cap = budget.reserve()?; - if body.len() as u64 > cap { - return Err(LimitHit::new("pyc body exceeds budget".to_string())); - } - let member = body.to_vec(); - budget.commit(member.len() as u64); - if let Some(r) = visit(Entry::new("pyc-code".into(), member), budget) { - return Ok(Some(r)); - } - Ok(None) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/exav-unpack/src/formats/rar.rs b/crates/exav-unpack/src/formats/rar.rs index d667a0be..d90960bd 100644 --- a/crates/exav-unpack/src/formats/rar.rs +++ b/crates/exav-unpack/src/formats/rar.rs @@ -1,8 +1,9 @@ -//! RAR extractor for the **stored** (uncompressed) method, RAR4 and RAR5, -//! implemented from the public RAR file-format field layout. RAR's compressed -//! methods (its LZ + PPMd) are not implemented, so a compressed or encrypted -//! member is reported as a member we could not extract (the caller still scans -//! the raw archive in place). +//! RAR extractor, RAR4 and RAR5, implemented from the public RAR file-format +//! field layout. Stored members are copied; RAR3 (unpack 29, LZ + PPMd) members +//! go to `rar3_unpack` and RAR5 members to `rar5_unpack`, solid groups +//! included. RAR 1.5/2.x compression, encrypted members and members split +//! across volumes are reported as members we could not extract (the caller +//! still scans the raw archive in place). //! //! Layouts used (magic-byte / header-field level only): //! - RAR4 marker `52 61 72 21 1A 07 00`; blocks = `HEAD_CRC u16, HEAD_TYPE u8, diff --git a/crates/exav-unpack/src/formats/rar3_unpack.rs b/crates/exav-unpack/src/formats/rar3_unpack.rs index e2a18296..01f13b7f 100644 --- a/crates/exav-unpack/src/formats/rar3_unpack.rs +++ b/crates/exav-unpack/src/formats/rar3_unpack.rs @@ -6,15 +6,13 @@ //! (Copyright (c) 2015, Nicholas Waples, BSD-2-Clause). The upstream copyright //! and full licence text live in the repo's top-level `NOTICE` file. //! -//! Scope: single-file, non-solid, non-encrypted RAR3 (unpack version 29) file -//! members compressed with the LZ method (RAR `-m1..-m5`). The RAR3 filter VM -//! and the standard E8/E8E9/delta/itanium/RGB/audio filters are ported so -//! x86/audio/image-filtered streams decode correctly. PPMd-mode blocks (the -//! secondary RAR3 method) are not decoded — a member that switches to PPMd is -//! reported as undecodable and recorded by the caller as metadata-only. The -//! older unpack15/20/26 formats are not handled (rardecode does not support -//! them either). Everything is bounds-checked and never panics on malformed -//! input. +//! Scope: non-encrypted RAR3 (unpack version 29) members, solid groups +//! included, compressed with LZ (RAR `-m1..-m5`) or PPMd blocks (decoded +//! through the vendored `ppmd7` model; see `read_ppmd_header`). The RAR3 filter +//! VM and the standard E8/E8E9/delta/itanium/RGB/audio filters are ported so +//! x86/audio/image-filtered streams decode correctly. The older unpack15/20/26 +//! formats are not handled (rardecode does not support them either). +//! Everything is bounds-checked and never panics on malformed input. use super::ppmd7::{ Ppmd7, RarRangeDecoder, PPMD7_MAX_MEM_SIZE, PPMD7_MAX_ORDER, PPMD7_MIN_MEM_SIZE, diff --git a/crates/exav-unpack/src/formats/rtf.rs b/crates/exav-unpack/src/formats/rtf.rs index 4cfa6188..76a1761b 100644 --- a/crates/exav-unpack/src/formats/rtf.rs +++ b/crates/exav-unpack/src/formats/rtf.rs @@ -190,7 +190,7 @@ mod tests { hex(payload) ); let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Rtf, rtf.as_bytes(), &mut budget).unwrap(); + let entries = extract(Format::Rtf, &rtf.as_bytes(), &mut budget).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].name, "rtf-object-1"); assert_eq!(entries[0].data, payload); @@ -204,7 +204,7 @@ mod tests { let (a, b) = h.split_at(10); let rtf = format!("{{\\rtf1{{\\object{{\\objdata\n{a}\r\n\\par {b}\n}}}}}}"); let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Rtf, rtf.as_bytes(), &mut budget).unwrap(); + let entries = extract(Format::Rtf, &rtf.as_bytes(), &mut budget).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].data, payload); } @@ -223,7 +223,7 @@ mod tests { hex(payload).to_uppercase() ); let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Rtf, rtf.as_bytes(), &mut budget).unwrap(); + let entries = extract(Format::Rtf, &rtf.as_bytes(), &mut budget).unwrap(); let folded = entries .iter() .find(|e| e.name == "rtf-objdata-hex-1") @@ -240,7 +240,7 @@ mod tests { let payload = b"MALWARETEST-lower"; let rtf = format!("{{\\rtf1{{\\object{{\\objdata {}}}}}}}", hex(payload)); let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Rtf, rtf.as_bytes(), &mut budget).unwrap(); + let entries = extract(Format::Rtf, &rtf.as_bytes(), &mut budget).unwrap(); assert!(entries .iter() .all(|e| !e.name.starts_with("rtf-objdata-hex"))); @@ -255,7 +255,7 @@ mod tests { hex(p2) ); let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Rtf, rtf.as_bytes(), &mut budget).unwrap(); + let entries = extract(Format::Rtf, &rtf.as_bytes(), &mut budget).unwrap(); assert_eq!(entries.len(), 2); assert_eq!(entries[0].data, p1); assert_eq!(entries[1].data, p2); diff --git a/crates/exav-unpack/src/formats/screnc.rs b/crates/exav-unpack/src/formats/screnc.rs index f9792467..24097b76 100644 --- a/crates/exav-unpack/src/formats/screnc.rs +++ b/crates/exav-unpack/src/formats/screnc.rs @@ -1,4 +1,4 @@ -//! Microsoft Script Encoder (`#@~^`) decoder — VBScript.Encode / JScript.Encode. +//! Microsoft Script Encoder (`#@~^`) decoder: VBScript.Encode / JScript.Encode. //! //! `screnc.exe` (and the `Scripting.Encoder` COM object) wraps a plaintext //! script into an opaque blob so the payload is not visible at rest. The blob is @@ -26,13 +26,13 @@ //! The decode/combination tables are the authentic published ones (Didier //! Stevens' public-domain `decode-vbe.py`, equivalently the classic `scrdec` //! algorithm). Every index is bounds-checked; truncated or hostile input yields -//! whatever decoded cleanly (or nothing) and never panics — the crate is +//! whatever decoded cleanly (or nothing) and never panics. The crate is //! `#![forbid(unsafe_code)]`. use crate::*; /// Blob start marker. -const MARKER: &[u8] = b"#@~^"; +pub(crate) const MARKER: &[u8] = b"#@~^"; /// Blob end marker. const TERMINATOR: &[u8] = b"^#~@"; /// Bytes between the marker and the encoded body: `<6-char length>` + `==`. @@ -242,8 +242,8 @@ fn decode_body(body: &[u8]) -> Vec { } /// True iff `data` contains the 4-byte `#@~^` Script Encoder marker. -pub(crate) fn looks_like_screnc(data: &[u8]) -> bool { - memchr::memmem::find(data, MARKER).is_some() +pub(crate) fn looks_like_screnc(p: &crate::Probe) -> bool { + p.find(MARKER).is_some() } /// Find every `#@~^ … ^#~@` blob, decode each, and emit the recovered plaintext @@ -263,7 +263,7 @@ pub(crate) fn extract_screnc( if body_start >= data.len() { continue; } - // Locate the terminator; without one the blob is truncated — skip it. + // Locate the terminator; without one the blob is truncated, so skip it. let term_rel = match memchr::memmem::find(&data[body_start..], TERMINATOR) { Some(t) => t, None => continue, @@ -305,6 +305,10 @@ pub(crate) fn extract_screnc( mod tests { use super::*; + fn looks_like_screnc(data: &[u8]) -> bool { + super::looks_like_screnc(&crate::Probe::whole(data)) + } + /// Encoded blob produced by inverting the published table for the plaintext /// below, and confirmed to decode to that plaintext by the authentic /// public-domain reference decoder (Didier Stevens' `decode-vbe.py`). It @@ -318,7 +322,7 @@ mod tests { #[test] fn decodes_real_published_vector() { let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Screnc, REAL_VECTOR, &mut budget).unwrap(); + let entries = extract(Format::Screnc, &REAL_VECTOR, &mut budget).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].name, "screnc-decoded"); assert_eq!( diff --git a/crates/exav-unpack/src/formats/sevenz/aes.rs b/crates/exav-unpack/src/formats/sevenz/aes.rs index 99bd4c57..08a99af1 100644 --- a/crates/exav-unpack/src/formats/sevenz/aes.rs +++ b/crates/exav-unpack/src/formats/sevenz/aes.rs @@ -5,7 +5,7 @@ //! a single hash context is updated with `salt || password_utf16le || counter` //! (`counter` a little-endian `u64`, incremented each round) `2^numCyclesPower` //! times, then finalised. The IV and salt travel in the coder properties. This -//! is the decrypt half only — no key wrapping, no RNG. +//! is the decrypt half only: no key wrapping, no RNG. //! //! Only the *data* layer is handled here (the common `7z a -p…` case, plaintext //! header). Archives with an AES-encrypted *header* (`-mhe=on`) fail to parse @@ -16,12 +16,12 @@ //! the downstream codec rejects (never a panic, never emitted as data). use crate::LimitHit; -use cbc::cipher::{block_padding::NoPadding, BlockDecryptMut, KeyIvInit}; +use cbc::cipher::{block_padding::NoPadding, BlockModeDecrypt, KeyIvInit}; use sha2::{Digest, Sha256}; use std::io::{self, Cursor, Read}; /// The special `numCyclesPower` meaning "use the passphrase (with salt) verbatim -/// as the key" — no hashing. +/// as the key", with no hashing. const NO_HASH_CYCLES: u32 = 0x3f; /// Upper bound on `numCyclesPower` we will run the KDF for. The 7-Zip default is @@ -133,7 +133,7 @@ pub(super) struct Aes7zReader { impl Aes7zReader { /// Build the decryptor for a 7zAES coder. `password` is the passphrase to /// try. Returns an error if the properties are malformed, no password was - /// supplied, or the KDF cost is out of range — the caller maps that to the + /// supplied, or the KDF cost is out of range; the caller maps that to the /// existing "encrypted / unsupported" outcome. pub(super) fn new( inner: Box, @@ -165,7 +165,7 @@ impl Aes7zReader { let full = ct.len() - (ct.len() % 16); ct.truncate(full); let pt = cbc::Decryptor::::new(&self.key.into(), &self.iv.into()) - .decrypt_padded_mut::(&mut ct) + .decrypt_padded::(&mut ct) .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "7z AES decrypt"))? .to_vec(); self.plain = Cursor::new(pt); @@ -185,7 +185,7 @@ impl Read for Aes7zReader { #[cfg(test)] mod tests { use super::*; - use cbc::cipher::BlockEncryptMut; + use cbc::cipher::BlockModeEncrypt; /// AES-256-CBC encrypt with NoPadding (test-only, mirrors the decrypt path) /// to build a ciphertext the reader must recover. @@ -194,7 +194,7 @@ mod tests { data.resize(data.len() + pad, 0); let n = data.len(); cbc::Encryptor::::new(key.into(), iv.into()) - .encrypt_padded_mut::(&mut data, n) + .encrypt_padded::(&mut data, n) .unwrap() .to_vec() } diff --git a/crates/exav-unpack/src/formats/sevenz/decode.rs b/crates/exav-unpack/src/formats/sevenz/decode.rs index 93f2ba63..5b43b97f 100644 --- a/crates/exav-unpack/src/formats/sevenz/decode.rs +++ b/crates/exav-unpack/src/formats/sevenz/decode.rs @@ -4,6 +4,7 @@ use super::header::Coder; use super::parse::*; use crate::LimitHit; +use lzma_rust2::filter::bcj::BcjReader; use std::io::{self, Read}; /// Return true if `id` is a codec we can decode. @@ -68,7 +69,7 @@ pub(super) fn wrap_coder( // Bound the up-front dictionary allocation. `expected_size` is a // header var-int and nothing else bounds it, so it is no ceiling on // its own; `max_buffer` is. Keep the declared output as the tighter - // of the two — a dictionary larger than the bytes it will be used to + // of the two: a dictionary larger than the bytes it will be used to // look back into cannot be consulted. let dict_size = crate::bounded_dict(dict_size, (expected_size as u64).min(max_buffer)); let decoder = lzma_rust2::LzmaReader::new_with_props( @@ -115,7 +116,7 @@ pub(super) fn wrap_coder( } x if x == ID_BZIP2 => { - let decoder = bzip2_rs::DecoderReader::new(inner); + let decoder = crate::formats::bzip2_rs::DecoderReader::new(inner); Ok(Box::new(decoder)) } @@ -124,20 +125,9 @@ pub(super) fn wrap_coder( Ok(Box::new(decoder)) } - x if x == ID_BCJ_X86 => { - let filter = BcjX86Filter::new(inner, max_buffer); - Ok(Box::new(filter)) - } - - x if x == ID_BCJ_ARM => { - let filter = BcjArmFilter::new(inner, max_buffer); - Ok(Box::new(filter)) - } - - x if x == ID_BCJ_ARM64 => { - let filter = BcjArm64Filter::new(inner, max_buffer); - Ok(Box::new(filter)) - } + x if x == ID_BCJ_X86 => Ok(Box::new(BcjReader::new_x86(inner, 0))), + x if x == ID_BCJ_ARM => Ok(Box::new(BcjReader::new_arm(inner, 0))), + x if x == ID_BCJ_ARM64 => Ok(Box::new(BcjReader::new_arm64(inner, 0))), x if x == ID_DELTA => { let distance = if coder.properties.is_empty() { @@ -188,7 +178,7 @@ impl Ppmd7ZReader { } // The model arena is `mem_size` bytes and the field is a full u32, so a // tiny archive can otherwise ask for ~4 GiB before decoding anything. - // The allocation is fallible, so this is not a crash — but the caller's + // The allocation is fallible, so this is not a crash, but the caller's // buffer cap is what says how much memory this scan may claim, and the // model has to answer to it like everything else. The ZIP method-98 path // clamps for the same reason. @@ -212,7 +202,7 @@ impl Ppmd7ZReader { fn init_and_decode(&mut self) -> Result<(), LimitHit> { // Bound both the compressed input and the decoded output by the global - // peak-buffer limit — PPMd amplifies heavily. + // peak-buffer limit: PPMd amplifies heavily. let (buf, truncated) = crate::bounded_read(&mut self.inner, self.max_buffer) .map_err(|e| LimitHit::corrupt(format!("7z: PPMD read: {e}")))?; if truncated { @@ -276,245 +266,6 @@ impl Read for Ppmd7ZReader { } } -// ─── BCJ x86 filter ──────────────────────────────────────────────────────── - -struct BcjX86Filter { - inner: R, - buf: Vec, - pos: usize, - max_buffer: u64, -} - -impl BcjX86Filter { - fn new(inner: R, max_buffer: u64) -> Self { - Self { - inner, - buf: Vec::new(), - pos: 0, - max_buffer, - } - } -} - -impl Read for BcjX86Filter { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if self.pos >= self.buf.len() { - let (data, truncated) = crate::bounded_read(&mut self.inner, self.max_buffer) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - if truncated { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "7z BCJ input exceeds max-buffer", - )); - } - self.buf = data; - apply_bcj_x86(&mut self.buf); - self.pos = 0; - } - - if self.pos >= self.buf.len() { - return Ok(0); - } - - let available = &self.buf[self.pos..]; - let to_copy = buf.len().min(available.len()); - buf[..to_copy].copy_from_slice(&available[..to_copy]); - self.pos += to_copy; - Ok(to_copy) - } -} - -fn apply_bcj_x86(data: &mut [u8]) { - let mut i = 0; - let mut ip: u32 = 0; - - while i + 4 < data.len() { - let b = data[i]; - - if b == 0xE8 || b == 0xE9 { - let rel = i32::from_le_bytes([data[i + 1], data[i + 2], data[i + 3], data[i + 4]]); - let addr = ip.wrapping_add(5).wrapping_add(rel as u32); - data[i + 1] = addr as u8; - data[i + 2] = (addr >> 8) as u8; - data[i + 3] = (addr >> 16) as u8; - data[i + 4] = (addr >> 24) as u8; - } - - let len = match b { - 0x0F => { - if i + 1 < data.len() { - match data[i + 1] { - 0x80..=0x8F => 6, - _ => 2, - } - } else { - 1 - } - } - 0x80..=0x83 => 6, - 0x88..=0x8B => 2, - 0xA1..=0xA3 => 5, - 0xB8..=0xBF => 5, - 0xC2..=0xC3 => 1, - 0xC8 => 4, - 0xE8 => 5, - 0xE9 => 5, - 0xEB => 2, - 0x68 => 5, - 0x6A => 2, - 0xFF => { - if i + 1 < data.len() && (data[i + 1] & 0x38) == 0x10 { - 6 - } else { - 2 - } - } - _ => 1, - }; - - i += len; - ip = ip.wrapping_add(len as u32); - } -} - -// ─── BCJ ARM filter ──────────────────────────────────────────────────────── - -struct BcjArmFilter { - inner: R, - buf: Vec, - pos: usize, - max_buffer: u64, -} - -impl BcjArmFilter { - fn new(inner: R, max_buffer: u64) -> Self { - Self { - inner, - buf: Vec::new(), - pos: 0, - max_buffer, - } - } -} - -impl Read for BcjArmFilter { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if self.pos >= self.buf.len() { - let (data, truncated) = crate::bounded_read(&mut self.inner, self.max_buffer) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - if truncated { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "7z BCJ input exceeds max-buffer", - )); - } - self.buf = data; - apply_bcj_arm(&mut self.buf); - self.pos = 0; - } - if self.pos >= self.buf.len() { - return Ok(0); - } - let available = &self.buf[self.pos..]; - let n = buf.len().min(available.len()); - buf[..n].copy_from_slice(&available[..n]); - self.pos += n; - Ok(n) - } -} - -fn apply_bcj_arm(data: &mut [u8]) { - let mut i = 0; - while i + 4 <= data.len() { - let w = u32::from_le_bytes([data[i], data[i + 1], data[i + 2], data[i + 3]]); - if (w >> 24) == 0xEB || (w >> 24) == 0xFB { - let rel = (w & 0x00FFFFFF) << 2; - let sign = if w & 0x00800000 != 0 { - 0xFF000000u32 - } else { - 0 - }; - let addr = (i as u32).wrapping_add(8).wrapping_add(sign | rel); - let corrected = addr.wrapping_sub(i as u32); - let new_rel = corrected >> 2; - data[i] = (new_rel & 0xFF) as u8; - data[i + 1] = ((new_rel >> 8) & 0xFF) as u8; - data[i + 2] = ((new_rel >> 16) & 0xFF) as u8; - data[i + 3] = (data[i + 3] & 0xF0) | ((new_rel >> 24) as u8 & 0x0F); - } - i += 4; - } -} - -// ─── BCJ ARM64 filter ────────────────────────────────────────────────────── - -struct BcjArm64Filter { - inner: R, - buf: Vec, - pos: usize, - max_buffer: u64, -} - -impl BcjArm64Filter { - fn new(inner: R, max_buffer: u64) -> Self { - Self { - inner, - buf: Vec::new(), - pos: 0, - max_buffer, - } - } -} - -impl Read for BcjArm64Filter { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if self.pos >= self.buf.len() { - let (data, truncated) = crate::bounded_read(&mut self.inner, self.max_buffer) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - if truncated { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "7z BCJ input exceeds max-buffer", - )); - } - self.buf = data; - apply_bcj_arm64(&mut self.buf); - self.pos = 0; - } - if self.pos >= self.buf.len() { - return Ok(0); - } - let available = &self.buf[self.pos..]; - let n = buf.len().min(available.len()); - buf[..n].copy_from_slice(&available[..n]); - self.pos += n; - Ok(n) - } -} - -fn apply_bcj_arm64(data: &mut [u8]) { - let mut i = 0; - while i + 4 <= data.len() { - let w = u32::from_le_bytes([data[i], data[i + 1], data[i + 2], data[i + 3]]); - if (w >> 26) == 0x05 || (w >> 26) == 0x17 { - let imm26 = w & 0x03FFFFFF; - let sign = if imm26 & 0x02000000 != 0 { - 0xFC000000u32 - } else { - 0 - }; - let addr = (i as u32).wrapping_add(sign | (imm26 << 2)); - let corrected = addr.wrapping_sub(i as u32); - let new_imm = corrected >> 2; - data[i] = (new_imm & 0xFF) as u8; - data[i + 1] = ((new_imm >> 8) & 0xFF) as u8; - data[i + 2] = ((new_imm >> 16) & 0xFF) as u8; - data[i + 3] = (data[i + 3] & 0xFC) | ((new_imm >> 24) as u8 & 0x03); - } - i += 4; - } -} - // ─── Delta filter ────────────────────────────────────────────────────────── struct DeltaFilter { @@ -547,16 +298,3 @@ impl Read for DeltaFilter { Ok(n) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn bcj_x86_basic() { - let mut data = vec![0xE8, 0x10, 0x00, 0x00, 0x00]; - data.resize(1024, 0); - apply_bcj_x86(&mut data); - assert_eq!(data[0], 0xE8); - } -} diff --git a/crates/exav-unpack/src/formats/sevenz/entry.rs b/crates/exav-unpack/src/formats/sevenz/entry.rs index 9c3829a4..d91d36ad 100644 --- a/crates/exav-unpack/src/formats/sevenz/entry.rs +++ b/crates/exav-unpack/src/formats/sevenz/entry.rs @@ -274,6 +274,18 @@ fn skip_reader(r: &mut dyn Read, n: u64) -> (u64, bool) { (skipped, false) } +/// Walk a 7z archive. Its header is at the end, so the container is read whole; +/// each member's decompressed output then streams, which is where a solid +/// block can be far larger than the archive. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let data = crate::stream::read_whole(Format::SevenZip, src, budget)?; + stream_sevenz(&data, budget, visit) +} + /// Streaming 7z extraction (pattern A): each file's solid block is built as a /// forward-only `Read`; the file is emitted by skipping to its offset and handing /// the visitor a `take(size)` window — the decompressed block is never buffered, @@ -281,12 +293,12 @@ fn skip_reader(r: &mut dyn Read, n: u64) -> (u64, bool) { /// buffered CRC/password path (rare). The 7z container itself is still parsed /// from `data` (the header lives at the end → random access), but its members' /// decompressed output streams. -pub(crate) fn stream_sevenz( +fn stream_sevenz( data: &[u8], budget: &mut Budget, - visit: crate::stream::StreamVisit, + visit: crate::stream::Visit, ) -> Result, LimitHit> { - use crate::stream::{visit_member, MemberMeta}; + use crate::stream::{emit_bytes, emit_stream, MemberMeta}; let archive = match parse_archive(data, &budget.passwords) { Ok(a) => a, Err(e) => { @@ -300,6 +312,7 @@ pub(crate) fn stream_sevenz( let meta = MemberMeta { name: "encrypted.7z".to_string(), comp_size: data.len() as u64, + size: None, encrypted: true, unsupported: Some("7z: encrypted header, unsupported codec"), }; @@ -331,6 +344,7 @@ pub(crate) fn stream_sevenz( let meta = MemberMeta { name, comp_size: file.size, + size: Some(file.size), encrypted: true, unsupported: Some(reason), }; @@ -429,16 +443,17 @@ pub(crate) fn stream_sevenz( let meta = MemberMeta { name, comp_size: file.size, + size: Some(file.size), encrypted: true, unsupported: None, }; - let mut cur = Cursor::new(fd); - visit(&meta, Some(&mut cur), budget) + emit_bytes(&meta, Some(fd), budget, visit)? } None => { let meta = MemberMeta { name, comp_size: file.size, + size: Some(file.size), encrypted: true, unsupported: Some("7z: wrong or missing password"), }; @@ -459,6 +474,7 @@ pub(crate) fn stream_sevenz( let meta = MemberMeta { name, comp_size: file.size, + size: Some(file.size), encrypted: false, unsupported: Some("7z: member starts past the peak-buffer limit"), }; @@ -492,6 +508,7 @@ pub(crate) fn stream_sevenz( let meta = MemberMeta { name, comp_size: file.size, + size: Some(file.size), encrypted: false, unsupported: Some(if failed { "7z: solid block failed to decode before this member" @@ -508,243 +525,13 @@ pub(crate) fn stream_sevenz( let meta = MemberMeta { name, comp_size: file.size, + size: Some(file.size), encrypted: false, unsupported: None, }; - if let Some(t) = visit_member(&meta, &mut window, budget, visit)? { + if let Some(t) = emit_stream(&meta, &mut window, budget, visit)? { return Ok(Some(t)); } } Ok(None) } - -/// Extract files from a 7z archive. -pub(crate) fn extract_sevenz( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let archive = match parse_archive(data, &budget.passwords) { - Ok(a) => a, - Err(e) => { - // If parsing fails (e.g. encrypted header), emit a single - // encrypted+unsupported entry so callers can detect the format. - let reason = e.reason.as_str(); - let is_encrypted = reason.contains("unsupported codec") - || reason.contains("unsupported 7z codec") - || reason.contains("AES") - || reason.contains("encrypted header"); - if is_encrypted { - budget.count_entry()?; - let entry = Entry::unsupported( - "encrypted.7z".to_string(), - data.len() as u64, - true, - "7z: encrypted header, unsupported codec", - ); - return Ok(visit(entry, budget)); - } - return Err(e); - } - }; - - // Clone candidate passwords out of the budget so they can be borrowed while - // `budget` is used mutably (count_entry/visit) in the loop below. - let passwords: Vec = budget.passwords.clone(); - - for (file_idx, file) in archive.files.iter().enumerate() { - if !file.has_stream || file.size == 0 { - continue; - } - - budget.count_entry()?; - - let name = if file.name.is_empty() { - format!("entry_{file_idx}") - } else { - file.name.clone() - }; - - // Find which block this file belongs to - match archive.stream_map.file_block.get(file_idx) { - Some(Some(block_idx)) => { - let block_idx = *block_idx; - let block = &archive.blocks[block_idx]; - - let aes = block_has_aes(block); - - // Check for unsupported codecs (encryption, etc.). AES is - // decodable when a password is available; otherwise unsupported. - if let Some(reason) = has_unsupported_codec(block, !passwords.is_empty()) { - let entry = Entry::unsupported(name, file.size, true, reason); - if let Some(r) = visit(entry, budget) { - return Ok(Some(r)); - } - continue; - } - - // Compute pack stream offset - let pack_stream_idx = archive - .stream_map - .block_first_pack_stream - .get(block_idx) - .copied() - .unwrap_or(0); - - if pack_stream_idx >= archive.pack_sizes.len() { - continue; - } - - // pack_pos / pack_stream_offsets / pack_sizes are all derived from - // attacker-controlled var-ints; a bare `+` panics under - // overflow-checks. An offset that overflows can only point past - // the file, so skip such a member. - let pack_offset = match SIGNATURE_HEADER_SIZE - .checked_add(archive.pack_pos) - .and_then(|v| { - v.checked_add(archive.stream_map.pack_stream_offsets[pack_stream_idx]) - }) - .and_then(|v| usize::try_from(v).ok()) - { - Some(v) => v, - None => continue, - }; - // Cover every packed stream of the folder (BCJ2 folders have - // four, laid end to end), keeping the individual sizes so the - // graph decoder can split them apart. - let n_pack = block.packed_streams.len().max(1); - let block_pack_sizes: Vec = (0..n_pack) - .filter_map(|k| archive.pack_sizes.get(pack_stream_idx + k).copied()) - .collect(); - let pack_size: usize = block_pack_sizes.iter().sum::() as usize; - - let pack_end = match pack_offset.checked_add(pack_size) { - Some(e) if e <= data.len() => e, - _ => continue, - }; - - let pack_data = &data[pack_offset..pack_end]; - - // Find which sub-stream index this file is within the block, and - // how many bytes of the (whole-block) output precede it. - let block_first_file = archive - .stream_map - .block_first_file - .get(block_idx) - .copied() - .unwrap_or(0); - // `block_first_file` is expected to be <= `file_idx`, but the - // stream map is built from attacker data — guard the subtraction - // so a stale/inconsistent value can't underflow-panic. - let sub_index = file_idx.saturating_sub(block_first_file); - let mut bytes_to_skip: u64 = 0; - for si in 0..sub_index { - let sub_file_idx = block_first_file + si; - if sub_file_idx < archive.files.len() { - // `size` comes from attacker sub-stream sizes; saturate the - // running total instead of overflow-panicking. - bytes_to_skip = - bytes_to_skip.saturating_add(archive.files[sub_file_idx].size); - } - } - let block_total: u64 = super::header::folder_out_size(block); - - // Candidate passwords: for an AES block, try each supplied - // passphrase; otherwise a single non-encrypted decode. - let candidates: Vec> = if aes { - passwords.iter().map(|p| Some(p.as_str())).collect() - } else { - vec![None] - }; - - let mut file_data: Option> = None; - for pw in candidates { - match decode_block( - block, - pack_data, - &block_pack_sizes, - block_total, - pw, - budget.limits.max_buffer_bytes, - ) { - Ok(decompressed) => { - let start = bytes_to_skip as usize; - if start >= decompressed.len() { - continue; - } - // `start` and `file.size` are attacker-controlled; - // saturate so the `+` can't overflow-panic before the - // `.min(len)` clamps it back into range. - let end = start - .saturating_add(file.size as usize) - .min(decompressed.len()); - let fd = decompressed[start..end].to_vec(); - // For an AES member, verify the stored CRC so a wrong - // passphrase (which yields plausible-looking garbage - // for a stored/copy stream) is rejected rather than - // emitted as data. - if aes && file.has_crc { - let mut h = crc32fast::Hasher::new(); - h.update(&fd); - let got = h.finalize(); - if got != file.crc { - continue; // wrong password — try the next one - } - } - file_data = Some(fd); - break; - } - // A decode error on an AES block is a wrong-password - // symptom; try the next candidate rather than aborting. - Err(_) if aes => continue, - Err(e) => return Err(e), - } - } - - match file_data { - Some(fd) => { - let entry = Entry::new(name, fd); - if let Some(r) = visit(entry, budget) { - return Ok(Some(r)); - } - } - // AES block we couldn't decrypt (no/incorrect password): - // report encrypted/unsupported, never silently clean. - None if aes => { - let entry = Entry::unsupported( - name, - file.size, - true, - "7z: wrong or missing password", - ); - if let Some(r) = visit(entry, budget) { - return Ok(Some(r)); - } - } - // Not encrypted, and still no data: the block decoded - // shorter than the sub-stream table says it should, so this - // member's offset falls past the end of it. In a solid block - // every member after the cut is in the same position, and - // dropping them quietly leaves a truncated archive looking - // like a short and harmless one. - None => { - let entry = Entry::unsupported( - name, - file.size, - false, - "7z: solid block ended before this member", - ); - if let Some(r) = visit(entry, budget) { - return Ok(Some(r)); - } - } - } - } - _ => { - // File has no associated block (empty file or directory) - } - } - } - - Ok(None) -} diff --git a/crates/exav-unpack/src/formats/sevenz/header.rs b/crates/exav-unpack/src/formats/sevenz/header.rs index 6f5238b0..92806551 100644 --- a/crates/exav-unpack/src/formats/sevenz/header.rs +++ b/crates/exav-unpack/src/formats/sevenz/header.rs @@ -991,7 +991,7 @@ mod tests { // is what actually regresses the fix). let _ = parse_archive(data, &[]); let mut budget = crate::Budget::new(crate::Limits::default()); - let _ = crate::extract(crate::Format::SevenZip, data, &mut budget); + let _ = crate::extract(crate::Format::SevenZip, &data, &mut budget); } /// SubStreamsInfo with an absurd `NID_NUM_UNPACK_STREAM` count: the running diff --git a/crates/exav-unpack/src/formats/sevenz/mod.rs b/crates/exav-unpack/src/formats/sevenz/mod.rs index 7e5cb4bd..08c1fa29 100644 --- a/crates/exav-unpack/src/formats/sevenz/mod.rs +++ b/crates/exav-unpack/src/formats/sevenz/mod.rs @@ -16,7 +16,7 @@ mod entry; mod header; mod parse; -pub(crate) use entry::{extract_sevenz, stream_sevenz}; +pub(crate) use entry::walk; #[cfg(test)] mod tests { @@ -34,7 +34,7 @@ mod tests { fn extract_entries(blob: &[u8]) -> HashMap> { let mut budget = Budget::new(Limits::default()); let entries = - extract(Format::SevenZip, blob, &mut budget).expect("extract should not fail"); + extract(Format::SevenZip, &blob, &mut budget).expect("extract should not fail"); entries.into_iter().map(|e| (e.name, e.data)).collect() } @@ -152,12 +152,12 @@ mod tests { let pws = passwords.iter().map(|s| s.to_string()).collect(); let mut budget = Budget::with_passwords(Limits::default(), pws); let entries = - extract(Format::SevenZip, blob, &mut budget).expect("extract should not fail"); + extract(Format::SevenZip, &blob, &mut budget).expect("extract should not fail"); entries.into_iter().map(|e| (e.name, e.data)).collect() } /// A real 7-Zip archive with an AES-encrypted *header* (`7z a -psecret - /// -mhe=on`) decrypts with the password — the encoded header itself is + /// -mhe=on`) decrypts with the password: the encoded header itself is /// decrypted so the file listing and content are recovered. #[cfg(feature = "decrypt")] #[test] @@ -187,7 +187,7 @@ mod tests { } /// Real 7-Zip-produced AES-256 archives (password `hunter2`) decrypt back to - /// the original bytes — end-to-end proof the SHA-256 KDF matches 7-Zip. The + /// the original bytes: end-to-end proof the SHA-256 KDF matches 7-Zip. The /// fixtures were made with `7z a -phunter2 -mhe=off`. #[cfg(feature = "decrypt")] #[test] @@ -291,4 +291,56 @@ mod tests { assert_eq!(entries.len(), 1); assert_eq!(entries[0].data, payload); } + + /// Code-like bytes: noise with x86 `call rel32` (high byte 0x00 or 0xFF, + /// the ones the filter converts), ARM `BL` and ARM64 `BL`/`ADRP` words + /// every few dozen bytes. + fn branchy_payload() -> Vec { + let mut seed = 0x2545_f491_u32; + let mut out = Vec::with_capacity(256 * 1024); + while out.len() < 256 * 1024 { + seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); + let r = seed.to_le_bytes(); + match seed % 5 { + 0 => out.extend([0xE8, r[0], r[1], 0x00, 0x00]), + 1 => out.extend([0xE9, r[0], r[1], 0xFF, 0xFF]), + 2 => out.extend([r[0], r[1], r[2], 0xEB]), + 3 => out.extend([r[0], r[1], r[2], 0x94]), + _ => out.extend([r[0] | 0x80, r[1], r[2], 0x90 | (r[3] & 0x0F)]), + } + out.extend(&r[..seed as usize % 4]); + } + out + } + + /// Each branch filter undoes what 7-Zip's encoder did: the member comes out + /// byte for byte as it went in. + #[test] + fn branch_filters_round_trip() { + use sevenz_rust2::{ArchiveEntry, ArchiveWriter, EncoderMethod}; + use std::io::Cursor; + + let payload = branchy_payload(); + for filter in [ + EncoderMethod::BCJ_X86_FILTER, + EncoderMethod::BCJ_ARM_FILTER, + EncoderMethod::BCJ_ARM64_FILTER, + ] { + let mut sink = Cursor::new(Vec::new()); + { + let mut w = ArchiveWriter::new(&mut sink).unwrap(); + w.set_content_methods(vec![EncoderMethod::LZMA2.into(), filter.into()]); + w.push_archive_entry( + ArchiveEntry::new_file("code.bin"), + Some(Cursor::new(payload.clone())), + ) + .unwrap(); + w.finish().unwrap(); + } + let mut budget = Budget::new(Limits::default()); + let entries = extract(Format::SevenZip, &sink.into_inner(), &mut budget).unwrap(); + assert_eq!(entries.len(), 1, "{filter:?}"); + assert!(entries[0].data == payload, "{filter:?} altered the member"); + } + } } diff --git a/crates/exav-unpack/src/formats/sfx.rs b/crates/exav-unpack/src/formats/sfx.rs index c275b223..c88dd22a 100644 --- a/crates/exav-unpack/src/formats/sfx.rs +++ b/crates/exav-unpack/src/formats/sfx.rs @@ -47,7 +47,7 @@ fn starts_with_exe(data: &[u8]) -> bool { /// Does a plausible ARJ main header start here? /// /// `60 EA` is two bytes, so across a megabyte of executable it turns up by -/// chance roughly once — and a false hit is not harmless: the carve hands the +/// chance roughly once, and a false hit is not harmless: the carve hands the /// ARJ extractor a fragment of the stub, extraction fails, and an ordinary file /// is reported `UNSCANNABLE`. A .NET assembly did exactly that. /// @@ -98,16 +98,70 @@ fn find_archive_from(data: &[u8], from: usize) -> Option<(usize, &'static str)> /// True when `data` looks like a self-extracting archive: an executable stub /// with an archive magic embedded past [`MIN_SFX_OFFSET`] (used by `detect`). -pub(crate) fn looks_like_sfx(data: &[u8]) -> bool { - if !starts_with_exe(data) { +pub(crate) fn looks_like_sfx(p: &crate::Probe) -> bool { + if !starts_with_exe(p.head) { return false; } - matches!(find_embedded_archive(data), Some((off, _)) if off > MIN_SFX_OFFSET) + let off = match p.source() { + None => find_embedded_archive(p.head).map(|(off, _)| off), + Some(_) => p.sfx_payload(), + }; + matches!(off, Some(off) if off > MIN_SFX_OFFSET) +} + +/// The search [`payload_offset`] makes, fed a window at a time by a pass +/// that reads the object for other searches too. +pub(crate) struct PayloadSearch { + found: Option>, +} + +impl PayloadSearch { + /// Bytes each window must run into the next: the longest magic and the + /// ARJ header check after it, so a match across the seam is seen whole. + pub(crate) const OVERLAP: usize = 16; + + pub(crate) fn new() -> Self { + PayloadSearch { found: None } + } + + /// Search window `w`, at `base` in the object, the object's last or not. + pub(crate) fn feed(&mut self, base: usize, w: &[u8], last: bool) { + if self.found.is_some() { + return; + } + // Offset 0 is the executable's own magic, never its payload. + let from = if base == 0 { 1 } else { 0 }; + if let Some((off, _)) = find_archive_from(w, from) { + // Near the end of a window a match may be cut short, and an ARJ + // candidate before it wrongly passed over; the next window sees + // both whole. + if last || off + Self::OVERLAP <= w.len() { + self.found = Some(Some(base + off)); + return; + } + } + if last { + self.found = Some(None); + } + } + + /// Whether the search has its answer. + pub(crate) fn done(&self) -> bool { + self.found.is_some() + } + + /// Where the first embedded archive starts, if any. + pub(crate) fn offset(&self) -> Option { + self.found.flatten() + } } /// Offset of the archive [`find_embedded_archive`] finds in the first `limit` /// bytes of `source`, read a window at a time rather than held whole. -fn payload_offset(source: &mut R, limit: u64) -> Result, LimitHit> { +pub(crate) fn payload_offset( + source: &mut R, + limit: u64, +) -> Result, LimitHit> { /// Bytes read per window. const WINDOW: usize = 1 << 20; /// Kept from one window into the next: the longest magic and the ARJ @@ -147,28 +201,29 @@ fn payload_offset(source: &mut R, limit: u64) -> Result(source: &mut R, limit: u64) -> Result { - if !starts_with_exe(&crate::read_at(source, 0, 4)?) { - return Ok(false); - } - Ok(payload_offset(source, limit)?.is_some_and(|off| off > MIN_SFX_OFFSET as u64)) +/// Walk a self-extractor: its appended archive, streamed from where it lies. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let mut source = crate::source::Reader::new(src); + let members = stream_offsets(&mut source)?; + crate::stream::stream_stored(&mut source, budget, visit, members) } -/// Reader-based streaming: find the appended archive's offset in a bounded -/// prefix (the stub is small), then stream the payload `[off, EOF)` via -/// seek+take — so a self-extracting installer with a multi-gigabyte payload is -/// scanned without buffering it. Matches [`extract_sfx`]'s single `sfx-payload` -/// member. +/// Find the appended archive's offset, searching as far as detection does (a +/// window at a time), then stream the payload `[off, EOF)` via seek+take, so a +/// self-extracting installer with a multi-gigabyte payload is scanned without +/// buffering it. A shorter search than detection's would type a file as an SFX +/// and then find nothing in it. pub(crate) fn stream_offsets( source: &mut R, - max_buffer: u64, ) -> Result, LimitHit> { let len = source .seek(std::io::SeekFrom::End(0)) .map_err(|e| LimitHit::corrupt(format!("sfx: {e}")))?; - let Some(off) = payload_offset(source, max_buffer.min(len))? else { + let Some(off) = payload_offset(source, len)? else { return Ok(Vec::new()); }; if off >= len { @@ -177,35 +232,14 @@ pub(crate) fn stream_offsets( Ok(vec![("sfx-payload".to_string(), off, len - off)]) } -pub(crate) fn extract_sfx( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - // First embedded archive past the stub, or nothing to carve. - let Some((off, _label)) = find_embedded_archive(data) else { - return Ok(None); - }; - let start = off.min(data.len()); - let slice = &data[start..]; - - budget.count_entry()?; - let cap = budget.reserve()?; - if slice.len() as u64 > cap { - return Err(LimitHit::new("sfx payload exceeds budget".to_string())); - } - let bytes = slice.to_vec(); - budget.commit(bytes.len() as u64); - if let Some(r) = visit(Entry::new("sfx-payload".to_string(), bytes), budget) { - return Ok(Some(r)); - } - Ok(None) -} - #[cfg(test)] mod tests { use super::*; + fn looks_like_sfx(data: &[u8]) -> bool { + super::looks_like_sfx(&crate::Probe::whole(data)) + } + /// An `MZ` stub of zero filler up to `payload_at`, then `payload`. fn mz_stub(payload: &[u8], payload_at: usize) -> Vec { let mut out = vec![b'M', b'Z']; @@ -214,6 +248,23 @@ mod tests { out } + /// The payload is found as far into the file as detection looks, whatever + /// the buffer limit: the search holds a window at a time. + #[test] + fn a_payload_past_the_buffer_limit_is_found() { + let mut zip = b"PK\x03\x04".to_vec(); + zip.extend_from_slice(b"local-header::MALWARETEST::body"); + let blob = mz_stub(&zip, 64 * 1024); + assert!(looks_like_sfx(&blob)); + let limits = Limits { + max_buffer_bytes: 16 * 1024, + ..Limits::default() + }; + let entries = extract(Format::Sfx, &blob, &mut Budget::new(limits)).unwrap(); + assert_eq!(entries.len(), 1); + assert!(entries[0].data.starts_with(b"PK\x03\x04")); + } + #[test] fn carves_appended_zip_payload() { let mut zip = b"PK\x03\x04".to_vec(); @@ -299,7 +350,7 @@ mod tests { fn truncated_and_garbage_do_not_panic() { let mut budget = Budget::new(Limits::default()); for input in [b"".as_slice(), b"M", b"MZ", &[0x60], b"PK\x03\x04"] { - let _ = extract(Format::Sfx, input, &mut budget).unwrap(); + let _ = extract(Format::Sfx, &input, &mut budget).unwrap(); } } } diff --git a/crates/exav-unpack/src/formats/sniff.rs b/crates/exav-unpack/src/formats/sniff.rs index 9e1eef3c..d8c50daa 100644 --- a/crates/exav-unpack/src/formats/sniff.rs +++ b/crates/exav-unpack/src/formats/sniff.rs @@ -19,7 +19,7 @@ //! tests in their own modules and are not detected in a build that excludes //! them. -use crate::Format; +use crate::{Format, Probe}; /// Does `data` look like `fmt`? /// @@ -27,7 +27,12 @@ use crate::Format; /// `false`, because "not recognised by a magic" is not the same claim as "not /// this format" and the structural sniffs own that answer. pub(crate) fn is(data: &[u8], fmt: Format) -> bool { - let d = data; + is_in(&Probe::whole(data), fmt) +} + +/// [`is`] for the object `p` looks at. +pub(crate) fn is_in(p: &Probe, fmt: Format) -> bool { + let d = p.head; match fmt { // `**ACE**` at offset 7 — after the header's CRC-16, size, type, flags. Format::Ace => d.get(7..14) == Some(b"**ACE**".as_slice()), @@ -44,8 +49,8 @@ pub(crate) fn is(data: &[u8], fmt: Format) -> bool { // The `conectix` footer cookie. A fixed-disk image carries only the // trailing copy, so the head alone is not enough to look for. Format::Vhd => { - d.len() >= 1024 - && (d.starts_with(b"conectix") || d[d.len() - 512..].starts_with(b"conectix")) + p.len >= 1024 + && (d.starts_with(b"conectix") || p.window(p.len - 512, 8).starts_with(b"conectix")) } // `compress(1)`: the magic plus a plausible maximum code width. @@ -156,7 +161,7 @@ pub(crate) fn is(data: &[u8], fmt: Format) -> bool { && dirs > 0 && toc >= 51 && toc < size - && size as u64 <= d.len() as u64, + && size as u64 <= p.len as u64, ) })() .unwrap_or(false) diff --git a/crates/exav-unpack/src/formats/swf.rs b/crates/exav-unpack/src/formats/swf.rs index 396b9c7f..3f1f7a36 100644 --- a/crates/exav-unpack/src/formats/swf.rs +++ b/crates/exav-unpack/src/formats/swf.rs @@ -9,110 +9,79 @@ //! decompressed body. //! //! `FWS` is already uncompressed, so there is nothing to unpack; only `CWS` and -//! `ZWS` are decoded. All sizing is bounded by the [`Budget`]: an attacker- -//! controlled `FileLength` can never drive an allocation — the LZMA decode target -//! and the zlib inflate are both clamped to the remaining budget, and a body that -//! would exceed it is rejected. -use crate::*; -use std::io::Cursor; +//! `ZWS` are decoded, as they are read. An attacker-controlled `FileLength` +//! never drives an allocation: the LZMA dictionary is clamped to the buffer +//! limit, and the output is bounded by the scan budget as it flows. +use std::io::{Cursor, Read}; -pub(crate) fn extract_swf( - data: &[u8], +use crate::source::{ByteSource, Reader}; +use crate::stream::{emit_stream, MemberMeta, Visit}; +use crate::{Budget, LimitHit}; + +/// Walk an SWF movie: the rebuilt `FWS` header, then the body decoded as it is +/// read. A non-SWF input or an `FWS` movie yields no member: its bytes are the +/// container's own, scanned as they are. +pub(crate) fn walk( + src: &dyn ByteSource, budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - // Need the full 8-byte header to know the encoding and FileLength. - if data.len() < 8 { - return Ok(None); - } - let sig = &data[0..3]; - // FWS is already uncompressed — nothing to unpack. Non-SWF magic: ignore. - if sig != b"CWS" && sig != b"ZWS" { + visit: Visit, +) -> Result, LimitHit> { + let hdr = src.window(0, 17); + if hdr.len() < 8 || (&hdr[0..3] != b"CWS" && &hdr[0..3] != b"ZWS") { return Ok(None); } - budget.count_entry()?; - let cap = budget.reserve()?; - - // Decompress the body (bytes 8..) per the signature. - let body = if sig == b"CWS" { - use flate2::read::ZlibDecoder; - let (out, truncated) = bounded_read(ZlibDecoder::new(&data[8..]), cap) - .map_err(|e| LimitHit::corrupt(format!("swf zlib: {e}")))?; - if truncated { - return Err(LimitHit::new( - "swf: decompressed size exceeds budget".into(), - )); - } - out - } else { - // ZWS / LZMA. If the stream doesn't decode cleanly, report the member as - // unsupported rather than failing the whole scan or panicking. - match decode_zws_lzma(data, cap) { - Ok((out, truncated)) => { - if truncated { - return Err(LimitHit::new( - "swf: decompressed size exceeds budget".into(), - )); - } - out - } - Err(()) => { - let comp = (data.len() as u64).saturating_sub(8); - return Ok(visit( - Entry::unsupported( - "movie.swf".into(), - comp, - false, - "SWF LZMA decode unsupported", - ), - budget, - )); - } - } - }; - - // Rebuild an FWS movie: the original 8-byte header with the signature forced - // to `FWS` (version + FileLength kept verbatim), then the decompressed body. - let mut fws = Vec::with_capacity(8 + body.len()); + let mut fws = Vec::with_capacity(8); fws.extend_from_slice(b"FWS"); - fws.extend_from_slice(&data[3..8]); - fws.extend_from_slice(&body); - budget.commit(fws.len() as u64); - Ok(visit(Entry::new("movie.swf".into(), fws), budget)) -} - -/// Decode a `ZWS` (LZMA) SWF body. After the 8-byte movie header the layout is a -/// 4-byte compressed-length u32 LE, then a 5-byte LZMA properties header (1 props -/// byte + u32 LE dictionary size), then the range-coded stream. The uncompressed -/// body size is `FileLength - 8`. Returns `(bytes, truncated)` — `truncated` true -/// if the output hit the budget cap — or `Err(())` if the stream can't be decoded -/// (short/malformed input, unsupported variant), so the caller can fall back to -/// an `unsupported` member instead of aborting. -fn decode_zws_lzma(data: &[u8], cap: u64) -> Result<(Vec, bool), ()> { - // 8 header + 4 comp-length + 5 LZMA props = 17 bytes before the stream. - if data.len() < 17 { - return Err(()); + fws.extend_from_slice(&hdr[3..8]); + let meta = MemberMeta { + name: "movie.swf".to_string(), + comp_size: (src.len() as u64).saturating_sub(8), + size: Some(u32::from_le_bytes([hdr[4], hdr[5], hdr[6], hdr[7]]) as u64), + encrypted: false, + unsupported: None, + }; + if &hdr[0..3] == b"CWS" { + let body = flate2::read::ZlibDecoder::new(Reader::range(src, 8, src.len())); + return emit_stream(&meta, &mut Cursor::new(fws).chain(body), budget, visit); } - let file_length = u32::from_le_bytes([data[4], data[5], data[6], data[7]]) as u64; - // Declared uncompressed body size. Untrusted: used only as the decode target, - // and memory is bounded by `bounded_read(cap)` regardless of how large it is. + // ZWS / LZMA: 8 header + 4 comp-length + 5 LZMA props before the stream. + let unsupported = MemberMeta { + unsupported: Some("SWF LZMA decode unsupported"), + ..meta.clone() + }; + if hdr.len() < 17 { + return Ok(visit(&unsupported, None, budget)); + } + let file_length = u32::from_le_bytes([hdr[4], hdr[5], hdr[6], hdr[7]]) as u64; let want = file_length.saturating_sub(8); - let props = data[12]; - let dict_size = crate::bounded_dict( - u32::from_le_bytes([data[13], data[14], data[15], data[16]]), - cap, + let dict = dict_size( + u32::from_le_bytes([hdr[13], hdr[14], hdr[15], hdr[16]]), + want, + budget.limits().max_buffer_bytes, ); - let stream = &data[17..]; - let reader = - lzma_rust2::LzmaReader::new_with_props(Cursor::new(stream), want, props, dict_size, None) - .map_err(|_| ())?; - bounded_read(reader, cap).map_err(|_| ()) + let stream = Reader::range(src, 17, src.len()); + match lzma_rust2::LzmaReader::new_with_props(stream, want, hdr[12], dict, None) { + Ok(body) => emit_stream(&meta, &mut Cursor::new(fws).chain(body), budget, visit), + Err(_) => Ok(visit(&unsupported, None, budget)), + } +} + +/// Pick the LZMA dictionary size for a `ZWS` movie. Both `declared` (the props +/// header's dictionary field) and `want` (the movie header's own `FileLength`, +/// less the 8-byte header) are attacker-controlled, and the dictionary is +/// allocated up front — so `want` is no ceiling on its own: a movie declaring +/// 4 GiB would buy itself a 4 GiB dictionary. `max_buffer` is the real bound; +/// `want` only ever tightens it, since a dictionary larger than the bytes it +/// will be used to look back into cannot be consulted. +fn dict_size(declared: u32, want: u64, max_buffer: u64) -> u32 { + crate::bounded_dict(declared, want.min(max_buffer)) } #[cfg(test)] mod tests { use super::*; + use crate::{extract, Format, Limits}; use flate2::{write::ZlibEncoder, Compression}; use std::io::Write; @@ -186,7 +155,7 @@ mod tests { let mut budget = Budget::new(Limits::default()); let entries = extract(Format::Swf, &zws, &mut budget).unwrap(); assert_eq!(entries.len(), 1); - assert_eq!(entries[0].unsupported, Some("SWF LZMA decode unsupported")); + assert!(entries[0].unsupported.is_some()); } #[test] @@ -197,4 +166,30 @@ mod tests { .unwrap() .is_empty()); } + + // A `ZWS` header carries two attacker-chosen sizes, and the LZMA dictionary + // is allocated before a single byte is decoded. Neither may set that size. + #[test] + fn dictionary_is_bounded_by_the_buffer_limit() { + let max_buffer = Limits::default().max_buffer_bytes; + // The sizes from a movie that asked for a 2.7 GiB dictionary by declaring + // a ~4 GiB FileLength: neither number may be believed. + assert_eq!( + dict_size(0xA1A1_C32B, 0xF04A_0957 - 8, max_buffer), + max_buffer as u32, + "a huge declared dictionary is clamped to the buffer limit" + ); + // A movie small enough to be honest keeps its own (smaller) dictionary. + assert_eq!( + dict_size(1 << 16, 1 << 20, max_buffer), + 1 << 16, + "a dictionary under both bounds is used as-is" + ); + // `want` still tightens: no point holding more history than output. + assert_eq!( + dict_size(u32::MAX, 1 << 20, max_buffer), + 1 << 20, + "the declared output bounds the dictionary when it is the smaller" + ); + } } diff --git a/crates/exav-unpack/src/formats/szdd.rs b/crates/exav-unpack/src/formats/szdd.rs index 534f5617..89ce2908 100644 --- a/crates/exav-unpack/src/formats/szdd.rs +++ b/crates/exav-unpack/src/formats/szdd.rs @@ -11,6 +11,7 @@ //! 12 bits, so truncated/hostile streams decode partially without panicking. use crate::*; +use std::io::Read; const SZDD_MAGIC: &[u8; 8] = b"SZDD\x88\xF0\x27\x33"; const KWAJ_MAGIC: &[u8; 8] = b"KWAJ\x88\xF0\x27\xD1"; @@ -33,63 +34,145 @@ pub(crate) fn extract_szdd( } } -/// Decode the SZDD LZSS stream starting at `input`, stopping at `declared` -/// output bytes or input exhaustion. Never panics: the ring index is masked and -/// every input read is `get`-guarded. -fn lzss_decompress(input: &[u8], declared: usize, cap: u64) -> Result, LimitHit> { - const WIN: usize = 4096; - let mut window = [0x20u8; WIN]; - let mut wpos = WIN - 16; // 4080 - let mut out = Vec::new(); - let mut i = 0usize; - while i < input.len() && out.len() < declared { - let flags = input[i]; - i += 1; - for bit in 0..8 { - if out.len() >= declared { +/// Walk an MS-Compress file. SZDD is decoded as it is read; KWAJ is decoded +/// whole. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + use crate::stream::{emit_stream, whole, MemberMeta}; + let hdr = src.window(0, 14); + if !hdr.starts_with(SZDD_MAGIC) { + return whole(Format::Szdd, src, budget, visit); + } + if hdr.len() < 14 { + return Err(LimitHit::corrupt("szdd: truncated header".to_string())); + } + let declared = u32::from_le_bytes([hdr[10], hdr[11], hdr[12], hdr[13]]) as u64; + budget.count_entry()?; + let mut body = crate::source::Reader::range(src, 14, src.len()); + let mut rdr = SzddReader::new(&mut body, declared); + let meta = MemberMeta { + name: szdd_name(&hdr), + comp_size: (src.len() - 14) as u64, + size: Some(declared), + encrypted: false, + unsupported: None, + }; + emit_stream(&meta, &mut rdr, budget, visit) +} + +/// SZDD's LZSS decoder as a `Read`: a 4 KiB ring preset to spaces, driven by +/// flag bytes whose bits select a literal (1) or a 12-bit offset and 4-bit +/// length back-reference (0). Output stops at the declared size or when the +/// input runs out. The ring index is masked, so hostile input cannot panic. +struct SzddReader<'a> { + src: &'a mut dyn Read, + window: [u8; 4096], + wpos: usize, + declared: u64, + produced: u64, + out: [u8; 18], // one token emits at most 18 bytes (back-reference length 3..18) + out_len: usize, + out_pos: usize, + flags: u32, + bits_left: u8, + done: bool, +} + +impl<'a> SzddReader<'a> { + fn new(src: &'a mut dyn Read, declared: u64) -> Self { + Self { + src, + window: [0x20u8; 4096], + wpos: 4096 - 16, + declared, + produced: 0, + out: [0u8; 18], + out_len: 0, + out_pos: 0, + flags: 0, + bits_left: 0, + done: false, + } + } + fn next_byte(&mut self) -> std::io::Result> { + let mut b = [0u8; 1]; + match self.src.read(&mut b)? { + 0 => Ok(None), + _ => Ok(Some(b[0])), + } + } + fn emit(&mut self, b: u8) { + self.out[self.out_len] = b; + self.out_len += 1; + self.window[self.wpos] = b; + self.wpos = (self.wpos + 1) & 4095; + self.produced += 1; + } +} + +impl Read for SzddReader<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let mut w = 0; + while w < buf.len() { + if self.out_pos < self.out_len { + buf[w] = self.out[self.out_pos]; + self.out_pos += 1; + w += 1; + continue; + } + if self.done || self.produced >= self.declared { break; } - if (flags >> bit) & 1 == 1 { - // Literal byte. - let Some(&b) = input.get(i) else { - return finish(out, cap); - }; - i += 1; - out.push(b); - window[wpos] = b; - wpos = (wpos + 1) & (WIN - 1); + self.out_len = 0; + self.out_pos = 0; + if self.bits_left == 0 { + match self.next_byte()? { + Some(f) => { + self.flags = f as u32; + self.bits_left = 8; + } + None => { + self.done = true; + break; + } + } + } + let is_literal = (self.flags & 1) == 1; + self.flags >>= 1; + self.bits_left -= 1; + if is_literal { + match self.next_byte()? { + Some(b) => self.emit(b), + None => { + self.done = true; + break; + } + } } else { - // Back-reference: two bytes -> 12-bit offset + 4-bit length. - let (Some(&b0), Some(&b1)) = (input.get(i), input.get(i + 1)) else { - return finish(out, cap); + let (b0, b1) = match (self.next_byte()?, self.next_byte()?) { + (Some(a), Some(b)) => (a, b), + _ => { + self.done = true; + break; + } }; - i += 2; let mut mpos = (b0 as usize) | (((b1 as usize) & 0xF0) << 4); let len = ((b1 as usize) & 0x0F) + 3; for _ in 0..len { - if out.len() >= declared { + if self.produced >= self.declared { break; } - let b = window[mpos & (WIN - 1)]; - out.push(b); - window[wpos] = b; - wpos = (wpos + 1) & (WIN - 1); + let b = self.window[mpos & 4095]; + self.emit(b); mpos = mpos.wrapping_add(1); } } - if out.len() as u64 > cap { - return Err(LimitHit::new("szdd member exceeds budget".to_string())); - } } + Ok(w) } - finish(out, cap) -} - -fn finish(out: Vec, cap: u64) -> Result, LimitHit> { - if out.len() as u64 > cap { - return Err(LimitHit::new("szdd member exceeds budget".to_string())); - } - Ok(out) } fn extract_szdd_inner( @@ -101,15 +184,19 @@ fn extract_szdd_inner( if data.len() < 14 { return Err(LimitHit::corrupt("szdd: truncated header".to_string())); } - let declared = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize; - let body = &data[14..]; + let declared = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as u64; + let mut body = &data[14..]; budget.count_entry()?; let cap = budget.reserve()?; - // Clamp the attacker-declared size so a huge value can't drive a giant - // allocation; the ratio/total caps still bound the real output. - let want = declared.min((cap as usize).saturating_add(1)); - let out = lzss_decompress(body, want, cap)?; + let mut out = Vec::new(); + // Reading a slice cannot fail. + let _ = SzddReader::new(&mut body, declared) + .take(cap.saturating_add(1)) + .read_to_end(&mut out); + if out.len() as u64 > cap { + return Err(LimitHit::new("szdd member exceeds budget".to_string())); + } budget.commit(out.len() as u64); if let Some(r) = visit(Entry::new(szdd_name(data), out), budget) { return Ok(Some(r)); diff --git a/crates/exav-unpack/src/formats/tar.rs b/crates/exav-unpack/src/formats/tar.rs index 73452ec9..e1681d6e 100644 --- a/crates/exav-unpack/src/formats/tar.rs +++ b/crates/exav-unpack/src/formats/tar.rs @@ -1,38 +1,31 @@ -#![allow(unused_imports)] -use crate::*; -use std::io::{BufReader, Cursor, Read, Seek, Write}; +//! tar: each member read off the source where it lies, a member the visitor +//! does not read skipped by seeking past it. +use std::io::{Seek, SeekFrom}; -/// True when an I/O error means the input simply ran out (a truncated stream), -/// as opposed to structurally invalid data. The `tar` crate surfaces the -/// end-of-input case both as `UnexpectedEof` and as a generic-kind error whose -/// message names it, so check both. -pub(crate) fn is_truncation(e: &std::io::Error) -> bool { - e.kind() == std::io::ErrorKind::UnexpectedEof || { - let m = e.to_string(); - m.contains("unexpected EOF") || m.contains("unexpected end") - } -} +use crate::source::{ByteSource, Reader}; +use crate::stream::{emit_stream, MemberMeta, Visit}; +use crate::{Budget, LimitHit}; -pub(crate) fn extract_tar( - data: &[u8], +pub(crate) fn walk( + src: &dyn ByteSource, budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let mut archive = ::tar::Archive::new(Cursor::new(data)); - let iter = archive - .entries() - .map_err(|e| LimitHit::new(format!("tar: {e}")))?; - for entry in iter { + visit: Visit, +) -> Result, LimitHit> { + let mut source = Reader::new(src); + source + .seek(SeekFrom::Start(0)) + .map_err(|e| LimitHit::corrupt(format!("tar seek: {e}")))?; + let mut archive = ::tar::Archive::new(source); + let entries = archive + .entries_with_seek() + .map_err(|e| LimitHit::corrupt(format!("tar: {e}")))?; + for entry in entries { budget.count_entry()?; let mut entry = match entry { Ok(e) => e, - // A truncated tar: the stream ran out mid-header / mid-skip. The - // entries before the cut were already yielded and scanned; the - // missing tail is absent, not hidden. exav scans for malware, it is - // not an integrity validator — stop cleanly rather than raising a - // not-fully-scanned verdict on a damaged-but-recovered archive. (The - // `tar` crate reports the mid-skip case with a generic error kind, so - // match the message too.) + // A truncated archive: entries before the cut were already scanned, + // and the missing tail is absent rather than hidden. exav scans for + // malware, it is not an integrity validator. Err(e) if is_truncation(&e) => return Ok(None), Err(e) => return Err(LimitHit::corrupt(format!("tar entry: {e}"))), }; @@ -40,16 +33,28 @@ pub(crate) fn extract_tar( .path() .map(|p| p.to_string_lossy().into_owned()) .unwrap_or_else(|_| "tar-entry".to_string()); - let cap = budget.reserve()?; - let (buf, truncated) = - bounded_read(&mut entry, cap).map_err(|e| LimitHit::new(format!("tar read: {e}")))?; - if truncated { - return Err(LimitHit::new(format!("tar member '{name}' exceeds budget"))); - } - budget.commit(buf.len() as u64); - if let Some(r) = visit(Entry::new(name, buf), budget) { - return Ok(Some(r)); + let size = entry.size(); + let meta = MemberMeta { + name, + comp_size: size, + size: Some(size), + encrypted: false, + unsupported: None, + }; + if let Some(t) = emit_stream(&meta, &mut entry, budget, visit)? { + return Ok(Some(t)); } } Ok(None) } + +/// True when an I/O error means the input simply ran out (a truncated stream), +/// as opposed to structurally invalid data. The `tar` crate surfaces the +/// end-of-input case both as `UnexpectedEof` and as a generic-kind error whose +/// message names it, so check both. +pub(crate) fn is_truncation(e: &std::io::Error) -> bool { + e.kind() == std::io::ErrorKind::UnexpectedEof || { + let m = e.to_string(); + m.contains("unexpected EOF") || m.contains("unexpected end") + } +} diff --git a/crates/exav-unpack/src/formats/tnef.rs b/crates/exav-unpack/src/formats/tnef.rs index 6d03a6c5..40c47339 100644 --- a/crates/exav-unpack/src/formats/tnef.rs +++ b/crates/exav-unpack/src/formats/tnef.rs @@ -32,6 +32,17 @@ fn attachment_name(bytes: &[u8]) -> String { String::from_utf8_lossy(&bytes[..end]).into_owned() } +/// Walk a TNEF blob, each attachment streamed from where it lies. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + let mut source = crate::source::Reader::new(src); + let members = stream_offsets(&mut source, budget.limits().max_buffer_bytes)?; + crate::stream::stream_stored(&mut source, budget, visit, members) +} + /// Streaming variant: walk the TLV records off a seekable source and return each /// attachment payload as `(name, offset, size)`; titles are read to name them. pub(crate) fn stream_offsets( @@ -90,68 +101,6 @@ pub(crate) fn stream_offsets( Ok(out) } -pub(crate) fn extract_tnef( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - let len = data.len(); - if len < 6 || !data.starts_with(&TNEF_SIGNATURE_LE) { - return Ok(None); - } - let le32 = |p: usize| u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]); - let mut pos = 6usize; // signature + key - let mut name: Option = None; - - while pos < len { - let level = data[pos]; - pos += 1; - if level == 0 { - break; - } - if pos + 8 > len { - break; - } - let tag = (le32(pos) & 0xFFFF) as u16; - let length = le32(pos + 4) as usize; - pos += 8; - if length == 0 { - continue; - } - let data_start = pos; - let clamped = length.min(len - data_start); - let truncated = clamped < length; - let record = &data[data_start..data_start + clamped]; - - if level == LVL_ATTACHMENT { - match tag { - ATT_ATTACHTITLE => name = Some(attachment_name(record)), - ATT_ATTACHDATA | ATT_ATTACHMENT => { - budget.count_entry()?; - let cap = budget.reserve()?; - if clamped as u64 > cap { - return Err(LimitHit::new("tnef attachment exceeds budget".to_string())); - } - let member_name = name - .clone() - .unwrap_or_else(|| "tnef-attachment".to_string()); - let bytes = record.to_vec(); - budget.commit(bytes.len() as u64); - if let Some(r) = visit(Entry::new(member_name, bytes), budget) { - return Ok(Some(r)); - } - } - _ => {} - } - } - pos = (data_start + clamped).saturating_add(2); - if truncated { - break; - } - } - Ok(None) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/exav-unpack/src/formats/udf.rs b/crates/exav-unpack/src/formats/udf.rs index 01b4a43b..9aca7f52 100644 --- a/crates/exav-unpack/src/formats/udf.rs +++ b/crates/exav-unpack/src/formats/udf.rs @@ -39,8 +39,8 @@ use std::borrow::Cow; use std::collections::HashSet; use std::io::{self, Read, Seek, SeekFrom}; -use crate::stream::{visit_member, MemberMeta, StreamVisit}; -use crate::{Budget, Entry, LimitHit, Sink}; +use crate::stream::{emit_stream, MemberMeta, Visit}; +use crate::{Budget, LimitHit}; /// UDF descriptors are addressed in 2048-byte sectors regardless of the logical /// block size the volume declares. @@ -921,42 +921,12 @@ fn walk( Ok(None) } -/// The UDF tree of an in-memory image, as [`Entry`]s. -pub(crate) fn extract_udf( - data: &[u8], - budget: &mut Budget, - visit: Sink, - seen_extents: &mut HashSet, -) -> Result, LimitHit> { - let max_dir = budget.limits.max_buffer_bytes; - let mut img = data; - walk(&mut img, max_dir, seen_extents, &mut |img, item| { - budget.count_entry()?; - Ok(match item { - Item::Unreadable { path, size, reason } => { - visit(Entry::unsupported(path, size, false, reason), budget) - } - Item::File { path, size, runs } => { - // Checked before the copy: unwritten extents are zeroes that - // cost nothing in the image and everything once copied out. - let cap = budget.reserve()?; - if size > cap { - return Err(LimitHit::new(format!("udf member '{path}' exceeds budget"))); - } - let content = read_runs(img, &runs)?; - budget.commit(content.len() as u64); - visit(Entry::new(path, content), budget) - } - }) - }) -} - /// The UDF tree of a seekable image, each file streamed from its runs. A no-op /// when the image carries no UDF recognition sequence. pub(crate) fn stream_udf( source: &mut R, budget: &mut Budget, - visit: StreamVisit, + visit: Visit, seen_extents: &mut HashSet, ) -> Result, LimitHit> { let head = crate::read_at(source, 0, RECOGNITION_END)?; @@ -975,6 +945,7 @@ pub(crate) fn stream_udf( let meta = MemberMeta { name: path, comp_size: size, + size: Some(size), encrypted: false, unsupported: Some(reason), }; @@ -984,6 +955,7 @@ pub(crate) fn stream_udf( let meta = MemberMeta { name: path, comp_size: size, + size: Some(size), encrypted: false, unsupported: None, }; @@ -992,7 +964,7 @@ pub(crate) fn stream_udf( runs: runs.into_iter(), cur: None, }; - visit_member(&meta, &mut reader, budget, &mut *visit) + emit_stream(&meta, &mut reader, budget, &mut *visit) } } }) diff --git a/crates/exav-unpack/src/formats/udif.rs b/crates/exav-unpack/src/formats/udif.rs index dcf701a8..afed0b72 100644 --- a/crates/exav-unpack/src/formats/udif.rs +++ b/crates/exav-unpack/src/formats/udif.rs @@ -61,8 +61,8 @@ impl Partition { local < self.total_sectors() } - fn run_for(&self, local_sec: u64) -> Option<&BlkxRun> { - self.runs.iter().find(|r| { + fn run_for(&self, local_sec: u64) -> Option<(usize, &BlkxRun)> { + self.runs.iter().enumerate().find(|(_, r)| { r.entry_type != BLK_TERM && r.entry_type != BLK_COMMENT && local_sec >= r.sector_start @@ -71,12 +71,18 @@ impl Partition { } } -struct DmgReader { +/// Decompressed runs kept, in bytes: a filesystem walk goes back and forth +/// between its B-tree nodes and the files' data. +const RUN_CACHE_BYTES: usize = 32 * 1024 * 1024; + +pub(crate) struct DmgReader { inner: R, sector_count: u64, file_size: u64, partitions: Vec, position: u64, + /// Decompressed runs by (partition, run), the most recently used last. + runs: Vec<((usize, usize), Vec)>, } impl DmgReader { @@ -130,9 +136,38 @@ impl DmgReader { file_size, partitions, position: 0, + runs: Vec::new(), }) } + /// The decompressed run `key`, decoded on first use. + fn run_data(&mut self, key: (usize, usize), run: &BlkxRun, file_pos: u64) -> io::Result<&[u8]> { + if let Some(i) = self.runs.iter().position(|(k, _)| *k == key) { + let hit = self.runs.remove(i); + self.runs.push(hit); + } else { + let expected = (run.sector_count as usize).saturating_mul(512); + // A run's compressed form is not much larger than the run itself. + if expected > MAX_RUN_BYTES || run.data_length > 2 * MAX_RUN_BYTES as u64 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "block decompressed size exceeds cap", + )); + } + self.inner.seek(SeekFrom::Start(file_pos))?; + let mut compressed = vec![0u8; run.data_length as usize]; + self.inner.read_exact(&mut compressed)?; + let decompressed = decompress(run.entry_type, &compressed, expected) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let mut held: usize = self.runs.iter().map(|(_, d)| d.len()).sum(); + while held + decompressed.len() > RUN_CACHE_BYTES && !self.runs.is_empty() { + held -= self.runs.remove(0).1.len(); + } + self.runs.push((key, decompressed)); + } + Ok(&self.runs.last().expect("just pushed").1) + } + fn virtual_disk_size(&self) -> u64 { self.sector_count.saturating_mul(512) } @@ -151,31 +186,33 @@ impl Read for DmgReader { let vsec = self.position / 512; let sec_offset = self.position % 512; - let part = self + let (pi, part) = self .partitions .iter() - .find(|p| p.contains_sector(vsec)) + .enumerate() + .find(|(_, p)| p.contains_sector(vsec)) .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "no partition"))?; let local_sec = vsec - part.sector_base; - let run = part + let (ri, run) = part .run_for(local_sec) .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "no run"))?; + let run = run.clone(); + let file_data_offset = part.file_data_offset; let bytes_into_run = (local_sec - run.sector_start) .saturating_mul(512) .saturating_add(sec_offset); let run_total_bytes = run.sector_count.saturating_mul(512); let available_in_run = run_total_bytes.saturating_sub(bytes_into_run); - let to_read = buf.len().min(available_in_run as usize); + let mut to_read = buf.len().min(available_in_run as usize); match run.entry_type { BLK_ZERO | BLK_IGNORE => { buf[..to_read].fill(0); } BLK_RAW => { - let file_pos = part - .file_data_offset + let file_pos = file_data_offset .checked_add(run.data_offset) .and_then(|p| p.checked_add(bytes_into_run)) .filter(|&p| p.saturating_add(to_read as u64) <= self.file_size) @@ -186,8 +223,7 @@ impl Read for DmgReader { self.inner.read_exact(&mut buf[..to_read])?; } BLK_ADC | BLK_ZLIB | BLK_BZIP2 | BLK_LZFSE | BLK_LZMA => { - let file_pos = part - .file_data_offset + let file_pos = file_data_offset .checked_add(run.data_offset) .ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidData, "block offset overflow") @@ -201,18 +237,7 @@ impl Read for DmgReader { "compressed block extends past end of file", )); } - let expected = (run.sector_count as usize).saturating_mul(512); - if expected > MAX_RUN_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "block decompressed size exceeds cap", - )); - } - self.inner.seek(SeekFrom::Start(file_pos))?; - let mut compressed = vec![0u8; run.data_length as usize]; - self.inner.read_exact(&mut compressed)?; - let decompressed = decompress(run.entry_type, &compressed, expected) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let decompressed = self.run_data((pi, ri), &run, file_pos)?; let start = bytes_into_run as usize; if start >= decompressed.len() { return Err(io::Error::new( @@ -222,6 +247,8 @@ impl Read for DmgReader { } let end = (start + to_read).min(decompressed.len()); buf[..end - start].copy_from_slice(&decompressed[start..end]); + // A run that decoded short ends here: the next read reports it. + to_read = end - start; } t => { return Err(io::Error::new( @@ -236,6 +263,59 @@ impl Read for DmgReader { } } +impl Seek for DmgReader { + fn seek(&mut self, to: SeekFrom) -> io::Result { + let pos = match to { + SeekFrom::Start(p) => Some(p), + SeekFrom::End(d) => self.virtual_disk_size().checked_add_signed(d), + SeekFrom::Current(d) => self.position.checked_add_signed(d), + } + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "seek outside the disk"))?; + self.position = pos; + Ok(pos) + } +} + +/// The disk a DMG holds, read on demand: a UDIF image's runs are decompressed +/// as they are reached, and anything else is the disk itself. +pub(crate) enum Disk { + Udif(DmgReader), + Raw(R), +} + +pub(crate) fn disk(mut src: R) -> Result, LimitHit> { + let len = src + .seek(SeekFrom::End(0)) + .map_err(|e| LimitHit::corrupt(format!("UDIF: seek to end: {e}")))?; + let mut sig = [0u8; 4]; + let koly = len >= KOLY_SIZE + && src.seek(SeekFrom::Start(len - KOLY_SIZE)).is_ok() + && src.read_exact(&mut sig).is_ok() + && u32::from_be_bytes(sig) == KOLY_MAGIC; + if koly { + return DmgReader::open(src).map(Disk::Udif); + } + Ok(Disk::Raw(src)) +} + +impl Read for Disk { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Disk::Udif(d) => d.read(buf), + Disk::Raw(r) => r.read(buf), + } + } +} + +impl Seek for Disk { + fn seek(&mut self, to: SeekFrom) -> io::Result { + match self { + Disk::Udif(d) => d.seek(to), + Disk::Raw(r) => r.seek(to), + } + } +} + fn decompress( entry_type: u32, compressed: &[u8], @@ -251,7 +331,7 @@ fn decompress( .map_err(|e| LimitHit::corrupt(format!("UDIF zlib: {e}")))?; } BLK_BZIP2 => { - bzip2_rs::DecoderReader::new(Cursor::new(compressed)) + super::bzip2_rs::DecoderReader::new(Cursor::new(compressed)) .take(cap) .read_to_end(&mut out) .map_err(|e| LimitHit::corrupt(format!("UDIF bzip2: {e}")))?; @@ -455,30 +535,3 @@ fn parse_mish(data: &[u8]) -> Result { runs, }) } - -/// Decompress a UDIF DMG into a raw disk image. -/// -/// Handles all UDIF codecs (zlib, bzip2, LZFSE, LZMA/XZ, ADC, raw) in pure -/// Rust. Vendored from `dmg-core` 0.1.2 (Apache-2.0, Albert Hui / SecurityRonin). -pub(crate) fn decompress_udif(data: &[u8], max_buffer: u64) -> Result, LimitHit> { - if data.len() < 512 || &data[data.len() - 512..data.len() - 508] != b"koly" { - if data.len() as u64 > max_buffer { - return Err(LimitHit::new("UDIF image exceeds max-buffer".to_string())); - } - return Ok(data.to_vec()); - } - - let mut reader = DmgReader::open(Cursor::new(data))?; - - // The whole disk image is decompressed into one buffer (the filesystem - // crates need random access over it). Bound it by the global peak-buffer - // limit — the declared disk size is attacker-controlled, so a plain - // `read_to_end` is otherwise unbounded. - let (raw, truncated) = crate::bounded_read(&mut reader, max_buffer) - .map_err(|e| LimitHit::corrupt(format!("UDIF decompress: {e}")))?; - if truncated { - return Err(LimitHit::new("UDIF image exceeds max-buffer".to_string())); - } - - Ok(raw) -} diff --git a/crates/exav-unpack/src/formats/upx.rs b/crates/exav-unpack/src/formats/upx.rs index 532a56d9..76ccd9af 100644 --- a/crates/exav-unpack/src/formats/upx.rs +++ b/crates/exav-unpack/src/formats/upx.rs @@ -14,7 +14,7 @@ //! Stored, NRV2B, NRV2D, NRV2E, LZMA and DEFLATE are all decompressed. The NRV //! decoders here are written independently from the **published NRV2B/D/E //! bitstream description** (a bit-flag literal/match scheme) and verified purely -//! by **black-box round-trip against the `upx` CLI's own output** — they are NOT +//! by **black-box round-trip against the `upx` CLI's own output**. They are NOT //! ported, translated, or otherwise derived from the source of UCL (Oberhumer, //! GPL) or ClamAV (GPL); the code expression (bit reader, variable naming, //! control flow) is exav's own. The LZMA 2-byte property framing is likewise @@ -70,7 +70,7 @@ pub(crate) fn find_packheader(data: &[u8]) -> Option { /// Strong evidence the image really is UPX-packed: the `UPX!` PackHeader magic /// or the conventional `UPX0`/`UPX1` section names. Used to decide whether a -/// failed unpack is worth REPORTING — speculative calls on unrelated bytes must +/// failed unpack is worth REPORTING: speculative calls on unrelated bytes must /// stay silent, but a file that advertises itself as UPX and whose payload we /// could not recover is content present-and-unexamined. fn looks_upx(data: &[u8]) -> bool { @@ -89,7 +89,7 @@ fn looks_upx(data: &[u8]) -> bool { /// /// Before reporting, the stub is *run*: a patched header defeats the static /// reader, which needs the header to find the compressed blocks, but it does not -/// defeat the stub — the stub still has to decompress the image to run it. Only +/// defeat the stub, which still has to decompress the image to run it. Only /// when that comes back empty too is the file reported unopened. fn report_packed( data: &[u8], @@ -132,8 +132,8 @@ fn adler32(data: &[u8]) -> u32 { /// `find_packheader` does not model. /// /// That function understands the `l_info`/`p_info`/`b_info` chain. A PE can -/// instead carry a 32-byte PackHeader — `UPX!`, version, format, method, level, -/// the two Adler-32 sums, then `u_len`/`c_len`/`u_file_size` — with the +/// instead carry a 32-byte PackHeader (`UPX!`, version, format, method, level, +/// the two Adler-32 sums, then `u_len`/`c_len`/`u_file_size`) with the /// compressed stream following immediately. Read as an `l_info`, that header's /// `c_len`/`u_file_size` fields land where a `b_info` block's sizes are expected /// and fail validation, so the file matched no unpacker at all and scanned @@ -227,7 +227,7 @@ const CLAMAV_DOS_STUB_B64: &str = concat!( const STUB_LEN: usize = 208; /// Decode [`CLAMAV_DOS_STUB_B64`]. The input is a constant this crate controls, -/// so a failure here is a build-time mistake, not bad input — the unit test +/// so a failure here is a build-time mistake, not bad input. The unit test /// below pins both the length and an Adler-32 of the result. fn clamav_dos_stub() -> Option<[u8; STUB_LEN]> { use base64::Engine; @@ -250,13 +250,13 @@ fn clamav_dos_stub() -> Option<[u8; STUB_LEN]> { /// section's RVA). The original PE header block survives in the tail of that /// run, which is what makes the rebuild possible at all. /// -/// The transformations are ClamAV's, recovered by diffing against its output — -/// black-box, not from its source: +/// The transformations are ClamAV's, recovered by diffing against its output +/// (black-box, not from its source): /// /// * `TimeDateStamp` := `"CLAM"`, /// * `FileAlignment` := `SectionAlignment`, /// * every section: `VirtualSize` = `SizeOfRawData` = `VirtualSize` rounded up -/// to `SectionAlignment`, and `PointerToRawData` := `VirtualAddress` — i.e. +/// to `SectionAlignment`, and `PointerToRawData` := `VirtualAddress`, so /// the file is flattened so a file offset equals its RVA. fn rebuild_clamav_pe(image: &[u8], first_rva: u32) -> Option> { // The original header block sits near the end of the decompressed run; take @@ -349,14 +349,14 @@ pub(crate) fn extract_upx( let Some(li) = find_packheader(data) else { // No `l_info` chain. Try the PackHeader layout before giving up; if that // fails too and the image still advertises UPX, the payload is there and - // unexamined — say so rather than returning clean. + // unexamined: say so rather than returning clean. budget.count_entry()?; let cap = budget.reserve()?; if let Some(image) = packheader_image(data, cap) { // Hand over the rebuilt PE rather than the raw run: it carries the // same bytes plus the headers, so ordinary pattern signatures still - // match, and ClamAV's hash signatures over packed malware — computed - // across its own rebuilt artifact — match too. Fall back to the raw + // match, and ClamAV's hash signatures over packed malware, computed + // across its own rebuilt artifact, match too. Fall back to the raw // image if the headers could not be recovered, since scanning the // content still beats reporting nothing. let out = first_section_rva(data) @@ -390,7 +390,7 @@ pub(crate) fn extract_upx( } let dstart = pos + 12; if sz_cpr == 0 || dstart.saturating_add(sz_cpr) > data.len() { - break; // malformed chain — stop, keep what we have + break; // malformed chain: stop, keep what we have } if out.len() as u64 + sz_unc as u64 > cap { return Err(LimitHit::new( @@ -409,7 +409,7 @@ pub(crate) fn extract_upx( } else if method == M_LZMA { match lzma_block_decompress(cdata, sz_unc) { Ok(b) => b, - Err(_) => break, // unfilter/variant we can't handle — keep what we have + Err(_) => break, // unfilter/variant we can't handle: keep what we have } } else if method == M_DEFLATE { match deflate_block_decompress(cdata, sz_unc) { @@ -417,7 +417,7 @@ pub(crate) fn extract_upx( Err(_) => break, } } else { - break; // unknown method — stop the walk, keep what we have + break; // unknown method: stop the walk, keep what we have }; if block.len() != sz_unc { break; @@ -427,7 +427,7 @@ pub(crate) fn extract_upx( } // NOTE: `b_ftid` (x86 call/jmp filter) is intentionally not reversed. Modern // `upx` does not filter ELF (the id is 0), and the filter only rewrites the - // 4-byte operands of CALL/JMP — strings and data, which signatures mostly key + // 4-byte operands of CALL/JMP; strings and data, which signatures mostly key // on, are recovered exactly regardless. Reversing it for PE needs a nonzero // `addvalue` derived from the unpacked PE layout, and no filtered-ELF test // vector exists to validate against, so it is deferred rather than shipped @@ -572,12 +572,11 @@ fn copy_match( Ok(false) } -/// Upper bound for any NRV gamma (offset/length) value. UPX match offsets and -/// lengths are bounded by the output size (≤ `max_buffer_bytes`, ~256 MiB), far -/// below this; a larger value means a corrupt stream and would only ever be a -/// rejected match. Capping keeps the doubling loops and the downstream -/// `(m-3)*256` arithmetic from overflowing on hostile input. -const NRV_GAMMA_MAX: usize = u32::MAX as usize; +/// Upper bound for any NRV gamma (offset/length) value: the end-of-stream +/// marker's, the largest for which `(m-3)*256 + byte` fits in a `u32`. Real +/// offsets and lengths are far below it. The cap keeps the doubling loops and +/// that arithmetic from overflowing, on 32-bit targets too. +const NRV_GAMMA_MAX: usize = 0xff_ffff + 3; /// NRV2B offset-gamma: `m=1; do { m=2m+bit } while(!bit)`. #[inline] @@ -721,7 +720,7 @@ fn nrv2d_decompress(src: &[u8], dst_len: usize) -> Result, LimitHit> { Ok(out) } -/// NRV2E (UCL) decompressor — UPX's *default* method. Same offset code as NRV2D, +/// NRV2E (UCL) decompressor, UPX's *default* method. Same offset code as NRV2D, /// but a distinct match-length prefix tree: seeded bit 1 ⇒ len 1+bit (1..2); /// else next bit 1 ⇒ len 3+bit (3..4); else a gamma length with a `+3` bias. /// Long-match threshold `0x500`. @@ -775,7 +774,7 @@ mod rebuild_tests { /// The stub is a compatibility constant: a large family of ClamAV hash /// signatures is computed over a rebuilt PE that begins with exactly these /// bytes. If it is ever "tidied up", those signatures stop matching and the - /// failure is silent — detections just stop happening — so pin it by + /// failure is silent (detections just stop happening), so pin it by /// checksum rather than by quoting the text back. #[test] fn clamav_dos_stub_is_byte_stable() { @@ -831,6 +830,39 @@ mod rebuild_tests { mod tests { use super::*; + /// An NRV2B gamma stream for `m`: the bits after its leading 1, each + /// followed by a stop bit (1 on the last), packed as `Br` reads them. + fn nrv_gamma_bits(m: u64) -> Vec { + let width = 64 - m.leading_zeros(); + let mut bits = Vec::new(); + for i in (0..width - 1).rev() { + bits.push((m >> i) & 1 == 1); + bits.push(i == 0); + } + bits.resize(bits.len().div_ceil(32) * 32, false); + bits.chunks(32) + .flat_map(|w| { + let v = w.iter().fold(0u32, |acc, &b| acc << 1 | b as u32); + v.to_le_bytes() + }) + .collect() + } + + /// The cap is the end-of-stream marker's gamma, so `(m - 3) * 256 + byte` + /// fits in a `u32`, and a value past it is refused alike on 32- and 64-bit. + #[test] + fn nrv_gamma_is_capped_at_the_end_marker() { + let end = 0xff_ffff + 3; + for (m, ok) in [(2, true), (end, true), (end + 1, false), (1 << 31, false)] { + let src = nrv_gamma_bits(m); + let got = nrv_gamma(&mut Br::new(&src)); + assert_eq!(got.is_ok(), ok, "m = {m:#x}"); + if ok { + assert_eq!(got.unwrap() as u64, m); + } + } + } + #[test] fn deflate_block_roundtrips() { // UPX method 15 is a raw DEFLATE stream; verify the decoder recovers the @@ -860,8 +892,8 @@ mod tests { } /// NRV2D, NRV2E and LZMA each pack the SAME marker ELF, so all must - /// decompress to byte-identical output containing the marker — validating the - /// decoders against real `upx` output. + /// decompress to byte-identical output containing the marker. This checks + /// the decoders against real `upx` output. #[test] fn nrv2d_nrv2e_lzma_decode_byte_exact() { let nrv2b = include_bytes!("../../tests/fixtures/upx_nrv2b_min.bin"); @@ -876,7 +908,7 @@ mod tests { max_compression_ratio: u64::MAX, ..Default::default() }); - extract(Format::Upx, data, &mut b) + extract(Format::Upx, &data, &mut b) .unwrap() .into_iter() .next() @@ -910,6 +942,6 @@ mod tests { let data = include_bytes!("../../tests/fixtures/upx_nrv2b_min.bin"); let mut budget = Budget::new(Limits::default()); // Chop the compressed data mid-block; must not panic. - let _ = extract(Format::Upx, &data[..data.len() - 80], &mut budget); + let _ = extract(Format::Upx, &&data[..data.len() - 80], &mut budget); } } diff --git a/crates/exav-unpack/src/formats/uuencode.rs b/crates/exav-unpack/src/formats/uuencode.rs index 465f96a4..5e883a20 100644 --- a/crates/exav-unpack/src/formats/uuencode.rs +++ b/crates/exav-unpack/src/formats/uuencode.rs @@ -161,21 +161,93 @@ pub(crate) fn extract_uuencode( /// True if `data` (leading whitespace skipped) opens a uuencode stream: a /// `begin `/`begin-base64 ` line with a matching `end`/`====` terminator later. /// Conservative — used by [`crate::detect`], which is the single source of truth. -pub(crate) fn looks_like_uuencode(data: &[u8]) -> bool { +pub(crate) fn looks_like_uuencode(p: &crate::Probe) -> bool { + let data = p.head; let head = trim_ascii_start(&data[..data.len().min(4096)]); let base64 = head.starts_with(b"begin-base64 "); if !base64 && !head.starts_with(b"begin ") { return false; } // Require a terminator so a stray "begin " prose line isn't claimed. - lines(data).any(|l| { - let t = trim_ascii(l); - if base64 { - t.starts_with(b"====") - } else { - t == b"end" + let mut scan = Terminator::new(base64); + match p.source() { + None => { + scan.feed(data); + } + Some(src) => src.chunks(0, src.len(), &mut |_, chunk| !scan.feed(chunk)), + } + scan.finish() +} + +/// Looks for a line that, trimmed of ASCII whitespace, is `end` (or starts +/// with `====` for base64), over bytes fed a piece at a time. +struct Terminator { + base64: bool, + found: bool, + /// Past the line's leading whitespace. + in_text: bool, + /// Bytes of the line from its first non-whitespace one. + pos: usize, + /// The first four of those. + first: [u8; 4], + /// Index of the last non-whitespace byte among them. + last_text: usize, +} + +impl Terminator { + fn new(base64: bool) -> Self { + Terminator { + base64, + found: false, + in_text: false, + pos: 0, + first: [0; 4], + last_text: 0, + } + } + + /// `true` once a terminator is found. + fn feed(&mut self, data: &[u8]) -> bool { + for &b in data { + if b == b'\n' { + self.end_line(); + if self.found { + return true; + } + } else if self.in_text { + if self.pos < 4 { + self.first[self.pos] = b; + } + if !b.is_ascii_whitespace() { + self.last_text = self.pos; + } + self.pos += 1; + } else if !b.is_ascii_whitespace() { + self.in_text = true; + self.first[0] = b; + self.last_text = 0; + self.pos = 1; + } + } + false + } + + fn end_line(&mut self) { + if self.in_text { + self.found |= if self.base64 { + self.pos >= 4 && self.first == *b"====" + } else { + self.last_text == 2 && self.first[..3] == *b"end" + }; } - }) + self.in_text = false; + self.pos = 0; + } + + fn finish(mut self) -> bool { + self.end_line(); + self.found + } } /// `[u8]::trim_ascii_start` is stable but re-implemented here to also cover the @@ -207,6 +279,56 @@ fn trim_ascii(mut b: &[u8]) -> &[u8] { mod tests { use super::*; + fn looks_like_uuencode(data: &[u8]) -> bool { + super::looks_like_uuencode(&crate::Probe::whole(data)) + } + + /// Whether a line of `data` is a terminator, as it was decided by + /// splitting it into lines. + fn has_terminator_by_lines(data: &[u8], base64: bool) -> bool { + lines(data).any(|l| { + let t = trim_ascii(l); + if base64 { + t.starts_with(b"====") + } else { + t == b"end" + } + }) + } + + #[test] + fn the_terminator_scan_decides_as_the_line_split_did() { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let alphabet = b"end=\n\r \t x"; + for len in 0..40 { + for _ in 0..300 { + let data: Vec = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + alphabet[(state % alphabet.len() as u64) as usize] + }) + .collect(); + for base64 in [false, true] { + let want = has_terminator_by_lines(&data, base64); + // Whole, then a byte at a time. + let mut whole = Terminator::new(base64); + whole.feed(&data); + assert_eq!( + whole.finish(), + want, + "{:?} base64={base64}", + String::from_utf8_lossy(&data) + ); + let mut bytes = Terminator::new(base64); + let found = data.iter().any(|b| bytes.feed(std::slice::from_ref(b))); + assert_eq!(found || bytes.finish(), want); + } + } + } + } + /// Reference classic-uuencode encoder for the roundtrip test. fn uuencode(name: &str, data: &[u8]) -> Vec { let enc = |v: u8| -> u8 { diff --git a/crates/exav-unpack/src/formats/xz.rs b/crates/exav-unpack/src/formats/xz.rs index 65d2a5aa..fe887062 100644 --- a/crates/exav-unpack/src/formats/xz.rs +++ b/crates/exav-unpack/src/formats/xz.rs @@ -1,25 +1,26 @@ use crate::*; -pub(crate) fn extract_xz( - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - budget.count_entry()?; - let cap = budget.reserve()?; - let (out, truncated) = decode_xz(data, cap)?; - if truncated { - return Err(LimitHit::new("xz member exceeds budget".to_string())); - } - ratio_guard(data.len() as u64, out.len() as u64, budget)?; - budget.commit(out.len() as u64); - Ok(visit(Entry::new("xz-content".to_string(), out), budget)) -} - /// The dictionary size exav is willing to allocate for an XZ stream. Matches the /// decoder's own allocation cap below — one number, not two that can drift. pub const XZ_MAX_DICT: u64 = 64 * 1024 * 1024; +/// Walk an `.xz` file: one member, every stream decoded in turn as it is read. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + use crate::stream::{emit_stream, single_meta}; + budget.count_entry()?; + let mut dec = content_reader(src); + emit_stream( + &single_meta("xz-content", src, None), + &mut dec, + budget, + visit, + ) +} + /// The dictionary size an XZ stream *declares*, if it can be read from the first /// block's LZMA2 filter properties. /// @@ -101,25 +102,21 @@ fn skip_varint(d: &[u8], p: usize) -> Option { /// decompressing to gigabytes be scanned without ever being held: the file on /// disk is bounded, its output is not. /// -/// Concatenated streams need no special handling here — the decoder resets at +/// Concatenated streams need no special handling here: the decoder resets at /// each end-of-stream and carries on, so multi-stream files stream like any -/// other. (bzip2 cannot do this: recovering from a spurious stream magic needs -/// re-reading the whole input, which a reader that has already emitted bytes -/// cannot do.) -pub(crate) fn content_reader(data: &[u8]) -> XzReader<'_> { - XzReader { - data, - input_pos: 0, - decoder: xz4rust::XzDecoder::with_alloc_dict_size(8192, XZ_MAX_DICT as usize), - staged: Vec::new(), - taken: 0, - done: false, - } +/// other. +pub(crate) fn content_reader( + data: &dyn crate::source::ByteSource, +) -> XzReader> { + XzReader::new(crate::source::Reader::new(data)) } -pub(crate) struct XzReader<'a> { - data: &'a [u8], - input_pos: usize, +/// Every stream of an `.xz` input read from `input`, decoded as it is read. +pub(crate) struct XzReader { + input: R, + inbuf: Vec, + in_pos: usize, + in_len: usize, /// The decoder owns its dictionary allocation, so its lifetime is /// independent of the input; `'static` keeps it out of the caller's way. decoder: xz4rust::XzDecoder<'static>, @@ -131,7 +128,37 @@ pub(crate) struct XzReader<'a> { done: bool, } -impl std::io::Read for XzReader<'_> { +impl XzReader { + pub(crate) fn new(input: R) -> Self { + XzReader { + input, + inbuf: vec![0; 8192], + in_pos: 0, + in_len: 0, + decoder: xz4rust::XzDecoder::with_alloc_dict_size(8192, XZ_MAX_DICT as usize), + staged: Vec::new(), + taken: 0, + done: false, + } + } + + /// Refill the input buffer once it is used up. `false` at the end of the + /// input. + fn fill(&mut self) -> std::io::Result { + if self.in_pos == self.in_len { + self.in_pos = 0; + self.in_len = loop { + match self.input.read(&mut self.inbuf) { + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + r => break r?, + } + }; + } + Ok(self.in_pos < self.in_len) + } +} + +impl std::io::Read for XzReader { fn read(&mut self, out: &mut [u8]) -> std::io::Result { loop { if self.taken < self.staged.len() { @@ -145,33 +172,36 @@ impl std::io::Read for XzReader<'_> { } self.staged.clear(); self.taken = 0; - let mut buf = [0u8; 8192]; - let feed = self.data[self.input_pos..].len().min(buf.len()); - if feed == 0 { + if !self.fill()? { self.done = true; return Ok(0); } + let mut buf = [0u8; 8192]; match self .decoder - .decode(&self.data[self.input_pos..self.input_pos + feed], &mut buf) + .decode(&self.inbuf[self.in_pos..self.in_len], &mut buf) { Ok(result) => { self.staged .extend_from_slice(&buf[..result.output_produced()]); - self.input_pos += result.input_consumed(); + self.in_pos += result.input_consumed(); if let xz4rust::XzNextBlockResult::EndOfStream(_, _) = result { self.decoder.reset(); // Concatenated streams may be separated by zero padding. - while self.input_pos < self.data.len() && self.data[self.input_pos] == 0 { - self.input_pos += 1; - } - if self.input_pos >= self.data.len() { - self.done = true; + loop { + if !self.fill()? { + self.done = true; + break; + } + if self.inbuf[self.in_pos] != 0 { + break; + } + self.in_pos += 1; } } } Err(xz4rust::XzError::NeedsLargerInputBuffer) => { - self.input_pos += feed; + self.in_pos = self.in_len; } // A malformed stream is corruption, not a limit. Surfacing it as // an io error lets the caller report Unscannable without killing @@ -181,61 +211,3 @@ impl std::io::Read for XzReader<'_> { } } } - -/// Decode all concatenated XZ streams using xz4rust's block-based API -/// (pure Rust, no unsafe, zero runtime deps with no_unsafe + no sha256). Shared -/// with the ZIP path (method 95 = XZ) via [`super::zip`]. -pub(crate) fn decode_xz(data: &[u8], cap: u64) -> Result<(Vec, bool), LimitHit> { - let mut decoder = xz4rust::XzDecoder::with_alloc_dict_size(8192, XZ_MAX_DICT as usize); - let mut out = Vec::new(); - let mut input_pos = 0; - let mut out_buf = [0u8; 8192]; - - loop { - let remaining = cap.saturating_sub(out.len() as u64); - if remaining == 0 { - return Ok((out, true)); - } - - let feed = data[input_pos..].len().min(out_buf.len()); - if feed == 0 { - break; - } - - match decoder.decode(&data[input_pos..input_pos + feed], &mut out_buf) { - Ok(result) => { - let produced = result.output_produced(); - if produced > 0 { - out.extend_from_slice(&out_buf[..produced]); - // Check budget after each output chunk (catches bombs). - if out.len() as u64 > cap { - return Ok((out, true)); - } - } - input_pos += result.input_consumed(); - if let xz4rust::XzNextBlockResult::EndOfStream(_, _) = result { - decoder.reset(); - // Skip padding zeros between concatenated streams - while input_pos < data.len() && data[input_pos] == 0 { - input_pos += 1; - } - if input_pos >= data.len() { - break; - } - } - } - Err(xz4rust::XzError::NeedsLargerInputBuffer) => { - input_pos += feed; - } - Err(e) => { - // A malformed/undecodable xz stream is a *corruption*, not a - // resource limit. Marking it `corrupt` makes it an Unscannable - // signal that does NOT abort the enclosing container: a bad `.xz` - // member inside a tar must not stop the sibling members (which may - // carry the actual detection) from being scanned. - return Err(LimitHit::corrupt(format!("xz: {e}"))); - } - } - } - Ok((out, false)) -} diff --git a/crates/exav-unpack/src/formats/zip.rs b/crates/exav-unpack/src/formats/zip.rs index c9fbaf3c..9edb49d7 100644 --- a/crates/exav-unpack/src/formats/zip.rs +++ b/crates/exav-unpack/src/formats/zip.rs @@ -108,93 +108,131 @@ pub(crate) fn zip_method_code(m: &::zip::CompressionMethod) -> u16 { // `Unsupported(code)`, so the real method number comes through there. match m { C::Stored => 0, + C::Shrink => 1, + C::Reduce(n) => 1 + u16::from(*n), + C::Implode => 6, C::Deflated => 8, C::Unsupported(n) => *n, _ => 0xffff, } } -/// Decode a ZIP member the `zip` crate itself can't (LZMA/BZIP2/ZSTD/XZ) from its -/// RAW compressed bytes, using exav's own decoders — so a payload hidden behind a -/// codec the crate lacks is still extracted and scanned. `usz` is the header's -/// declared uncompressed size (LZMA needs it as the decode target; memory is -/// still bounded by `cap`). Returns `None` when the method isn't decodable in this -/// build; the caller then emits an `unsupported` member — never a silent clean. -#[allow(unused_variables)] -pub(crate) fn decode_zip_raw( +/// Whether the `zip` crate itself decodes `method`: Store, Deflate, and the +/// PKZIP 1.x methods. Other codecs go to [`raw_decoder`]. +fn crate_decodes(method: u16) -> bool { + matches!(method, 0..=6 | 8) +} + +/// Shrink, Reduce and Implode (methods 1-6) are decoded whole by the `zip` +/// crate: it reads the compressed bytes whole, then reserves the declared +/// size for the output. Both sizes come from the archive, so both have to fit +/// the buffer limit before the member is opened. +fn decoded_whole_too_big(method: u16, comp: u64, size: u64, max_buffer: u64) -> bool { + (1..=6).contains(&method) && comp.max(size) > max_buffer +} + +/// Why [`raw_decoder`] gave no decoder for a member. +pub(crate) enum RawRefusal { + /// No decoder for this method in this build, or its header is malformed. + Unsupported(&'static str), + /// Decoding needs more memory than `--max-object-bytes` allows. + TooBig, +} + +/// A decoder, over a member's RAW compressed bytes, for a method the `zip` +/// crate itself can't decode, using exav's own decoders, so a payload behind +/// a codec the crate lacks is still scanned. The member is decoded as it is +/// read. `usz` is the declared uncompressed size (LZMA's decode target). +/// `max_buffer` bounds what a decoder allocates up front. +#[allow(unused_variables, unused_mut)] +pub(crate) fn raw_decoder<'a, R: Read + 'a>( method: u16, - raw: &[u8], + mut raw: R, usz: u64, - cap: u64, -) -> Option<(Vec, bool)> { + max_buffer: u64, +) -> Result, RawRefusal> { + const MALFORMED: RawRefusal = RawRefusal::Unsupported("malformed zip member header"); match method { // Method 9 = Deflate64 ("enhanced deflate"): deflate with a 64 KiB // window, length code 285 redefined to take 16 extra bits (lengths up to // 65538), and distance codes 30/31 made valid (14 extra bits, distances // up to 64 KiB). Produced by 7-Zip (`-mm=Deflate64`) and by Windows' - // own compressed-folder writer, and decoded by ClamAV — so a payload - // behind it must not hide. The member bytes are a raw stream (no - // wrapper), exactly like method 8. - 9 => bounded_read( - deflate64::Deflate64Decoder::with_buffer(Cursor::new(raw)), - cap, - ) - .ok(), + // own compressed-folder writer, and decoded by ClamAV. The member bytes + // are a raw stream (no wrapper), exactly like method 8. + 9 => Ok(Box::new(deflate64::Deflate64Decoder::new(raw))), // APPNOTE 5.8.8: 2-byte version, 2-byte props-size (=5), then the 5-byte // LZMA properties (1 lc/lp/pb byte + 4-byte dict size), then the stream. #[cfg(feature = "lzip")] 14 => { - if raw.len() < 9 { - return None; + let mut hdr = [0u8; 9]; + raw.read_exact(&mut hdr).map_err(|_| MALFORMED)?; + let declared = u32::from_le_bytes([hdr[5], hdr[6], hdr[7], hdr[8]]); + // The dictionary is allocated up front. One larger than the output + // is never consulted, so the output size bounds it for free; past + // that, it is memory this scan may not claim. + if u64::from(declared).min(usz) > max_buffer { + return Err(RawRefusal::TooBig); } - let props = raw[4]; - let dict = - crate::bounded_dict(u32::from_le_bytes([raw[5], raw[6], raw[7], raw[8]]), cap); - let reader = lzma_rust2::LzmaReader::new_with_props( - Cursor::new(&raw[9..]), - usz, - props, - dict, - None, - ) - .ok()?; - bounded_read(reader, cap).ok() + let dict = crate::bounded_dict(declared, usz); + lzma_rust2::LzmaReader::new_with_props(raw, usz, hdr[4], dict, None) + .map(|r| Box::new(r) as Box) + .map_err(|_| MALFORMED) } #[cfg(feature = "bzip2")] - 12 => bounded_read(bzip2_rs::DecoderReader::new(Cursor::new(raw)), cap).ok(), + 12 => Ok(Box::new(super::bzip2_rs::DecoderReader::new(raw))), #[cfg(feature = "zstd")] - 93 => { - let d = ruzstd::decoding::StreamingDecoder::new(Cursor::new(raw)).ok()?; - bounded_read(d, cap).ok() - } + 93 => ruzstd::decoding::StreamingDecoder::new(raw) + .map(|r| Box::new(r) as Box) + .map_err(|_| MALFORMED), // Method 95 = XZ: the raw member bytes are a complete .xz stream. #[cfg(feature = "xz")] - 95 => super::xz::decode_xz(raw, cap).ok(), - // Method 98 = PPMd var.H — the SAME variant 7z uses, so it reuses the - // in-tree PPMd7 decoder. ZIP packs the model parameters into a 2-byte + 95 => Ok(Box::new(super::xz::XzReader::new(raw))), + // Method 98 = PPMd var.H, the variant 7z uses, so it reuses the in-tree + // PPMd7 decoder. ZIP packs the model parameters into a 2-byte // little-endian header at the front of the member data (APPNOTE 5.9) // instead of carrying them in coder properties as 7z does. #[cfg(feature = "sevenz")] 98 => { - let (order, mem_size, body) = zip_ppmd_params(raw)?; - let rd = - super::sevenz::decode::Ppmd7ZReader::new(Cursor::new(body), order, mem_size, cap) - .ok()?; - bounded_read(rd, cap).ok() + let mut hdr = [0u8; 2]; + raw.read_exact(&mut hdr).map_err(|_| MALFORMED)?; + let (order, mem_size) = zip_ppmd_params(hdr).ok_or(MALFORMED)?; + match super::sevenz::decode::Ppmd7ZReader::new(raw, order, mem_size, max_buffer) { + Ok(r) => Ok(Box::new(r)), + Err(e) if e.is_corrupt() => Err(MALFORMED), + Err(_) => Err(RawRefusal::TooBig), + } } - _ => None, + _ => Err(RawRefusal::Unsupported( + "unsupported zip compression method", + )), } } -/// Split a ZIP method-98 (PPMd) member into its model parameters and payload. +/// [`raw_decoder`] over bytes in hand, read whole up to `cap`. `None` when +/// there is no decoder or decoding failed; the flag is set when the member +/// is larger than `cap`. +pub(crate) fn decode_zip_raw( + method: u16, + raw: &[u8], + usz: u64, + cap: u64, +) -> Option<(Vec, bool)> { + match raw_decoder(method, raw, usz, cap) { + Ok(r) => bounded_read(r, cap).ok(), + Err(RawRefusal::TooBig) => Some((Vec::new(), true)), + Err(RawRefusal::Unsupported(_)) => None, + } +} + +/// A ZIP method-98 (PPMd) member's model parameters, from its 2-byte header. /// -/// APPNOTE 5.9: a 2-byte little-endian word precedes the data — order in bits -/// 0-3 (biased by 1), model memory in MB in bits 4-11 (biased by 1), and the +/// APPNOTE 5.9: a little-endian word precedes the data: order in bits 0-3 +/// (biased by 1), model memory in MB in bits 4-11 (biased by 1), and the /// restoration method in bits 12-15. Only the parameters matter here; the /// restoration method is a property of the encoder's model resets, which the /// decoder follows from the stream itself. -fn zip_ppmd_params(raw: &[u8]) -> Option<(u32, u32, &[u8])> { - let w = u16::from_le_bytes([*raw.first()?, *raw.get(1)?]); +fn zip_ppmd_params(hdr: [u8; 2]) -> Option<(u32, u32)> { + let w = u16::from_le_bytes(hdr); let order = (w & 0x0f) as u32 + 1; let mem_mb = ((w >> 4) & 0xff) as u32 + 1; // Guard the shift as well as the decoder's own range check: 256 MB is the @@ -202,7 +240,7 @@ fn zip_ppmd_params(raw: &[u8]) -> Option<(u32, u32, &[u8])> { if !(2..=64).contains(&order) || mem_mb > 256 { return None; } - Some((order, mem_mb << 20, raw.get(2..)?)) + Some((order, mem_mb << 20)) } /// Parse the WinZip-AES 0x9901 extra field: `len(2)=7, ver(2), "AE"(2), @@ -441,7 +479,7 @@ const RESERVED_FLAG_BITS: u16 = 0b1101_0111_1000_0000; /// data descriptor declares its compressed size as zero in the header, so its /// end is not knowable until the stream is walked — counting those would /// invent overlaps on ordinary streamed archives. -pub fn overlapping_local_records(data: &[u8]) -> usize { +pub fn overlapping_local_records(data: &dyn crate::source::ByteSource) -> usize { /// Bit 3: sizes are deferred to a trailing data descriptor. const FLAG_DATA_DESCRIPTOR: u16 = 0x0008; /// Bound the scan so a hostile file cannot make this quadratic-expensive. @@ -467,26 +505,30 @@ pub fn overlapping_local_records(data: &[u8]) -> usize { // in the tool that produced it), while a bare pile of local records carrying // no directory is not something a normal writer emits. let declared: Option> = - ::zip::ZipArchive::new(Cursor::new(data)).ok().map(|mut z| { - (0..z.len()) - .filter_map(|i| z.by_index_raw(i).ok().map(|f| f.header_start() as usize)) - .collect() - }); + ::zip::ZipArchive::new(crate::source::Reader::new(data)) + .ok() + .map(|mut z| { + (0..z.len()) + .filter_map(|i| z.by_index_raw(i).ok().map(|f| f.header_start() as usize)) + .collect() + }); let mut extents: Vec<(usize, usize)> = Vec::new(); let mut pos = 0usize; while pos + LFH_LEN <= data.len() && extents.len() < MAX_RECORDS { - let Some(rel) = memfind(&data[pos..], b"PK\x03\x04") else { + let Some(off) = data.find(b"PK\x03\x04", pos, data.len()) else { break; }; - let off = pos + rel; pos = off + 4; - if declared.as_ref().is_some_and(|d| !d.contains(&off)) - || !plausible_local_header(data, off) - { + if declared.as_ref().is_some_and(|d| !d.contains(&off)) { continue; } - let h = &data[off..off + LFH_LEN]; + // The header and its name: all `plausible_header_at` reads. + let h = data.window(off, LFH_LEN + MAX_NAME); + if !plausible_header_at(&h, data.len() - off) { + continue; + } + let h = &h[..LFH_LEN]; let flags = u16::from_le_bytes([h[6], h[7]]); if flags & FLAG_DATA_DESCRIPTOR != 0 { continue; // extent unknown by construction @@ -593,7 +635,18 @@ pub fn directory_is_consistent(data: &[u8]) -> bool { /// consistently; deliberately strict, because every false positive here becomes /// a spurious `UNSCANNABLE` on an otherwise ordinary file. fn plausible_local_header(data: &[u8], off: usize) -> bool { - let Some(h) = data.get(off..off + LFH_LEN) else { + data.get(off..) + .is_some_and(|rest| plausible_header_at(rest, rest.len())) +} + +/// Longest local-header name taken for a real one. +const MAX_NAME: usize = 4096; + +/// [`plausible_local_header`] given `rest`, the bytes from the header on (at +/// least `LFH_LEN + MAX_NAME` of them, or up to the end of the archive), and +/// `left`, how many bytes the archive holds from the header on. +fn plausible_header_at(rest: &[u8], left: usize) -> bool { + let Some(h) = rest.get(..LFH_LEN) else { return false; }; let version = u16::from_le_bytes([h[4], h[5]]); @@ -610,10 +663,10 @@ fn plausible_local_header(data: &[u8], off: usize) -> bool { } // A path, not arbitrary bytes: non-empty, within the APPNOTE limit, present // in the file, and free of NULs (which no ZIP path may contain). - if name_len == 0 || name_len > 4096 { + if name_len == 0 || name_len > MAX_NAME { return false; } - let Some(name) = data.get(off + LFH_LEN..off + LFH_LEN + name_len) else { + let Some(name) = rest.get(LFH_LEN..LFH_LEN + name_len) else { return false; }; if name.contains(&0) { @@ -634,7 +687,7 @@ fn plausible_local_header(data: &[u8], off: usize) -> bool { { return false; } - data.len() >= off + LFH_LEN + name_len + extra_len + left >= LFH_LEN + name_len + extra_len } /// Find the first occurrence of `needle` in `hay`. @@ -1155,7 +1208,7 @@ pub fn extract_zip_from( // unsupported method. A decode we can't do (or an unknown codec) yields a // metadata-only unsupported member and the archive keeps going — one bad // member never aborts the whole ZIP. - let (buf, truncated, part_way) = if method != 0 && method != 8 { + let (buf, truncated, part_way) = if !crate_decodes(method) { let (raw, _) = bounded_read(&mut file, budget.limits.max_buffer_bytes) .map_err(|e| LimitHit::new(format!("zip raw read: {e}")))?; drop(file); @@ -1169,9 +1222,15 @@ pub fn extract_zip_from( } } } else { - // Store/Deflate: re-open with the decoding reader (the raw reader - // returns still-compressed bytes). + // Re-open with the decoding reader (the raw reader returns + // still-compressed bytes). drop(file); + if decoded_whole_too_big(method, comp, usz, cap) { + if let Some(r) = oversized(budget, "archive member exceeds size budget") { + return Ok(Some(r)); + } + continue; + } let mut file = match member_reader(&mut zip, i) { Ok(f) => f, Err(_) => { @@ -1211,30 +1270,6 @@ pub fn extract_zip_from( Ok(None) } -/// Lazy, seekable ZIP reader: yields one [`Entry`] at a time so a range-backed -/// reader (e.g. an HTTP range request over a remote ZIP) only fetches the -/// member currently being read, and the caller can stop early — e.g. on the -/// first detection — without touching later members. This is the seekable -/// counterpart to [`extract`], which buffers everything. -pub struct ZipMembers { - zip: ::zip::ZipArchive, - next: usize, - /// The bytes the central directory does not account for, with where each - /// run starts. Read at `open` and walked once the directory's own members - /// are exhausted. Empty for a well-formed archive. - unclaimed: Vec<(u64, Vec)>, - /// Where each hidden member's header sits, as `(run, offset in run)`. - /// - /// These take indices straight after the directory's own, so a hidden member - /// is addressable exactly like a listed one — `list` reports it, `extract` - /// takes its index, and the two agree. - orphans: Vec<(usize, usize)>, - /// The hidden-member search hit its cap with space still unexamined, so the - /// member list may be short. Reported as a member of its own rather than - /// left for the caller to not notice. - search_truncated: bool, -} - /// The byte ranges between the members the central directory claims, up to the /// directory itself. /// @@ -1403,6 +1438,227 @@ fn read_spans( /// gigabytes unclaimed would otherwise turn opening it into a full read. const MAX_ORPHAN_SCAN: u64 = 16 * 1024 * 1024; +/// Walk a ZIP off its source. Only the central directory and the members +/// actually read are fetched. A cleartext member is handed over as the +/// crate's decompressing reader, so a member decompressing to any size is +/// never held. An encrypted member is decrypted whole (decryption needs the +/// full ciphertext), bounded by the buffer limit. +pub(crate) fn walk( + src: &dyn crate::source::ByteSource, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + use crate::stream::{emit_entry, emit_stream, whole, MemberMeta}; + // A forged or corrupt central directory (a bad CDFH offset in the EOCD, + // say) makes the seekable walk impossible, but the members' local file + // headers are still in the file. Fall back to reading the archive whole + // and the local-header salvage, so a malformed archive still has its + // members scanned rather than being written off (matching clamd). + let mut zip = match ::zip::ZipArchive::new(crate::source::Reader::new(src)) { + Ok(z) => z, + Err(_) => return whole(Format::Zip, src, budget, visit), + }; + // A member left unread for a limit is reported after the rest are scanned. + let mut deferred = None; + for i in 0..zip.len() { + budget.count_entry()?; + // Peek metadata with the RAW reader: it never invokes the crate + // decryptor, so it succeeds for encrypted members too. + // A member whose header will not parse costs only itself (the same + // guard as in `extract_zip_from`). Abandoning the walk here would + // strand every later member: the salvage pass skips them as already + // covered, so a payload behind one bad directory pointer would never + // be scanned. + if let Err(e) = zip.by_index_raw(i).map(|_| ()) { + let hit = zip_entry_error(i, &e); + if !hit.is_corrupt() { + return Err(hit); + } + let meta = MemberMeta { + name: format!("zip entry {i}"), + comp_size: 0, + size: None, + encrypted: false, + unsupported: Some("ZIP member header will not parse"), + }; + if let Some(t) = visit(&meta, None, budget) { + return Ok(Some(t)); + } + continue; + } + let (name, is_file, encrypted, comp, size, method) = { + let f = zip.by_index_raw(i).map_err(|e| zip_entry_error(i, &e))?; + ( + f.name().to_string(), + f.is_file(), + f.encrypted(), + f.compressed_size(), + f.size(), + zip_method_code(&f.compression()), + ) + }; + // `is_file()` is false purely because the name ends in '/'. A JAR packer + // buys exactly that: `kingDavid/9.class/` holds a real deflate-compressed + // class that the JVM loads by name, while every ZIP tool discards it as a + // folder. Skip only what carries nothing at all: a "directory" with + // content is content. (The same guard lives in `extract_zip_from`.) + if !is_file && comp == 0 { + continue; // directory: counted toward the file budget above, skipped + } + let meta = MemberMeta { + name, + comp_size: comp, + size: Some(size), + encrypted, + unsupported: None, + }; + if encrypted { + if let Some(t) = walk_encrypted(&mut zip, i, meta, budget, visit)? { + return Ok(Some(t)); + } + continue; + } + let max_buffer = budget.limits.max_buffer_bytes; + if decoded_whole_too_big(method, comp, size, max_buffer) { + deferred.get_or_insert_with(|| { + LimitHit::new(format!( + "zip member '{}' is decoded whole and is larger than \ + --max-object-bytes {max_buffer}", + meta.name + )) + }); + continue; + } + // A codec the `zip` crate lacks fails here rather than in our own + // decoder, so fall back to the raw bytes and decode them ourselves, + // as `extract_zip_from` does. Probe first so the borrow of `zip` ends + // before the fallback needs it. + let decodable = zip.by_index(i).is_ok(); + let mut file = match decodable { + true => member_reader(&mut zip, i).map_err(|e| zip_entry_error(i, &e))?, + false => { + let out = walk_raw_decode(&mut zip, i, meta, budget, visit, &mut deferred)?; + if let Some(t) = out { + return Ok(Some(t)); + } + continue; + } + }; + if let Some(t) = emit_stream(&meta, &mut file, budget, visit)? { + return Ok(Some(t)); + } + } + // Members left out of the central directory while their local headers and + // data stay in the file, which is what the target extracts. + for entry in hidden_members(zip, budget)? { + if let Some(t) = emit_entry(entry, budget, visit)? { + return Ok(Some(t)); + } + } + deferred.map_or(Ok(None), Err) +} + +/// Decode a member the `zip` crate refused, from its RAW bytes, using exav's own +/// codec set, as it is read. The member is visited once: decoded, or as a +/// marker saying why it could not be. A decoder needing more memory than the +/// buffer limit leaves the member unread, and the limit goes in `deferred` +/// for the walk to report once the other members are scanned. +fn walk_raw_decode( + zip: &mut ::zip::ZipArchive, + i: usize, + mut meta: crate::stream::MemberMeta, + budget: &mut Budget, + visit: crate::stream::Visit, + deferred: &mut Option, +) -> Result, LimitHit> { + let cap = budget.limits.max_buffer_bytes; + let f = match zip.by_index_raw(i) { + Ok(f) => f, + Err(_) => { + meta.unsupported = Some("ZIP member header will not parse"); + return Ok(visit(&meta, None, budget)); + } + }; + let method = zip_method_code(&f.compression()); + let usz = f.size(); + match raw_decoder(method, f, usz, cap) { + Ok(mut r) => crate::stream::emit_stream(&meta, &mut r, budget, visit), + Err(RawRefusal::Unsupported(reason)) => { + meta.unsupported = Some(reason); + Ok(visit(&meta, None, budget)) + } + Err(RawRefusal::TooBig) => { + deferred.get_or_insert_with(|| { + LimitHit::new(format!( + "zip member '{}' needs more memory to decode than \ + --max-object-bytes {cap}", + meta.name + )) + }); + Ok(None) + } + } +} + +/// One encrypted ZIP member: try the password pool and hand over the +/// plaintext on success, else a metadata-only unsupported member, so the +/// caller reports it without masking later members. +fn walk_encrypted( + zip: &mut ::zip::ZipArchive, + i: usize, + meta: crate::stream::MemberMeta, + budget: &mut Budget, + visit: crate::stream::Visit, +) -> Result, LimitHit> { + use crate::stream::{emit_bytes, MemberMeta}; + let unsupported_meta = MemberMeta { + encrypted: true, + unsupported: Some("encrypted ZIP member"), + ..meta.clone() + }; + // Disprove the flag BEFORE any `decrypt` gating: the check decrypts nothing, + // it proves by CRC-32 that these bytes were never ciphertext. + let max_buffer = budget.limits.max_buffer_bytes; + let (crc, enc) = { + let mut f = zip.by_index_raw(i).map_err(|e| zip_entry_error(i, &e))?; + let crc = f.crc32(); + match read_encrypted_member(&mut f, max_buffer) { + Ok(e) => (crc, e), + Err(_) => return Ok(visit(&unsupported_meta, None, budget)), + } + }; + // The bit can lie: an APK packer sets it on every member because + // Android's ZIP reader ignores it, buying a PASSWORD-PROTECTED report on + // an archive the platform installs happily. A CRC-32 match over a plain + // decode proves the bytes were never encrypted. + if let Some(plain) = cleartext_despite_flag(&enc, crc) { + budget.commit(plain.len() as u64); + let meta = MemberMeta { + encrypted: false, + ..meta + }; + return emit_bytes(&meta, Some(plain), budget, visit); + } + // Actually encrypted. Without the `decrypt` feature there is no cipher + // stack compiled in, so report it: never decrypted, never silently clean. + #[cfg(not(feature = "decrypt"))] + { + Ok(visit(&unsupported_meta, None, budget)) + } + #[cfg(feature = "decrypt")] + { + match decrypt_zip_member(&enc, crc, budget)? { + Some(plain) => { + budget.commit(plain.len() as u64); + // Decrypted: the plaintext is scanned AND the member is still + // reported as having been encrypted. + emit_bytes(&meta, Some(plain), budget, visit) + } + None => Ok(visit(&unsupported_meta, None, budget)), + } + } +} + /// The members `zip`'s central directory omits, decoded, for a walk that has /// already been through the ones it lists. /// @@ -1429,7 +1685,7 @@ pub(crate) fn hidden_members( if truncated { budget.count_entry()?; out.push(Entry::unsupported( - ZipMembers::::TRUNCATED_MARKER.to_string(), + HIDDEN_SEARCH_TRUNCATED.to_string(), 0, false, "the search for members hidden from the central directory \ @@ -1439,312 +1695,8 @@ pub(crate) fn hidden_members( Ok(out) } -impl ZipMembers { - /// Open a seekable ZIP (reads only the central directory, and whatever - /// space that directory leaves unaccounted for). - pub fn open(reader: R) -> Result { - let mut zip = - ::zip::ZipArchive::new(reader).map_err(|e| LimitHit::new(format!("zip: {e}")))?; - let (gaps, known) = unclaimed_spans(&mut zip); - if gaps.is_empty() { - // The common case: members laid out end to end, nothing to look for - // and nothing extra read. - return Ok(Self { - zip, - next: 0, - unclaimed: Vec::new(), - orphans: Vec::new(), - search_truncated: false, - }); - } - // `ZipArchive` lends out no reader, so the only way to read elsewhere in - // the file is to take it back and reopen after. That costs a second - // parse of the central directory, which is why it is skipped entirely - // when there is nothing to scan. - let mut r = zip.into_inner(); - let (unclaimed, search_truncated) = read_spans(&mut r, &gaps, MAX_ORPHAN_SCAN)?; - let zip = ::zip::ZipArchive::new(r).map_err(|e| LimitHit::new(format!("zip: {e}")))?; - let orphans = find_orphans(&unclaimed, &known); - Ok(Self { - zip, - next: 0, - unclaimed, - orphans, - search_truncated, - }) - } - - /// The name the truncated-search marker carries, in both the listing and the - /// extraction, so the two agree about it as they do about real members. - const TRUNCATED_MARKER: &'static str = ""; - - /// Metadata for the members the central directory omits. - /// - /// Header parses only — nothing is decompressed, so a listing costs the same - /// whether an archive hides members or not. - fn orphan_infos(&self) -> Vec { - let base = self.zip.len(); - self.orphans - .iter() - .enumerate() - .filter_map(|(n, &(span, off))| { - let (_, buf) = self.unclaimed.get(span)?; - let (name, comp, usz, encrypted) = local_header_info(buf, off)?; - Some(crate::MemberInfo { - name, - index: base + n, - compressed_size: comp, - uncompressed_size: usz, - encrypted, - }) - }) - .collect() - } - - /// Extract the `n`th member the central directory omits. - fn extract_orphan(&mut self, n: usize, budget: &mut Budget) -> Result, LimitHit> { - // The index just past the last hidden member is the marker, when the - // search stopped early. It carries no data — its whole content is the - // statement that the list may be short. - if self.search_truncated && n == self.orphans.len() { - budget.count_entry()?; - return Ok(Some(Entry::unsupported( - Self::TRUNCATED_MARKER.to_string(), - 0, - false, - "the search for members hidden from the central directory \ - reached its size limit with space left unexamined", - ))); - } - let Some(&(span, off)) = self.orphans.get(n) else { - return Ok(None); - }; - let Some((_, buf)) = self.unclaimed.get(span) else { - return Ok(None); - }; - parse_local_member(buf, off, budget) - } - - /// Number of members, including the ones the central directory omits and the - /// marker for a search that stopped early — the same count `list_entries` - /// returns, so an index below it is addressable. - pub fn len(&self) -> usize { - self.zip.len() + self.orphans.len() + usize::from(self.search_truncated) - } - - /// Pre-parsed member metadata from the central directory. Directories are - /// included (with `uncompressed_size` 0). No member data is decompressed. - pub fn list_entries(&mut self) -> Vec { - let mut out = Vec::with_capacity(self.zip.len()); - for i in 0..self.zip.len() { - if let Ok(file) = self.zip.by_index_raw(i) { - out.push(crate::MemberInfo { - name: file.name().to_string(), - index: i, - compressed_size: file.compressed_size(), - uncompressed_size: file.size(), - encrypted: file.encrypted(), - }); - } - } - // The members the directory does not mention. Leaving them out would - // make a listing say the archive holds less than it does — and a member - // hidden this way is hidden on purpose. - out.extend(self.orphan_infos()); - if self.search_truncated { - out.push(crate::MemberInfo { - name: Self::TRUNCATED_MARKER.to_string(), - index: out.len(), - compressed_size: 0, - uncompressed_size: 0, - encrypted: false, - }); - } - out - } - - /// Read and decompress the next *file* member under `budget`; `None` when - /// the archive is exhausted. Directories are skipped (but still counted - /// toward the file-count budget). - pub fn next_member(&mut self, budget: &mut Budget) -> Option> { - while self.next < self.zip.len() { - let i = self.next; - self.next += 1; - if let Err(h) = budget.count_entry() { - return Some(Err(h)); - } - match self.extract_entry(i, budget) { - Ok(Some(e)) => return Some(Ok(e)), - Ok(None) => continue, - Err(h) => return Some(Err(h)), - } - } - // A member can be present as a local header and left OUT of the central - // directory, which hides it from every reader that walks the directory - // alone — while the tool that opens the archive extracts it anyway. So - // the walk continues past the directory into what it did not claim. - while self.next < self.len() { - let n = self.next - self.zip.len(); - self.next += 1; - if let Err(h) = budget.count_entry() { - return Some(Err(h)); - } - match self.extract_orphan(n, budget) { - Ok(Some(e)) => return Some(Ok(e)), - Ok(None) => continue, - Err(h) => return Some(Err(h)), - } - } - None - } - - /// Extract a specific member by index. Returns `Ok(None)` for directories. - pub(crate) fn extract_entry( - &mut self, - i: usize, - budget: &mut Budget, - ) -> Result, LimitHit> { - // Indices past the directory's own address the members it omits, which - // `list_entries` reports at exactly these positions. - if let Some(n) = i.checked_sub(self.zip.len()) { - return self.extract_orphan(n, budget); - } - // Peek with the RAW reader: it never invokes the crate decryptor, - // so it succeeds for encrypted members too (we build `zip` WITHOUT - // `aes-crypto`). Cleartext members are re-opened decompressing below. - let mut file = self - .zip - .by_index_raw(i) - .map_err(|e| zip_entry_error(i, &e))?; - if !file.is_file() { - return Ok(None); - } - let name = file.name().to_string(); - let comp = file.compressed_size(); - // Encrypted member: try the password pool; on success yield the - // decrypted+decompressed bytes, else a metadata-only encrypted - // member so the caller surfaces PasswordProtected (never a panic - // or an abort that would mask later members). - if file.encrypted() { - // See the buffered walker: the false-flag check is cipher-free. - #[cfg(not(feature = "decrypt"))] - { - let crc = file.crc32(); - let plain = read_encrypted_member(&mut file, budget.limits.max_buffer_bytes) - .ok() - .and_then(|enc| cleartext_despite_flag(&enc, crc)); - drop(file); - if let Some(plain) = plain { - let size = plain.len() as u64; - if size <= budget.reserve()? { - budget.commit(size); - return Ok(Some(Entry { - comp_size: comp, - encrypted: false, - unsupported: None, - name, - data: plain, - })); - } - } - return Ok(Some(Entry::unsupported( - name, - comp, - true, - "encrypted ZIP member", - ))); - } - #[cfg(feature = "decrypt")] - { - let crc = file.crc32(); - let enc = match read_encrypted_member(&mut file, budget.limits.max_buffer_bytes) { - Ok(e) => e, - Err(_) => { - return Ok(Some(Entry::unsupported( - name, - comp, - true, - "encrypted ZIP member", - ))) - } - }; - drop(file); - // Disprove the flag before spending password attempts on it — - // same check the buffered walker runs, for the same reason. - if let Some(plain) = cleartext_despite_flag(&enc, crc) { - let size = plain.len() as u64; - if size <= budget.reserve()? { - budget.commit(size); - return Ok(Some(Entry { - comp_size: comp, - encrypted: false, - unsupported: None, - name, - data: plain, - })); - } - } - return match decrypt_zip_member(&enc, crc, budget)? { - Some(plain) => { - budget.commit(plain.len() as u64); - // Decrypted: content AND encryption both reported. See the - // matching site in the buffered walker. - Ok(Some(Entry { - comp_size: comp, - encrypted: true, - unsupported: None, - name, - data: plain, - })) - } - None => Ok(Some(Entry::unsupported( - name, - comp, - true, - "encrypted ZIP member", - ))), - }; - } - } - // Cleartext member: re-open decompressing so `bounded_read` yields - // the *decompressed* content. - drop(file); - let mut file = member_reader(&mut self.zip, i).map_err(|e| zip_entry_error(i, &e))?; - // A member too large for the budget yields a metadata-only member - // (Unscannable, empty data) rather than aborting the archive walk, - // so name/size `.cdb` sigs still match and later members are scanned. - let cap = match budget.reserve() { - Ok(c) => c, - Err(_) => { - return Ok(Some(Entry::unsupported( - name, - comp, - false, - "archive member exceeds size budget", - ))) - } - }; - let s = bounded_read_salvage(&mut file, cap, !budget.should_verify_checksums()) - .map_err(|e| LimitHit::new(format!("zip read: {e}")))?; - if s.over_cap { - return Ok(Some(Entry::unsupported( - name, - comp, - false, - "archive member exceeds size budget", - ))); - } - ratio_guard(comp, s.data.len() as u64, budget)?; - budget.commit(s.data.len() as u64); - Ok(Some(Entry { - comp_size: comp, - encrypted: false, - unsupported: s.undecoded.then_some(PART_WAY), - name, - data: s.data, - })) - } -} +/// The member that says the search for hidden members stopped at its limit. +const HIDDEN_SEARCH_TRUNCATED: &str = ""; #[cfg(test)] mod orphan_scan_tests { @@ -1853,24 +1805,19 @@ mod zip_ppmd_tests { fn ppmd_params_follow_appnote_biases() { // order in bits 0-3 (+1), memory MB in bits 4-11 (+1). // w = 0x0107 -> order 8, mem 17 MB. - let raw = [0x07u8, 0x01, 0xAA, 0xBB]; - let (order, mem, body) = zip_ppmd_params(&raw).expect("valid params"); + let (order, mem) = zip_ppmd_params([0x07, 0x01]).expect("valid params"); assert_eq!(order, 8); assert_eq!(mem, 17 << 20); - assert_eq!( - body, - &[0xAA, 0xBB], - "the payload starts after the 2-byte header" - ); } #[test] fn ppmd_params_reject_out_of_range_models() { // order field 0 -> order 1, below PPMd7's minimum of 2. - assert!(zip_ppmd_params(&[0x00, 0x00, 0x00]).is_none()); + assert!(zip_ppmd_params([0x00, 0x00]).is_none()); // Too short to carry the header at all. - assert!(zip_ppmd_params(&[0x07]).is_none()); - assert!(zip_ppmd_params(&[]).is_none()); + for raw in [&[0x07u8][..], &[]] { + assert!(decode_zip_raw(98, raw, 64, 1 << 20).is_none()); + } } /// The wiring must be *reached* for method 98, and a payload it cannot diff --git a/crates/exav-unpack/src/formats/zip_crypto.rs b/crates/exav-unpack/src/formats/zip_crypto.rs index dc6824b1..fe7fe98c 100644 --- a/crates/exav-unpack/src/formats/zip_crypto.rs +++ b/crates/exav-unpack/src/formats/zip_crypto.rs @@ -2,18 +2,18 @@ //! PKWARE "ZipCrypto" stream cipher and WinZip AES (AE-1/AE-2). //! //! DECRYPTION with a *known* password needs only a cipher (AES) + a KDF -//! (PBKDF2-HMAC-SHA1) and a stream XOR — no RNG. So this links `aes`/`hmac`/ +//! (PBKDF2-HMAC-SHA1) and a stream XOR, no RNG. So this links `aes`/`hmac`/ //! `sha1`/`pbkdf2`/`ctr`, none of which pull `getrandom`, preserving the //! project's no-getrandom default build. We deliberately do NOT enable the //! `zip` crate's `aes-crypto`/encryption features (those pull `getrandom` for //! the salt/IV generation that only *encryption* needs). //! //! These routines take the already-read *raw encrypted member bytes* (what -//! `ZipFile::read` would yield without a decryptor — i.e. the stored/compressed +//! `ZipFile::read` would yield without a decryptor, i.e. the stored/compressed //! payload after the format's own crypto framing) and return the decrypted //! ciphertext-stripped bytes ready to be decompressed by the member's method. -use hmac::{Hmac, Mac}; +use hmac::{Hmac, KeyInit, Mac}; use sha1::Sha1; type HmacSha1 = Hmac; @@ -80,15 +80,14 @@ pub(crate) fn decrypt_aes(body: &[u8], strength: u8, password: &[u8]) -> Option< /// crate only offers big-endian / 32-/64-bit LE flavours, none of which is the /// 128-bit-LE WinZip needs). fn aes_ctr_le(key: &[u8], data: &mut [u8], key_len: usize) -> Option<()> { - use aes::cipher::{BlockEncrypt, KeyInit}; + use aes::cipher::{BlockCipherEncrypt, KeyInit}; let mut counter = [0u8; 16]; counter[0] = 1; // little-endian "1" macro_rules! run { ($ty:ty) => {{ let cipher = <$ty>::new_from_slice(key).ok()?; for chunk in data.chunks_mut(16) { - let mut block = - aes::cipher::generic_array::GenericArray::clone_from_slice(&counter); + let mut block = aes::Block::from(counter); cipher.encrypt_block(&mut block); for (b, k) in chunk.iter_mut().zip(block.iter()) { *b ^= *k; @@ -121,7 +120,7 @@ fn aes_ctr_le(key: &[u8], data: &mut [u8], key_len: usize) -> Option<()> { /// reference byte is the high byte of the CRC-32, EXCEPT when the archive was /// written with a streaming data descriptor (general-purpose bit 3), where the /// CRC isn't known at encryption time and the high byte of the DOS mod-time is -/// used instead — the form Info-ZIP's `zip` emits. The caller passes whichever +/// used instead (the form Info-ZIP's `zip` emits). The caller passes whichever /// candidate(s) apply; the full-payload CRC check downstream is the real gate. pub(crate) fn decrypt_zipcrypto( body: &[u8], @@ -141,7 +140,7 @@ pub(crate) fn decrypt_zipcrypto( *b = c; } // The classic password check: header[11] == high byte of CRC-32 (or DOS - // mod-time for streaming archives — see the doc comment). + // mod-time for streaming archives, see the doc comment). if !check_bytes.contains(&header[11]) { return None; } diff --git a/crates/exav-unpack/src/formats/zstd.rs b/crates/exav-unpack/src/formats/zstd.rs index 8250d5d7..69ba4580 100644 --- a/crates/exav-unpack/src/formats/zstd.rs +++ b/crates/exav-unpack/src/formats/zstd.rs @@ -1,21 +1,104 @@ -use crate::*; -use std::io::Cursor; +//! Zstandard: one member, every frame of the file decoded in turn as it is +//! read. Skippable frames carry no content and are passed over. +use std::io::{self, Read, Seek, SeekFrom}; -pub(crate) fn extract_zstd( - data: &[u8], +use ruzstd::decoding::errors::{FrameDecoderError, ReadFrameHeaderError}; +use ruzstd::decoding::{BlockDecodingStrategy, FrameDecoder}; + +use crate::source::{ByteSource, Reader}; +use crate::stream::{emit_stream, single_meta, Visit}; +use crate::{Budget, LimitHit}; + +pub(crate) fn walk( + src: &dyn ByteSource, budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { + visit: Visit, +) -> Result, LimitHit> { budget.count_entry()?; - let cap = budget.reserve()?; - let mut decoder = ruzstd::decoding::StreamingDecoder::new(Cursor::new(data)) - .map_err(|e| LimitHit::new(format!("zstd: {e}")))?; - let (out, truncated) = - bounded_read(&mut decoder, cap).map_err(|e| LimitHit::new(format!("zstd: {e}")))?; - if truncated { - return Err(LimitHit::new("zstd member exceeds budget".to_string())); + let mut frames = Frames { + source: Reader::new(src), + len: src.len() as u64, + decoder: FrameDecoder::new(), + in_frame: false, + }; + emit_stream( + &single_meta("zstd-content", src, None), + &mut frames, + budget, + visit, + ) +} + +/// The concatenation of a file's frames, as `zstd -d` outputs it. A decoder +/// that stopped after the first frame would leave the rest of the file unread +/// while the member looked complete. +struct Frames { + source: R, + len: u64, + decoder: FrameDecoder, + in_frame: bool, +} + +impl Read for Frames { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + loop { + if !self.in_frame { + if self.source.stream_position()? >= self.len { + return Ok(0); + } + match self.decoder.init(&mut self.source) { + Ok(()) => self.in_frame = true, + Err(FrameDecoderError::ReadFrameHeaderError( + ReadFrameHeaderError::SkipFrame { length, .. }, + )) => { + self.source.seek(SeekFrom::Current(i64::from(length)))?; + continue; + } + Err(e) => return Err(io::Error::other(e)), + } + } + // The same loop `ruzstd`'s `StreamingDecoder` runs: decoding is by + // block, and a block's bytes are only collectable once it is done. + while self.decoder.can_collect() < buf.len() && !self.decoder.is_finished() { + let wanted = buf.len() - self.decoder.can_collect(); + self.decoder + .decode_blocks(&mut self.source, BlockDecodingStrategy::UptoBytes(wanted)) + .map_err(io::Error::other)?; + } + let n = self.decoder.read(buf)?; + if n > 0 { + return Ok(n); + } + if !self.decoder.is_finished() { + return Ok(0); + } + self.in_frame = false; + } + } +} + +#[cfg(test)] +mod tests { + use crate::{extract, Budget, Format, Limits}; + use ruzstd::encoding::{compress_to_vec, CompressionLevel}; + + #[test] + fn every_frame_is_decoded() { + let mut blob = compress_to_vec(&b"first frame "[..], CompressionLevel::Fastest); + // A skippable frame: magic 0x184D2A50, a 4-byte length, then data. + blob.extend_from_slice(&0x184D_2A50u32.to_le_bytes()); + blob.extend_from_slice(&3u32.to_le_bytes()); + blob.extend_from_slice(b"xyz"); + blob.extend(compress_to_vec( + &b"PAYLOAD in frame two"[..], + CompressionLevel::Fastest, + )); + let mut budget = Budget::new(Limits::default()); + let entries = extract(Format::Zstd, &blob, &mut budget).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].data, b"first frame PAYLOAD in frame two"); } - ratio_guard(data.len() as u64, out.len() as u64, budget)?; - budget.commit(out.len() as u64); - Ok(visit(Entry::new("zstd-content".to_string(), out), budget)) } diff --git a/crates/exav-unpack/src/lib.rs b/crates/exav-unpack/src/lib.rs index 8bf0d4de..81f94d3d 100644 --- a/crates/exav-unpack/src/lib.rs +++ b/crates/exav-unpack/src/lib.rs @@ -1,63 +1,50 @@ //! Archive extraction with decompression-bomb limits. //! -//! Extraction is bounded by a [`Budget`] (total output bytes, file count, -//! recursion depth, compression ratio). Hitting a bound returns [`LimitHit`], -//! which the caller maps to `LimitsExceeded`. +//! [`detect`] names a container's format and [`walk`] hands its members, one +//! at a time, to a visitor, which may stop the walk. The container is any +//! [`ByteSource`]: bytes in memory, or a seekable source read through a +//! [`source::BlockCache`]. A member comes as its bytes when its format decodes +//! it whole, or as a reader that decodes it as it is read, so a member of any +//! size is never held here. [`extract`] collects every member into memory, for +//! callers that want a list. //! -//! The budget is reserved *before* each member is read, and each read is -//! capped to the bytes still remaining, so peak memory across an archive -//! (and across nested archives sharing the budget) never exceeds -//! `max_extracted_bytes`. -//! -//! # `Archive` — lazy member-by-member access -//! -//! The primary API for most callers. [`Archive::open`] detects the container -//! format and parses format-specific headers (e.g. the ZIP central directory). -//! Members are then extracted one at a time via [`Archive::extract_next`] under -//! a shared [`Budget`]. For seekable formats (ZIP) only the central directory -//! is read up front; individual members are fetched on demand. For all other -//! formats the data is buffered on open and members are extracted lazily from -//! the buffer. +//! Extraction is bounded by a [`Budget`] (members, output and scanned bytes, +//! recursion depth, the largest object held at once). Hitting a bound returns +//! [`LimitHit`], which the caller maps to `LimitsExceeded`, or to +//! `Unscannable` for content that could not be decoded. //! //! ```rust,no_run -//! use std::io::Cursor; -//! use exav_unpack::{Archive, Budget, Limits}; -//! -//! # fn example(data: &[u8]) -> Result<(), exav_unpack::LimitHit> { -//! let mut archive = Archive::open(Cursor::new(data))?; -//! println!("format: {:?}", archive.format()); +//! use exav_unpack::{detect, walk, Budget, Limits, Member}; //! -//! // Pre-parsed member metadata (free for ZIP). -//! for m in archive.list() { -//! println!(" {} ({} bytes)", m.name, m.uncompressed_size); -//! } -//! -//! // Extract members one at a time. +//! # fn example(data: Vec) -> Result<(), exav_unpack::LimitHit> { +//! let Some(fmt) = detect(&data) else { return Ok(()) }; //! let mut budget = Budget::new(Limits::default()); -//! while let Some(entry) = archive.extract_next(&mut budget)? { -//! println!("{}: {} bytes", entry.name, entry.data.len()); -//! } +//! walk::<()>(fmt, &data, &mut budget, &mut |meta, content, _| { +//! let size = match content { +//! Some(Member::Bytes(bytes)) => bytes.len() as u64, +//! Some(Member::Stream(reader)) => std::io::copy(reader, &mut std::io::sink()).unwrap_or(0), +//! None => 0, +//! }; +//! println!("{}: {size} bytes", meta.name); +//! None +//! })?; //! # Ok(()) //! # } //! ``` //! -//! # `extract_each` — visitor-based streaming -//! -//! Lower-level API for callers that own the extraction loop. [`extract_each`] -//! decodes one member at a time and hands it to a visitor closure, which scans -//! it, recurses into it, and drops it before the next member is decoded. Peak -//! memory is ~one member, and a visitor that returns `Some(r)` halts extraction -//! immediately — remaining members are never decompressed. -//! //! # Safety //! -//! This crate contains **no `unsafe` code** — the entire extraction layer, +//! This crate contains **no `unsafe` code**: the entire extraction layer, //! including the vendored PPMd7 sub-allocator (`formats/ppmd7/`), is 100% safe //! Rust over a bounds-checked byte arena. The forbid below is enforced //! crate-wide. #![forbid(unsafe_code)] -use std::io::{Read, Seek}; +use std::io::Read; + +pub use source::ByteSource; +#[cfg(feature = "base64scan")] +use source::{Bytes, Indexed, Stepper}; pub(crate) mod formats; // Every name this pulls in is behind a format feature, so the glob imports @@ -68,15 +55,16 @@ use formats::*; #[cfg(any(feature = "gzip", feature = "zip", feature = "pdf", feature = "ole"))] mod inflate; pub mod profile; +pub mod source; mod stream; pub mod volume; #[cfg(feature = "pdf")] pub use formats::has_obfuscated_name_object; -pub use stream::{ - is_budget_overflow, is_streamable, stream_members, BudgetReader, MemberMeta, StreamVisit, -}; +#[allow(unused_imports)] +pub(crate) use stream::Region; +pub use stream::{is_budget_overflow, walk, Member, MemberMeta, Visit}; -/// Count of overlapping ZIP local file records — the signal behind ClamAV's +/// Count of overlapping ZIP local file records, the signal behind ClamAV's /// `Heuristics.Zip.OverlappingFiles`. Zero for any well-formed archive. #[cfg(feature = "zip")] pub use formats::zip::overlapping_local_records; @@ -84,7 +72,7 @@ pub use formats::zip::overlapping_local_records; /// Without the `zip` feature there is no ZIP parser, so no ZIP heuristic can /// fire; answering zero keeps callers free of feature gates. #[cfg(not(feature = "zip"))] -pub fn overlapping_local_records(_data: &[u8]) -> usize { +pub fn overlapping_local_records(_data: &dyn ByteSource) -> usize { 0 } @@ -97,22 +85,6 @@ pub fn zip_directory_is_consistent(_data: &[u8]) -> bool { false } -/// Whether a seekable source is a self-extracting archive (an executable with -/// an archive appended past its stub), judged by the check [`detect`] runs on -/// one held in memory, with the archive looked for in the first `limit` bytes. -/// Only that check: the installers `detect` tries first are not ruled out. For a -/// source too large to buffer: those bytes are read a window at a time. Leaves -/// the source at an unspecified position. `false` without the `sfx` feature. -pub fn is_sfx(source: &mut R, limit: u64) -> Result { - #[cfg(feature = "sfx")] - return formats::sfx::is_sfx(source, limit); - #[cfg(not(feature = "sfx"))] - { - let _ = (source, limit); - Ok(false) - } -} - /// The dictionary size an XZ stream declares, and the largest exav will /// allocate. A declaration above the cap is ClamAV's /// `Heuristics.XZ.DicSizeLimit`: memory a decoder must commit before producing @@ -137,19 +109,17 @@ pub use formats::partition::intersection_alert as partition_intersection_alert; /// The first structural fault in an image, as a ClamAV /// `Heuristics.Broken.Media.*` name, or `None` when the file is well formed or -/// is not an image. Pure parsing — no decoding, no allocation of pixel data. +/// is not an image. Pure parsing: no decoding, no allocation of pixel data. pub use formats::mediacheck::broken_media_alert; /// Without the `pdf` feature there is no PDF parser, so no PDF heuristic can -/// fire. Answering `false` keeps every caller free of feature gates — the +/// fire. Answering `false` keeps every caller free of feature gates; the /// alternative left the documented `--no-default-features --features zip` build /// (the one the WASM size figures come from) failing to compile at all. #[cfg(not(feature = "pdf"))] pub fn has_obfuscated_name_object(_data: &[u8]) -> bool { false } -#[cfg(feature = "zip")] -use formats::ZipMembers; // RAR decompression primitives, exposed for the rar3/rar5 examples + tests. // Diagnostic surface, not part of the stable API: may change in any release. #[cfg(feature = "rar")] @@ -159,7 +129,7 @@ pub use formats::{unpack29, unpack50, window_size_from_comp_info}; /// Limits governing recursive extraction. /// /// New fields may appear in any release. Build with `Limits::default()` and -/// assign the fields you care about — never with a struct literal, which a +/// assign the fields you care about, never with a struct literal, which a /// new field would break. #[derive(Debug, Clone)] #[non_exhaustive] @@ -170,11 +140,11 @@ pub struct Limits { /// exav's default is deliberately HIGHER than ClamAV's 10,000, because the /// two count different populations for the same file. exav descends into /// nested archives that ClamAV does not, so it sees strictly more countable - /// objects — and adopting ClamAV's number would buy less coverage under the + /// objects, and adopting ClamAV's number would buy less coverage under the /// same-looking setting. /// /// Measured on a live example: the `litellm` PyPI source tarball holds 2,794 - /// tar members, 27 of which are `.whl` files — ZIP archives with their own + /// tar members, 27 of which are `.whl` files: ZIP archives with their own /// members. Counting those (correctly) puts the walk past 9,000, so at 10,000 /// it stopped short of a member ClamAV matched. Nothing was miscounted; there /// was simply more to count. A source tarball vendoring wheels is ordinary, @@ -189,7 +159,7 @@ pub struct Limits { /// /// Charged cumulatively by [`Budget::commit`] and never released, so it is /// also the ceiling on how much extracted data can be resident at one - /// moment — which makes it, not [`Self::max_buffer_bytes`], the limit that + /// moment, which makes it, not [`Self::max_buffer_bytes`], the limit that /// bounds live extraction memory. It has to fit the address space the /// process is given, or the kernel's limit is reached before this one and a /// reportable verdict becomes a killed worker; the daemon clamps it to the @@ -215,9 +185,9 @@ pub struct Limits { /// Cap on the cumulative bytes fed to the *matching core* across the whole /// recursive analysis of one top-level file (distinct from /// [`Self::max_extracted_bytes`], which counts what decompression - /// *produces*). Bounds re-scanning bombs — e.g. a disk image full of + /// *produces*). Bounds re-scanning bombs (e.g. a disk image full of /// embedded PEs, where the same suffix is carved and matched at many - /// offsets and depths — without limiting the legitimate one-pass scan of a + /// offsets and depths) without limiting the legitimate one-pass scan of a /// large file. Deterministic, so it trips identically on every machine /// (unlike a wall-clock deadline). pub max_scanned_bytes: u64, @@ -235,7 +205,7 @@ pub struct Limits { /// and a caller that does not. A deployment that only ever expects ZIP can /// say so without maintaining its own build. /// - /// A format excluded here is REPORTED, never skipped — the member comes + /// A format excluded here is REPORTED, never skipped: the member comes /// back `Entry::unsupported`, so the scan says "there is a container here /// and I did not open it" rather than passing over it in silence. Refusing /// to look is a policy; pretending there was nothing to see is a bug. @@ -263,7 +233,7 @@ impl Default for Limits { max_compression_ratio: 1000, max_buffer_bytes: 256 * 1024 * 1024, // 10 GiB fed to the matcher total. This is a CPU/time bound, NOT a - // memory bound — streamed members are scanned without being held in + // memory bound: streamed members are scanned without being held in // RAM (capped separately by `max_buffer_bytes`/`deep_analysis_max`), // so this can be generous: it exists only to stop re-scanning bombs // and runaway scan time, and lets a multi-gigabyte member be fully @@ -288,7 +258,7 @@ pub struct Budget { /// checked against would bypass them just as effectively. Set them once when /// constructing the budget. limits: Limits, - // Running counters — mutated only through `charge_scan`/`count_entry`/ + // Running counters, mutated only through `charge_scan`/`count_entry`/ // `reserve`/`commit` so the bomb-defense bounds can't be bypassed by a caller // writing them directly. Crate-private for that reason. pub(crate) files: u64, @@ -298,14 +268,14 @@ pub struct Budget { /// Emulator instructions spent so far (see [`Limits::max_pe_emulation_steps`]). pub(crate) pe_emulation_steps: u64, /// Candidate passwords tried (in order) when decrypting an encrypted member - /// (ZIP ZipCrypto/AES today). Empty by default — an encrypted member with no + /// (ZIP ZipCrypto/AES today). Empty by default: an encrypted member with no /// password yields an `Entry::unsupported(encrypted=true, …)` so the scanner /// reports `PasswordProtected`. The pool is the union of any `.pwdb` file and /// the runtime `ScanOptions::passwords`, threaded down by exav-core. pub passwords: Vec, /// Verify container checksums (CRCs) during extraction. **Off by default**: /// a malware scanner scans decompressed content regardless of integrity - /// metadata — a wrong CRC must never stop a member's bytes from being + /// metadata: a wrong CRC must never stop a member's bytes from being /// scanned (that would let an attacker downgrade a detection by flipping a /// checksum byte). This matches ClamAV, which ignores CRCs for scanning. /// Only has effect with the `checksums` Cargo feature compiled in; without @@ -341,7 +311,7 @@ impl Budget { /// Enable container-checksum verification (default off). Requires the /// `checksums` feature to have any effect. When off (default), extraction - /// scans decompressed content even if a CRC fails — the scanner default. + /// scans decompressed content even if a CRC fails (the scanner default). pub fn set_verify_checksums(&mut self, verify: bool) -> &mut Self { self.verify_checksums = verify; self @@ -372,8 +342,8 @@ impl Budget { /// Bytes still available in the cumulative scan budget /// ([`Limits::max_scanned_bytes`] − already scanned). Used by the **streaming** /// member API as the per-member reader cap: a streamed member is never - /// retained, so it is bounded by how much may still be fed to the matcher — - /// a *processing* limit — not by the per-member *buffer* cap + /// retained, so it is bounded by how much may still be fed to the matcher + /// (a *processing* limit), not by the per-member *buffer* cap /// ([`Self::reserve`]). This is what decouples "how large a member we will /// scan" from "how much we hold in RAM at once". pub fn remaining_scan(&self) -> u64 { @@ -381,7 +351,7 @@ impl Budget { } /// Count one archive member toward the file-count budget. Called for - /// every entry encountered — including directories and skipped entries — + /// every entry encountered, including directories and skipped entries, /// so a huge directory-only archive still trips the cap. pub fn count_entry(&mut self) -> Result<(), LimitHit> { self.files += 1; @@ -404,7 +374,8 @@ impl Budget { .saturating_sub(self.total_out); if remaining == 0 { return Err(LimitHit::new(format!( - "extracted bytes > {}", + "extracted bytes > {}, the total a scan may hold (exav's \ + --max-process-bytes sets it)", self.limits.max_extracted_bytes ))); } @@ -458,7 +429,7 @@ pub struct LimitHit { /// The budget that stopped a scan, named the way the signature format names it. /// -/// Deliberately no `Default`: there is no neutral kind — a bare default would +/// Deliberately no `Default`: there is no neutral kind. A bare default would /// read as one verdict or the other, so every construction names its kind. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] @@ -474,13 +445,13 @@ pub enum LimitKind { MaxRecursion, /// CPU work: emulator instructions across the scan (`max_pe_emulation_steps`). MaxScanTime, - /// Not a budget — the input was malformed. Kept in the same enum so every + /// Not a budget: the input was malformed. Kept in the same enum so every /// `LimitHit` has a kind and nothing has to guess. Corrupt, } impl LimitHit { - /// Whether the stop was undecodable content rather than a resource bound — + /// Whether the stop was undecodable content rather than a resource bound: /// the verdict discriminator (`true` → `Unscannable`, `false` → /// `LimitsExceeded`). pub fn is_corrupt(&self) -> bool { @@ -531,7 +502,7 @@ pub(crate) fn read_full(src: &mut R, buf: &mut [u8]) -> Result /// Up to `len` bytes at `off`; fewer only where the source ends. See /// [`read_full`] for why a failure is an error. #[allow(dead_code)] -pub(crate) fn read_at( +pub(crate) fn read_at( src: &mut R, off: u64, len: usize, @@ -564,7 +535,7 @@ pub struct Entry { impl Entry { /// An entry whose compressed size is unknown (use the decompressed length) - /// and which is not encrypted — the common case. + /// and which is not encrypted: the common case. pub fn new(name: String, data: Vec) -> Self { Entry { comp_size: data.len() as u64, @@ -594,19 +565,9 @@ impl Entry { } } -/// Metadata for one archive member (no data loaded). -#[derive(Debug, Clone, PartialEq)] -pub struct MemberInfo { - pub name: String, - pub index: usize, - pub compressed_size: u64, - pub uncompressed_size: u64, - pub encrypted: bool, -} - /// A container/archive format this crate can extract embedded files from /// (archives plus structured documents: OLE2, PDF, MIME email). -// `Ord`/`Hash` so a caller can hold a set of formats — see +// `Ord`/`Hash` so a caller can hold a set of formats; see // [`Limits::allowed_formats`]. The ordering has no meaning beyond making that // set cheap; nothing depends on which format sorts first. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -627,7 +588,8 @@ pub enum Format { Iso, Lha, Arj, - /// RAR4/RAR5 archive (stored members; compressed methods are not decoded). + /// RAR4/RAR5 archive. Stored, RAR3 (LZ + PPMd) and RAR5 members are + /// decoded; RAR 1.5/2.x compression and encryption are reported. Rar, /// UPX-packed PE/ELF/Mach-O (decompress the embedded original). Upx, @@ -642,7 +604,7 @@ pub enum Format { /// Microsoft Virtual Hard Disk (`conectix`), fixed or dynamic. The variant /// is always defined (like `Iso`/`Xar`) so it can be named in any build; /// only the detector and extractor are gated. Matches still need a - /// wildcard arm — the enum is `#[non_exhaustive]`, so no downstream match + /// wildcard arm: the enum is `#[non_exhaustive]`, so no downstream match /// is ever exhaustive. Vhd, /// Unix `compress` (`.Z`), LZW. @@ -651,44 +613,44 @@ pub enum Format { Qcow2, /// VMware virtual disk (sparse or streamOptimized). Vmdk, - /// Microsoft VHDX — the modern Windows virtual disk. + /// Microsoft VHDX, the modern Windows virtual disk. Vhdx, - /// Windows Imaging Format (`.wim`/`.esd`) — chunk-compressed file resources. + /// Windows Imaging Format (`.wim`/`.esd`): chunk-compressed file resources. Wim, /// LZ4 frame (`.lz4`). Lz4, - /// ARC / PKARC / PAK archive — the pre-ZIP SEA format. + /// ARC / PKARC / PAK archive, the pre-ZIP SEA format. Arc, - /// ACE archive — recognised so it is reported, never decoded. + /// ACE archive, recognised so it is reported, never decoded. Ace, StuffIt, - /// ALZ archive (ESTsoft ALZip) — recognised so it is reported, not decoded. + /// ALZ archive (ESTsoft ALZip): stored, bzip2 and deflate members. Alz, - /// EGG archive (ESTsoft) — recognised so it is reported, not decoded. + /// EGG archive (ESTsoft): stored, deflate, bzip2, LZMA and AZO members. Egg, - /// HWP v3 document (Hangul) — recognised so it is reported, not decoded. + /// HWP v3 document (Hangul): the deflated body is decoded. Hwp3, - /// InstallShield MSI installer — recognised so it is reported, not decoded. + /// InstallShield MSI installer, recognised so it is reported, not decoded. IshieldMsi, - /// InstallShield InstallScript cabinet — recognised, not decoded. + /// InstallShield InstallScript cabinet, recognised, not decoded. IshieldCab, - /// InstallShield Z archive — the older `.z` installer format, decoded. + /// InstallShield Z archive, the older `.z` installer format, decoded. IshieldZ, - /// CryptFF-encrypted file — recognised so it is reported, not decrypted. + /// CryptFF-encrypted file, recognised so it is reported, not decrypted. CryptFf, - /// ext2/3/4 filesystem image — recognised so it is reported, not walked. + /// ext2/3/4 filesystem image: walked, files reassembled from their extents. Ext, - /// lrzip stream — recognised so it is reported, not decoded. + /// lrzip stream: recognised so it is reported, not decoded. Lrzip, - /// ZOO archive — recognised so it is reported, not decoded. + /// ZOO archive: LZD and LZH members decoded, CRC-16 checked. Zoo, - /// AppleSingle / AppleDouble container — recognised, not read. + /// AppleSingle / AppleDouble container, recognised, not read. AppleSingle, - /// FAT12/16/32 filesystem — walked, so fragmented files come back whole. + /// FAT12/16/32 filesystem, walked, so fragmented files come back whole. Fat, - /// Inno Setup installer — recognised so it is reported, never decoded. + /// Inno Setup installer, recognised so it is reported, never decoded. Inno, - /// NTFS filesystem — walked through the master file table. + /// NTFS filesystem, walked through the master file table. Ntfs, /// Zstandard compressed stream. Zstd, @@ -702,36 +664,36 @@ pub enum Format { Szdd, /// TNEF (`winmail.dat`) MS email attachment container. Tnef, - /// SWF (Adobe Flash) movie — decompress the inner FWS body (CWS/ZWS). + /// SWF (Adobe Flash) movie: decompress the inner FWS body (CWS/ZWS). Swf, - /// BinHex 4.0 (`.hqx`) — classic-Mac 6-bit-encoded forked file. + /// BinHex 4.0 (`.hqx`), a classic-Mac 6-bit-encoded forked file. Binhex, - /// Windows Shell Link (`.lnk`) — extract command-line/target/icon strings. + /// Windows Shell Link (`.lnk`): extract command-line/target/icon strings. Lnk, /// Raw disk image partition map (GPT / Apple Partition Map / MBR). Partition, - /// Python compiled bytecode (`.pyc`) — surface the marshalled code body. + /// Python compiled bytecode (`.pyc`): surface the marshalled code body. Pyc, - /// NSIS (Nullsoft) installer — decompress the embedded data blocks. + /// NSIS (Nullsoft) installer: decompress the embedded data blocks. Nsis, - /// Mach-O universal ("fat") binary — split into per-architecture slices. + /// Mach-O universal ("fat") binary: split into per-architecture slices. Machofat, /// Self-extracting archive: an executable stub with an archive appended. Sfx, /// Compiled AutoIt3 script embedded in a PE (`AU3!EA05`/`AU3!EA06`). Autoit, - /// Microsoft OneNote (`.one`) section — carve embedded FileDataStoreObjects. + /// Microsoft OneNote (`.one`) section: carve embedded FileDataStoreObjects. OneNote, - /// RTF document — extract hex-encoded embedded objects (`\objdata`). + /// RTF document: extract hex-encoded embedded objects (`\objdata`). Rtf, /// PE packed by a runtime packer (Petite/FSG/NsPack aPLib families are /// decompressed; other packers are detected only). See `formats/pepack.rs`. PePacked, - /// Java `.class` — surface constant-pool strings and class/name references. + /// Java `.class`: surface constant-pool strings and class/name references. JavaClass, /// AI model: Python pickle (dangerous-import surfacing) or safetensors. AiModel, - /// Microsoft Script Encoder (`#@~^` VBScript/JScript.Encode) — decode. + /// Microsoft Script Encoder (`#@~^` VBScript/JScript.Encode): decode. Screnc, } @@ -813,7 +775,7 @@ mod format_all_tests { /// `ALL` is written by hand, so it can fall behind the enum. Round-tripping /// each variant through a match the compiler checks for exhaustiveness makes - /// a new variant a build error here rather than a silent omission — and an + /// a new variant a build error here rather than a silent omission, and an /// omission would let a container escape the scanner's dispatch coverage /// test in `exav-core`. #[test] @@ -901,7 +863,7 @@ mod format_all_tests { /// Whether `data` begins with a real bzip2 stream: `BZh` + a block-size digit /// (`1`–`9`) + the 48-bit block magic `0x314159265359` (a compressed block) or /// the empty-stream end magic `0x177245385090`. The bare 4-byte `BZh#` prefix is -/// too weak on its own — it occurs coincidentally in binary data (e.g. inside an +/// too weak on its own: it occurs coincidentally in binary data (e.g. inside an /// ISO or a PE overlay), and treating such a false hit as bzip2 makes the member /// fail to decode and poisons the whole scan as `UNSCANNABLE`. Every genuine /// bzip2 stream carries one of these two magics, so the check has no false @@ -924,8 +886,25 @@ fn is_cab_magic(data: &[u8]) -> bool { data.len() >= 8 && data.starts_with(b"MSCF") && data[4..8] == [0, 0, 0, 0] } +/// Whether `data` begins with an ARJ main header: `60 EA`, a size of at most +/// 2600, and the header's CRC-32 after it. The magic alone is two bytes and +/// collides with binary data; a false hit fails in the extractor and reports +/// the object `UNSCANNABLE`. +#[cfg(feature = "arj")] +fn is_arj_magic(data: &[u8]) -> bool { + let Some(&[0x60, 0xEA, lo, hi]) = data.get(..4) else { + return false; + }; + let size = u16::from_le_bytes([lo, hi]) as usize; + let Some(crc) = data.get(4 + size..8 + size) else { + return false; + }; + (1..=2600).contains(&size) + && crc32fast::hash(&data[4..4 + size]) == u32::from_le_bytes(crc.try_into().unwrap()) +} + /// Whether `data` begins with a real gzip member (RFC 1952): `1f 8b`, then the -/// compression method `08` (deflate — the only method gzip defines), then a flag +/// compression method `08` (deflate, the only method gzip defines), then a flag /// byte whose three high bits are reserved and must be zero. The bare `1f 8b` /// prefix is only two bytes and collides constantly with binary data (e.g. inside /// a PE); a false hit is routed to the inflate path, fails with a "corrupt @@ -936,7 +915,7 @@ fn is_gzip_magic(data: &[u8]) -> bool { } /// Whether `d` starts with a stand-alone executable / OLE magic (PE `MZ`, ELF, -/// Mach-O, or OLE2/CFB). Cheap prefix test — 6 decoded bytes suffice, so it also +/// Mach-O, or OLE2/CFB). Cheap prefix test: 6 decoded bytes suffice, so it also /// gates on just the first few decoded base64 chars. Deliberately excludes bare /// ZIP (`PK`): a 4-byte match collides too often with base64 noise. #[cfg(feature = "base64scan")] @@ -965,8 +944,8 @@ fn is_executable_payload(d: &[u8]) -> bool { true } -/// Decode base64 assets embedded in a markup buffer — `data:` URIs in HTML and -/// base64 element bodies in flat-XML Office documents — returning each decoded +/// Decode base64 assets embedded in a markup buffer (`data:` URIs in HTML and +/// base64 element bodies in flat-XML Office documents), returning each decoded /// payload. /// /// This is the inline-attachment channel of a document that has no archive to @@ -976,13 +955,21 @@ fn is_executable_payload(d: &[u8]) -> bool { /// self-contained file with nothing to fetch and nothing to extract with an /// ordinary unpacker. Signatures key on those images and scope them with /// `Container:` precisely so they fire on the document and not on the same image -/// standing alone — which they can only do if the image is pulled out of it. +/// standing alone, which they can only do if the image is pulled out of it. /// /// A run is decoded only when its first bytes are a real asset magic (image, /// executable, OLE2), so ordinary base64-looking text costs a 6-byte decode and /// nothing more. Bounded by `cap` per payload and by a cap on how many return. #[cfg(feature = "base64scan")] -pub fn markup_embedded_payloads(data: &[u8], cap: u64) -> Vec> { +pub fn markup_embedded_payloads(src: &dyn ByteSource, cap: u64) -> Vec> { + match src.as_slice() { + Some(data) => markup_payloads_on(&mut Indexed(data), cap), + None => markup_payloads_on(&mut Stepper::new(src), cap), + } +} + +#[cfg(feature = "base64scan")] +fn markup_payloads_on(data: &mut B, cap: u64) -> Vec> { use base64::Engine; /// Shortest base64 run worth decoding; below this it cannot be an asset. const MIN_RUN: usize = 64; @@ -995,17 +982,17 @@ pub fn markup_embedded_payloads(data: &[u8], cap: u64) -> Vec> { let mut i = 0usize; let n = data.len(); while i < n && out.len() < MAX_PAYLOADS { - if !is_b64(data[i]) { + if !is_b64(data.at(i)) { i += 1; continue; } // Only runs introduced by a `data:` URI or sitting directly inside an // element body are candidates. Anything else in markup is prose. - let before = &data[i.saturating_sub(96)..i]; + let before = data.range(i.saturating_sub(96), i); let introduced = before.ends_with(b";base64,") || before.last() == Some(&b'>'); let start = i; let mut j = i; - while j < n && is_b64(data[j]) { + while j < n && is_b64(data.at(j)) { j += 1; } i = j.max(start + 1); @@ -1014,26 +1001,26 @@ pub fn markup_embedded_payloads(data: &[u8], cap: u64) -> Vec> { } // Cheap gate first: 8 base64 chars decode to 6 bytes, enough for every // magic we care about, and costs no allocation on a miss. - match plain.decode(&data[start..start + 8]) { + match plain.decode(data.range(start, start + 8)) { Ok(head) if starts_asset_magic(&head) => {} _ => continue, } // Absorb the padding the run scan stopped at, so a well-formed URI // decodes whole. Without it the tail falls off the last 4-char group and - // the payload comes back one or two bytes short — invisible on an image, + // the payload comes back one or two bytes short: invisible on an image, // fatal to a hash signature. let mut end = j; - while end < n && end - j < 2 && data[end] == b'=' { + while end < n && end - j < 2 && data.at(end) == b'=' { end += 1; } i = i.max(end); - let run = &data[start..end]; - if run.len() as u64 > cap.saturating_mul(2) { + if (end - start) as u64 > cap.saturating_mul(2) { continue; } + let run = data.range(start, end); let decoded = engine - .decode(run) - .or_else(|_| plain.decode(&data[start..start + (j - start) / 4 * 4])); + .decode(&run) + .or_else(|_| plain.decode(&run[..(j - start) / 4 * 4])); if let Ok(dec) = decoded { if !dec.is_empty() && dec.len() as u64 <= cap { out.push(dec); @@ -1060,17 +1047,25 @@ fn starts_asset_magic(d: &[u8]) -> bool { /// Find base64-encoded executables embedded in a text/script buffer and return /// each decoded payload. Malware routinely stashes a PE/ELF as a long base64 -/// string in a script — e.g. PowerShell reflective loaders (`$PEBytes = -/// "TVqQAA…"`), JS/VBS droppers, HTA — where the executable is invisible to a +/// string in a script, such as a PowerShell reflective loader (`$PEBytes = +/// "TVqQAA…"`), a JS/VBS dropper or an HTA, where the executable is invisible to a /// signature that matches the *decoded* bytes. Each maximal run of base64 /// characters (internal whitespace tolerated, since scripts/RTF line-wrap the /// blob) at least `MIN_RUN` long is decoded; a decode is returned only when it /// passes `is_executable_payload`, so a coincidental base64-looking region in -/// binary data (which decodes to noise) is dropped — no false positives, and the +/// binary data (which decodes to noise) is dropped: no false positives, and the /// caller still validates + rescans through the normal type path. Bounded by /// `cap` (per-payload size), `MAX_PAYLOADS`, and `MAX_ATTEMPTS` (decode tries). #[cfg(feature = "base64scan")] -pub fn base64_payloads(data: &[u8], cap: u64) -> Vec> { +pub fn base64_payloads(src: &dyn ByteSource, cap: u64) -> Vec> { + match src.as_slice() { + Some(data) => base64_payloads_on(&mut Indexed(data), cap), + None => base64_payloads_on(&mut Stepper::new(src), cap), + } +} + +#[cfg(feature = "base64scan")] +fn base64_payloads_on(data: &mut B, cap: u64) -> Vec> { use base64::Engine; // A base64-encoded PE is at minimum a few hundred bytes; require a run that // decodes to ≥ ~1 KB so we never trial-decode short incidental runs. @@ -1086,19 +1081,19 @@ pub fn base64_payloads(data: &[u8], cap: u64) -> Vec> { let n = data.len(); let mut i = 0; while i < n && out.len() < MAX_PAYLOADS && attempts < MAX_ATTEMPTS { - if !is_b64(data[i]) { + if !is_b64(data.at(i)) { i += 1; continue; } - // Measure the run [start, j) — count base64 chars (internal whitespace - // from line-wrapping tolerated) and capture the first 8 — WITHOUT + // Measure the run [start, j): count base64 chars (internal whitespace + // from line-wrapping tolerated) and capture the first 8, WITHOUT // allocating. Stops at padding `=` or any other byte. let start = i; let mut nb64 = 0usize; let mut head = [0u8; 8]; let mut j = i; while j < n { - let b = data[j]; + let b = data.at(j); if is_b64(b) { if nb64 < 8 { head[nb64] = b; @@ -1116,7 +1111,7 @@ pub fn base64_payloads(data: &[u8], cap: u64) -> Vec> { continue; } // Cheap gate: decode only the first 8 base64 chars (6 bytes) and bail - // unless they start with an executable/OLE magic — so long runs that are + // unless they start with an executable/OLE magic, so long runs that are // NOT executables (RTF `\objdata` hex, benign base64 text) cost nothing // beyond the byte count above: no full decode, no payload allocation. match engine.decode(head) { @@ -1124,13 +1119,24 @@ pub fn base64_payloads(data: &[u8], cap: u64) -> Vec> { _ => continue, } attempts += 1; + // Whole 4-char groups decode to exactly 3 bytes each, so a run too + // large to keep is known before it is read. + if (nb64 / 4 * 3) as u64 > cap { + continue; + } // Executable-looking: now materialize the run (whitespace stripped) and // decode the whole 4-char groups (dropping any partial tail / padding). let mut run: Vec = Vec::with_capacity(nb64); - for &b in &data[start..j] { - if is_b64(b) { - run.push(b); - } + let mut at = start; + while at < j { + let piece_end = j.min(at + source::CHUNK); + run.extend( + data.range(at, piece_end) + .iter() + .copied() + .filter(|&b| is_b64(b)), + ); + at = piece_end; } run.truncate(run.len() / 4 * 4); if let Ok(dec) = engine.decode(&run) { @@ -1142,12 +1148,291 @@ pub fn base64_payloads(data: &[u8], cap: u64) -> Vec> { out } +/// What [`detect`] reads of an object: its start (all of it when it is held in +/// memory), its length, and the rest of it for the checks that read its end or +/// search it through. +pub(crate) struct Probe<'a> { + pub(crate) head: &'a [u8], + pub(crate) len: usize, + /// The object, when `head` is only its start. + src: Option<&'a dyn ByteSource>, + /// What the one read of `src` through [`search_through`] found. + #[cfg_attr( + not(any( + feature = "screnc", + feature = "autoit", + feature = "nsis", + feature = "sfx" + )), + allow(dead_code) + )] + found: std::cell::OnceCell, + /// The caller's read that makes the search, in place of one of its own. + #[cfg_attr( + not(any( + feature = "screnc", + feature = "autoit", + feature = "nsis", + feature = "sfx" + )), + allow(dead_code) + )] + prescan: Option<&'a dyn Fn() -> Prescan>, +} + +/// What [`detect`] searches a whole object for, found in one read of it the +/// first time one is asked for, rather than in a read each: where each of +/// [`searched`] first occurs, and for an executable, where the archive a +/// self-extractor carries starts. +#[derive(Default)] +// Which field a build reads depends on its format features. +#[allow(dead_code)] +struct Searched { + markers: Vec>, + sfx: Option, +} + +/// The needles [`detect`] searches a whole object for. +// Which pushes a build makes depends on its format features. +#[allow(clippy::vec_init_then_push)] +fn searched() -> Vec<&'static [u8]> { + #[allow(unused_mut)] + let mut needles: Vec<&'static [u8]> = Vec::new(); + #[cfg(feature = "screnc")] + needles.push(formats::SCRENC_MARKER); + #[cfg(feature = "nsis")] + needles.push(&formats::NSIS_SIG); + #[cfg(feature = "autoit")] + needles.extend([&formats::MARKER_EA05[..], &formats::MARKER_EA06[..]]); + needles +} + +/// What [`detect`] searches a whole object for, found as a caller reads the +/// object: fed every window of it in order, each running [`Self::OVERLAP`] +/// bytes into the next, and handed to [`detect_prescanned`] when it asks. One +/// read of an object then serves detection and whatever else the caller looks +/// for in it. +pub struct Prescan { + finders: Vec>, + markers: Vec>, + #[cfg(feature = "sfx")] + payload: Option, +} + +impl Prescan { + /// Bytes each window must run into the next, so that what it looks for is + /// seen whole across the seam: the longest marker, and a self-extractor's + /// archive magic with the header check after it. + pub const OVERLAP: usize = 16; + + /// Whether [`detect_prescanned`] can ask for the search on an object of + /// `len` bytes: not when the start it reads anyway is all of it. + pub fn needed(len: usize) -> bool { + len > DETECT_HEAD + } + + /// The search for an object that starts with `head`: a self-extractor's + /// payload is looked for only in an executable. + pub fn new(head: &[u8]) -> Self { + let needles = searched(); + #[cfg(not(feature = "sfx"))] + let _ = head; + Prescan { + finders: needles + .iter() + .map(|n| memchr::memmem::Finder::new(*n)) + .collect(), + markers: vec![None; needles.len()], + #[cfg(feature = "sfx")] + payload: (head.starts_with(b"MZ") || head.starts_with(b"\x7fELF")) + .then(formats::sfx::PayloadSearch::new), + } + } + + /// Search the window `w` at `base` in the object, its last or not. + pub fn feed(&mut self, base: usize, w: &[u8], last: bool) { + for (f, slot) in self.finders.iter().zip(&mut self.markers) { + if slot.is_none() { + *slot = f.find(w).map(|p| base + p); + } + } + #[cfg(feature = "sfx")] + if let Some(p) = &mut self.payload { + p.feed(base, w, last); + } + #[cfg(not(feature = "sfx"))] + let _ = last; + } + + /// Whether every search has its answer, so the rest of the object has + /// nothing more to tell it. + pub fn done(&self) -> bool { + #[cfg(feature = "sfx")] + let payload = self.payload.as_ref().is_none_or(|p| p.done()); + #[cfg(not(feature = "sfx"))] + let payload = true; + payload && self.markers.iter().all(Option::is_some) + } + + #[cfg_attr( + not(any( + feature = "screnc", + feature = "autoit", + feature = "nsis", + feature = "sfx" + )), + allow(dead_code) + )] + fn searched(self) -> Searched { + #[cfg(feature = "sfx")] + const { + assert!(formats::sfx::PayloadSearch::OVERLAP <= Prescan::OVERLAP) + }; + Searched { + markers: self.markers, + #[cfg(feature = "sfx")] + sfx: self.payload.and_then(|p| p.offset()), + #[cfg(not(feature = "sfx"))] + sfx: None, + } + } +} + +/// [`Searched`] in the first `len` bytes of `src`, which starts with `head`, +/// a window at a time. +#[cfg(any( + feature = "screnc", + feature = "autoit", + feature = "nsis", + feature = "sfx" +))] +fn search_through(src: &dyn ByteSource, len: usize, head: &[u8]) -> Searched { + let mut pre = Prescan::new(head); + let overlap = Prescan::OVERLAP; + let mut at = 0; + while at < len && !pre.done() { + let w = src.window(at, (len - at).min(source::CHUNK.max(overlap + 1))); + let last = w.len() <= overlap || at + w.len() >= len; + pre.feed(at, &w, last); + if last { + break; + } + at += w.len() - overlap; + } + pre.searched() +} + +impl<'a> Probe<'a> { + /// An object held in memory. + pub(crate) fn whole(data: &'a [u8]) -> Self { + Probe { + head: data, + len: data.len(), + src: None, + found: std::cell::OnceCell::new(), + prescan: None, + } + } + + /// Up to `n` bytes of the object from `off`. + pub(crate) fn window(&self, off: usize, n: usize) -> std::borrow::Cow<'a, [u8]> { + match self.src { + Some(src) => src.window(off, n), + None => ByteSource::window(self.head, off, n), + } + } + + /// Where `needle` first occurs in the object. + #[cfg(any(feature = "screnc", feature = "autoit", feature = "nsis"))] + pub(crate) fn find(&self, needle: &[u8]) -> Option { + let Some(src) = self.src else { + return memchr::memmem::find(self.head, needle); + }; + match searched().iter().position(|n| *n == needle) { + Some(i) => self.searched(src).markers[i], + None => src.find(needle, 0, self.len), + } + } + + /// For an object not held in memory, where the archive a self-extractor + /// carries starts, as `sfx::payload_offset` finds it. + #[cfg(feature = "sfx")] + pub(crate) fn sfx_payload(&self) -> Option { + self.searched(self.src?).sfx + } + + #[cfg(any( + feature = "screnc", + feature = "autoit", + feature = "nsis", + feature = "sfx" + ))] + fn searched(&self, src: &dyn ByteSource) -> &Searched { + self.found.get_or_init(|| match self.prescan { + Some(read) => read().searched(), + None => search_through(src, self.len, self.head), + }) + } + + /// The object, when it is not held whole in `head`. + #[cfg(any(feature = "uuencode", feature = "sfx"))] + pub(crate) fn source(&self) -> Option<&'a dyn ByteSource> { + self.src + } +} + +/// Bytes of an object's start the checks read, other than those that read its +/// end or search it. Inno's and InstallShield's markers are looked for in the +/// first 4 MiB, and InstallShield's fixed record lies past its marker. +const DETECT_HEAD: usize = 4 * 1024 * 1024 + 1024; + /// Best-effort recognition of a container by magic bytes. Detection is /// recognition-only: it reports what the bytes look like under the Cargo /// features compiled in, not what any build could extract. A format behind a /// disabled feature (CHM, FAT, ARC, among others) returns `None` rather than /// reaching extraction as `unsupported`. -pub fn detect(data: &[u8]) -> Option { +/// +/// Reads the object's start, and whatever else a check needs when the object +/// is not held in memory. +pub fn detect(src: &dyn ByteSource) -> Option { + detect_with(src, None) +} + +/// [`detect`], its search through `src` made by `read` when detection needs +/// it: a [`Prescan`] fed all of `src` by a read of it that serves the caller's +/// own searches too. +pub fn detect_prescanned(src: &dyn ByteSource, read: &dyn Fn() -> Prescan) -> Option { + detect_with(src, Some(read)) +} + +fn detect_with(src: &dyn ByteSource, prescan: Option<&dyn Fn() -> Prescan>) -> Option { + if let Some(data) = src.as_slice() { + return detect_probe(&Probe::whole(data)); + } + let head = src.window(0, DETECT_HEAD); + if head.len() == src.len() { + return detect_probe(&Probe::whole(&head)); + } + detect_probe(&Probe { + head: &head, + len: src.len(), + src: Some(src), + found: std::cell::OnceCell::new(), + prescan, + }) +} + +/// The archive `src` starts with, among those whose magic is at offset 0 and +/// is checked from its first bytes alone (ZIP, gzip, 7z, xz, bzip2, CAB, +/// RAR), as [`detect`] would type it: what an archive carved out of another +/// object at its magic needs confirmed, without a search through the rest. +pub fn detect_archive_start(src: &dyn ByteSource) -> Option { + let head = src.window(0, 16); + archive_magic(&head).or_else(|| is_rar_magic(&head).then_some(Format::Rar)) +} + +/// The first checks of [`detect_probe`], which read a few bytes of the start. +fn archive_magic(data: &[u8]) -> Option { if data.len() >= 4 && &data[..2] == b"PK" && matches!(data[2..4], [3, 4] | [5, 6] | [7, 8]) { return Some(Format::Zip); } @@ -1166,6 +1451,20 @@ pub fn detect(data: &[u8]) -> Option { if is_cab_magic(data) { return Some(Format::Cab); } + None +} + +fn is_rar_magic(data: &[u8]) -> bool { + data.starts_with(b"Rar!\x1a\x07\x00") || data.starts_with(b"Rar!\x1a\x07\x01\x00") +} + +fn detect_probe(p: &Probe) -> Option { + // Checks that read only the start: every length they compare against is + // well within `DETECT_HEAD`, so the start answers as the whole would. + let data = p.head; + if let Some(fmt) = archive_magic(data) { + return Some(fmt); + } // CHM (ITSS): "ITSF" header magic at offset 0. #[cfg(feature = "chm")] if data.starts_with(b"ITSF") { @@ -1183,17 +1482,17 @@ pub fn detect(data: &[u8]) -> Option { if data.len() >= 7 && data[2] == b'-' && data[6] == b'-' && matches!(data[3], b'l' | b'p') { return Some(Format::Lha); } - // ARJ: the 0x60 0xEA main-header magic. - if data.starts_with(&[0x60, 0xEA]) { + #[cfg(feature = "arj")] + if is_arj_magic(data) { return Some(Format::Arj); } - if data.starts_with(b"Rar!\x1a\x07\x00") || data.starts_with(b"Rar!\x1a\x07\x01\x00") { + if is_rar_magic(data) { return Some(Format::Rar); } if data.len() >= 32774 && &data[32769..32774] == b"CD001" { return Some(Format::Iso); } - // A UDF-only image has no ISO 9660 descriptor at all — Windows and macOS + // A UDF-only image has no ISO 9660 descriptor at all. Windows and macOS // still mount it, so it must not fall through as an unknown blob. Handled by // the same extractor, which walks whichever filesystems are present. #[cfg(feature = "iso")] @@ -1205,51 +1504,51 @@ pub fn detect(data: &[u8]) -> Option { } // Before DMG: a `.wim` carries its own 8-byte magic, while the DMG sniff is // structural and claims this file first if given the chance. - if formats::sniff::is(data, Format::Wim) { + if formats::sniff::is_in(p, Format::Wim) { return Some(Format::Wim); } - if formats::sniff::is(data, Format::Lz4) { + if formats::sniff::is_in(p, Format::Lz4) { return Some(Format::Lz4); } // ARC last of the archive magics: plus a method byte is only two bytes, // so the name-field validation in `is_arc` is what makes it safe, and a // stronger magic should still win the race. - if formats::sniff::is(data, Format::Ace) { + if formats::sniff::is_in(p, Format::Ace) { return Some(Format::Ace); } - if formats::sniff::is(data, Format::Alz) { + if formats::sniff::is_in(p, Format::Alz) { return Some(Format::Alz); } - if formats::sniff::is(data, Format::Egg) { + if formats::sniff::is_in(p, Format::Egg) { return Some(Format::Egg); } - if formats::sniff::is(data, Format::Hwp3) { + if formats::sniff::is_in(p, Format::Hwp3) { return Some(Format::Hwp3); } - if formats::sniff::is(data, Format::StuffIt) { + if formats::sniff::is_in(p, Format::StuffIt) { return Some(Format::StuffIt); } - if formats::sniff::is(data, Format::CryptFf) { + if formats::sniff::is_in(p, Format::CryptFf) { return Some(Format::CryptFf); } - if formats::sniff::is(data, Format::IshieldZ) { + if formats::sniff::is_in(p, Format::IshieldZ) { return Some(Format::IshieldZ); } - if formats::sniff::is(data, Format::IshieldCab) { + if formats::sniff::is_in(p, Format::IshieldCab) { return Some(Format::IshieldCab); } - if formats::sniff::is(data, Format::Lrzip) { + if formats::sniff::is_in(p, Format::Lrzip) { return Some(Format::Lrzip); } - if formats::sniff::is(data, Format::Zoo) { + if formats::sniff::is_in(p, Format::Zoo) { return Some(Format::Zoo); } - if formats::sniff::is(data, Format::AppleSingle) { + if formats::sniff::is_in(p, Format::AppleSingle) { return Some(Format::AppleSingle); } // ext last of the filesystem sniffs: its magic is 1080 bytes in, so a // partition table or boot sector at offset 0 is the more specific answer. - if formats::sniff::is(data, Format::Ext) { + if formats::sniff::is_in(p, Format::Ext) { return Some(Format::Ext); } // FAT before the partition check: a volume boot record and an MBR both end @@ -1258,30 +1557,30 @@ pub fn detect(data: &[u8]) -> Option { if formats::fat::is_fat(data) { return Some(Format::Fat); } - if formats::sniff::is(data, Format::Ntfs) { + if formats::sniff::is_in(p, Format::Ntfs) { return Some(Format::Ntfs); } #[cfg(feature = "arc")] - if formats::arc::is_arc(data) { + if formats::arc::is_arc_in(p) { return Some(Format::Arc); } #[cfg(feature = "dmg")] - if is_dmg(data) { + if is_dmg(p) { return Some(Format::Dmg); } - if formats::sniff::is(data, Format::Vhd) { + if formats::sniff::is_in(p, Format::Vhd) { return Some(Format::Vhd); } - if formats::sniff::is(data, Format::Lzw) { + if formats::sniff::is_in(p, Format::Lzw) { return Some(Format::Lzw); } - if formats::sniff::is(data, Format::Qcow2) { + if formats::sniff::is_in(p, Format::Qcow2) { return Some(Format::Qcow2); } - if formats::sniff::is(data, Format::Vmdk) { + if formats::sniff::is_in(p, Format::Vmdk) { return Some(Format::Vmdk); } - if formats::sniff::is(data, Format::Vhdx) { + if formats::sniff::is_in(p, Format::Vhdx) { return Some(Format::Vhdx); } if data.starts_with(&[0x28, 0xB5, 0x2F, 0xFD]) { @@ -1307,9 +1606,9 @@ pub fn detect(data: &[u8]) -> Option { return Some(Format::Szdd); } // uuencode has no byte-0 magic: require a `begin`/`begin-base64` opener with - // a matching terminator (conservative — see `looks_like_uuencode`). + // a matching terminator (conservative; see `looks_like_uuencode`). #[cfg(feature = "uuencode")] - if looks_like_uuencode(data) { + if looks_like_uuencode(p) { return Some(Format::Uuencode); } // Adobe XDP: XML wrapping a base64-encoded PDF. @@ -1331,7 +1630,7 @@ pub fn detect(data: &[u8]) -> Option { // Mach-O universal ("fat") binary: CAFEBABE/BF with a plausible arch table // (strict, to avoid the Java `.class` CAFEBABE collision). #[cfg(feature = "machofat")] - if looks_like_machofat(data) { + if looks_like_machofat(p) { return Some(Format::Machofat); } // Java `.class`: CAFEBABE followed by a plausible major version (>= 45, @@ -1347,12 +1646,12 @@ pub fn detect(data: &[u8]) -> Option { } // AI model: Python pickle (protocol 2–5 opener `80 0x`) or safetensors. #[cfg(feature = "aimodel")] - if is_aimodel(data) { + if is_aimodel(p) { return Some(Format::AiModel); } // Microsoft Script Encoder: the `#@~^` marker (VBScript/JScript.Encode). #[cfg(feature = "screnc")] - if looks_like_screnc(data) { + if looks_like_screnc(p) { return Some(Format::Screnc); } // OneNote (.one): the 16-byte OneStore section-header GUID at offset 0. @@ -1372,73 +1671,272 @@ pub fn detect(data: &[u8]) -> Option { ]) { return Some(Format::Lnk); } - // BinHex 4.0: a marker line (no byte-0 magic) — scan the head for it. + // BinHex 4.0: a marker line (no byte-0 magic); scan the head for it. #[cfg(feature = "binhex")] if looks_like_binhex(data) { return Some(Format::Binhex); } - // Python `.pyc`: weak magic (`\r\n` at offset 2) — checked late, conservative. + // Python `.pyc`: weak magic (`\r\n` at offset 2), checked late, conservative. #[cfg(feature = "pyc")] if data.len() >= 16 && data[2] == 0x0D && data[3] == 0x0A { return Some(Format::Pyc); } // Partition maps last: GPT/APM carry strong magic, but the MBR `55 AA` boot // signature is weak, so `is_partition` only accepts an MBR with a plausible - // entry — and being last means it never shadows a real format. + // entry, and being last means it never shadows a real format. // NSIS installer: PE stub + the NullsoftInst firstheader. Placed late (it // requires an MZ start, so it only claims a PE that is actually NSIS). #[cfg(feature = "nsis")] - if is_nsis(data) { + if is_nsis(p) { return Some(Format::Nsis); } // Compiled AutoIt3: the AU3!EA05/EA06 marker anywhere (embedded in a PE). #[cfg(feature = "autoit")] - if is_autoit(data) { + if is_autoit(p) { return Some(Format::Autoit); } // Inno Setup, before the generic SFX carve. The carve does produce the right - // block, but that block is Inno's own chunked LZMA container — emitted as an + // block, but that block is Inno's own chunked LZMA container. Emitted as an // ordinary member it read as clean, because compressed payload shows a // pattern scan nothing. Typing it here means it is reported instead. - if formats::sniff::is(data, Format::Inno) { + if formats::sniff::is_in(p, Format::Inno) { return Some(Format::Inno); } // After Inno and before the generic SFX carve, for the same reason: an // InstallShield installer is a PE whose payload the carve would mis-slice. - if formats::sniff::is(data, Format::IshieldMsi) { + if formats::sniff::is_in(p, Format::IshieldMsi) { return Some(Format::IshieldMsi); } // Self-extracting archive (PE/ELF stub + appended archive). After NSIS (more // specific) and only when a bare archive isn't at offset 0. #[cfg(feature = "sfx")] - if looks_like_sfx(data) { + if looks_like_sfx(p) { return Some(Format::Sfx); } #[cfg(feature = "partition")] - if is_partition(data) { + if is_partition(p) { return Some(Format::Partition); } None } -/// Per-member visitor for [`extract_each`]. Invoked once per extracted member -/// with the member and the shared [`Budget`] (so the visitor can recurse into a -/// nested container under the same budget). Returns `Some(r)` to stop extraction -/// immediately — `r` is propagated out of [`extract_each`] — or `None` to -/// continue to the next member. -pub type Sink<'a, R> = &'a mut dyn FnMut(Entry, &mut Budget) -> Option; +#[cfg(test)] +mod detect_source_tests { + use super::*; + use crate::source::BlockCache; + use std::io::Cursor; + + /// Longer than the start the probe reads. + const BIG: usize = DETECT_HEAD + 4096; + + fn same(data: &[u8], what: &str) -> Option { + let cache = BlockCache::with_sizes(Cursor::new(data.to_vec()), 4093, 64 * 4093).unwrap(); + let want = detect(&data); + assert_eq!(detect(&cache), want, "{what}"); + want + } + + fn fixtures(dir: &std::path::Path, out: &mut Vec<(String, Vec)>) { + for e in std::fs::read_dir(dir).unwrap().flatten() { + let path = e.path(); + if path.is_dir() { + fixtures(&path, out); + } else if let Ok(data) = std::fs::read(&path) { + if data.len() < 8 << 20 { + out.push((path.display().to_string(), data)); + } + } + } + } + + /// What one read of an object finds is where a search for each marker + /// finds it, and where a self-extractor's own search finds its payload, + /// whatever window seam they straddle, and nothing when absent. + #[cfg(all( + feature = "screnc", + feature = "autoit", + feature = "nsis", + feature = "sfx" + ))] + #[test] + fn markers_are_found_in_one_read() { + let needles = searched(); + let payload = |data: &[u8]| { + let mut r = source::Reader::new(&data); + formats::sfx::payload_offset(&mut r, data.len() as u64) + .unwrap() + .map(|o| o as usize) + }; + for shift in [0, 1, 7, 15, 16, 17] { + let mut data = vec![b'.'; 7 * source::CHUNK]; + data[..2].copy_from_slice(b"MZ"); + for (k, n) in needles.iter().enumerate() { + let at = (k + 1) * source::CHUNK - n.len() / 2 - shift; + data[at..at + n.len()].copy_from_slice(n); + // A second occurrence later, which must not be the one found. + let later = at + 3 * source::CHUNK / 2; + data[later..later + n.len()].copy_from_slice(n); + } + // An ARJ magic with no header behind it, then a ZIP, each across + // a seam. + let arj = 5 * source::CHUNK - 3 - shift; + data[arj..arj + 2].copy_from_slice(&[0x60, 0xEA]); + let zip = 6 * source::CHUNK - 2 - shift; + data[zip..zip + 4].copy_from_slice(b"PK\x03\x04"); + let cache = BlockCache::with_sizes(Cursor::new(data.clone()), 4093, 64 * 4093).unwrap(); + let want: Vec<_> = needles + .iter() + .map(|n| memchr::memmem::find(&data, n)) + .collect(); + let got = search_through(&cache, data.len(), &data[..2]); + assert_eq!(got.markers, want, "shift {shift}"); + assert_eq!(got.sfx, payload(&data), "shift {shift}"); + assert!(got.sfx.is_some()); + // Fed by a caller's own read, in windows of its own size. + let got = prescanned(&data, 10007); + assert_eq!(got.markers, want, "shift {shift}"); + assert_eq!(got.sfx, payload(&data), "shift {shift}"); + // Not an executable: no payload to look for. + data[..2].copy_from_slice(b".."); + assert_eq!(search_through(&cache, data.len(), &data[..2]).sfx, None); + } + let empty = vec![b'.'; 3 * source::CHUNK]; + let cache = BlockCache::with_sizes(Cursor::new(empty.clone()), 4093, 64 * 4093).unwrap(); + let got = search_through(&cache, empty.len(), b"MZ"); + assert!(got.markers.iter().all(Option::is_none) && got.sfx.is_none()); + for n in needles { + assert!(n.len() <= Prescan::OVERLAP + 1); + } + } + + fn prescanned(data: &[u8], window: usize) -> Searched { + let mut pre = Prescan::new(data); + let mut at = 0; + loop { + let end = (at + window).min(data.len()); + let last = end == data.len(); + pre.feed(at, &data[at..end], last); + if last { + return pre.searched(); + } + at = end - Prescan::OVERLAP; + } + } + + #[test] + fn a_large_object_is_typed_as_in_memory() { + let mut all = Vec::new(); + fixtures( + &std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"), + &mut all, + ); + assert!(all.len() > 100); + let mut typed = std::collections::HashSet::new(); + for (name, x) in &all { + // Longer than the probe's start, so the checks against the length + // and the searches run over the source. + let mut padded = x.clone(); + padded.resize(BIG.max(x.len() + 1), 0); + typed.insert(same(&padded, name)); + // And the fixture's own end kept at the end, for the checks that + // read there. + padded.extend_from_slice(&x[x.len().saturating_sub(512)..]); + typed.insert(same(&padded, &format!("{name}, end kept"))); + } + // Fewer with fewer format features. + assert!(typed.len() > 15, "{typed:?}"); + + // Markers only a search through the object finds, past its start. + let far = |head: &[u8], marker: &[u8], tail: &[u8]| { + let mut v = head.to_vec(); + v.resize(BIG, b'.'); + v.extend_from_slice(marker); + v.extend_from_slice(tail); + v + }; + let nsis_sig = [ + 0xEF, 0xBE, 0xAD, 0xDE, b'N', b'u', b'l', b'l', b's', b'o', b'f', b't', b'I', b'n', + b's', b't', + ]; + // Each case needs its format's detection compiled in. + let cases: Vec<(Vec, Option, bool)> = vec![ + ( + far(b"", b"#@~^", b""), + Some(Format::Screnc), + cfg!(feature = "screnc"), + ), + ( + far(b"", b"AU3!EA06", b""), + Some(Format::Autoit), + cfg!(feature = "autoit"), + ), + ( + far(b"MZ", &nsis_sig, b""), + Some(Format::Nsis), + cfg!(feature = "nsis"), + ), + ( + far(b"MZ", b"PK\x03\x04", b""), + Some(Format::Sfx), + cfg!(feature = "sfx"), + ), + ( + far(b"begin 644 x\n", b"\n end \n", b""), + Some(Format::Uuencode), + cfg!(feature = "uuencode"), + ), + (far(b"begin 644 x\n", b"\n endx\n", b""), None, true), + (far(b"", b"conectix", &[0; 504]), Some(Format::Vhd), true), + ( + far(b"", b"koly", &[0; 508]), + Some(Format::Dmg), + cfg!(feature = "dmg"), + ), + ]; + for (i, (data, want, on)) in cases.iter().enumerate() { + if *on { + assert_eq!(same(data, &format!("case {i}")), *want, "case {i}"); + } + } + + // A size read from the start, compared with the length. + if cfg!(feature = "partition") { + let mut mbr = vec![0u8; BIG + 1024]; + mbr[510..512].copy_from_slice(&[0x55, 0xAA]); + mbr[446] = 0x80; + mbr[450] = 0x0C; + mbr[458..462].copy_from_slice(&2u32.to_le_bytes()); + for (lba, want) in [ + ((BIG / 512) as u32, Some(Format::Partition)), + (u32::MAX / 2, None), + ] { + mbr[454..458].copy_from_slice(&lba.to_le_bytes()); + assert_eq!(same(&mbr, "mbr"), want, "lba {lba}"); + } + } + if cfg!(feature = "aimodel") { + let mut st = vec![b' '; BIG + 100]; + for (n, want) in [ + (BIG as u64, Some(Format::AiModel)), + (BIG as u64 + 100, None), + ] { + st[..8].copy_from_slice(&n.to_le_bytes()); + st[8..12].copy_from_slice(b"{\"a\""); + assert_eq!(same(&st, "safetensors"), want, "n {n}"); + } + } + } +} + +/// Per-member visitor of an extractor that decodes each member whole. Invoked +/// once per member with the shared [`Budget`]. Returns `Some(r)` to stop the +/// extraction, `None` to continue to the next member. +pub(crate) type Sink<'a, R> = &'a mut dyn FnMut(Entry, &mut Budget) -> Option; -/// Stream the immediate members of a container (one level), invoking `visit` for -/// each under the shared `budget`. Members are decoded one at a time, so peak -/// memory is ~one member and a `visit` that returns `Some(r)` halts extraction -/// before the remaining members are decompressed. -/// -/// Returns `Ok(Some(r))` if `visit` stopped early, `Ok(None)` if every member -/// was visited, or `Err` on a budget bound. /// Fail the way a decoder can, for input carrying a marker that asks for it. /// /// The marker is looked for ANYWHERE in the data, not just at the start, so the -/// trigger can be carried inside a genuinely well-formed container — which is +/// trigger can be carried inside a genuinely well-formed container, which is /// what lets the CLI be tested end to end: the file has to survive format /// detection to reach a decoder at all. /// @@ -1447,14 +1945,14 @@ pub type Sink<'a, R> = &'a mut dyn FnMut(Entry, &mut Budget) -> Option; /// firing: what a caller can observe then is the process's exit status, and a /// crash that looks like a clean scan is the worst outcome exav has. #[cfg(feature = "testing-faults")] -fn provoke(data: &[u8]) { +pub(crate) fn provoke(data: &[u8]) { let has = |m: &[u8]| data.windows(m.len()).any(|w| w == m); if has(b"__exav_panic__") { panic!("deliberate panic from the testing-faults feature"); } if has(b"__exav_abort__") { // `abort` rather than a real allocation failure. The outcome is the one - // being tested — `handle_alloc_error` aborts — and exhausting a + // being tested (`handle_alloc_error` aborts), and exhausting a // developer's machine to reach it would take the rest of the box with // it. On wasm32 the 4 GiB ceiling makes the genuine article safe; here // it is not. @@ -1472,175 +1970,61 @@ fn provoke(data: &[u8]) { } } -pub fn extract_each( +/// The extractors of the formats read whole, each decoding a member whole, for +/// [`walk`] (inside its panic boundary). +pub(crate) fn dispatch_extract( fmt: Format, data: &[u8], budget: &mut Budget, visit: Sink, ) -> Result, LimitHit> { - // Containment boundary. Some third-party decoders (e.g. `cab`, `sevenz`, - // `pdf`) panic on crafted/truncated input instead of returning an error; - // a scanner must never crash on the files it scans. Catch any unwinding - // panic here and turn it into a clean bound, so hostile input is reported - // (LimitsExceeded → never a silent Clean), not a process abort. This crate - // is `#![forbid(unsafe_code)]` and pure-Rust, so there is no UB to leak - // across the boundary. - // - // What this does NOT contain, because `catch_unwind` catches unwinding and - // nothing else: - // - // * an allocation large enough to abort — a decoder that sizes a buffer - // from a header field can ask for more than the machine has, and Rust - // aborts rather than unwinding, so the process is gone before this - // line runs; - // * stack exhaustion from a file nested into itself — `max_recursion` - // bounds CONTAINER nesting, not recursive descent inside one parser; - // * a loop that neither allocates nor returns. - // - // The daemon holds those with `RLIMIT_AS`, `RLIMIT_CPU` and worker - // replacement. A one-shot run and a library embedding have no equivalent, - // which is why `SECURITY.md` says so rather than implying this boundary is - // total. - // A format the caller excluded at runtime. Reported, not skipped: the - // container is there, exav declined to open it, and the scan has to be able - // to say so. Checked here rather than at each call site because every - // extraction — including one nested inside another archive — passes through - // this function. - if !budget.limits().allows(fmt) { - return Ok(visit( - Entry::unsupported( - format!("{fmt:?}"), - data.len() as u64, - false, - "format excluded by the caller's allowed_formats", - ), - budget, - )); - } - let caught = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - dispatch_extract(fmt, data, budget, visit) - })); - match caught { - Ok(r) => r, - Err(_) => Err(LimitHit::corrupt(format!( - "{fmt:?} decoder panicked on malformed input" - ))), - } -} - -/// Format dispatch for [`extract_each`]; kept separate so the panic-containment -/// boundary in `extract_each` wraps every decoder uniformly. -fn dispatch_extract( - fmt: Format, - data: &[u8], - budget: &mut Budget, - visit: Sink, -) -> Result, LimitHit> { - // Panic on demand, to test the boundary rather than the decoders. - // - // The fixtures in `panic_containment` cover inputs that once panicked a - // specific decoder — real regressions, worth keeping. But they do not test - // the `catch_unwind` in `extract_each`: fix every one of those decoders and - // they all still pass with the boundary deleted. This panics from inside the - // dispatch for input nothing else produces, so the only thing that can turn - // it into a `LimitHit` is the boundary itself. - #[cfg(feature = "testing-faults")] - provoke(data); match fmt { - #[cfg(feature = "gzip")] - Format::Gzip => extract_gzip(data, budget, visit), - #[cfg(feature = "tar")] - Format::Tar => extract_tar(data, budget, visit), + // A ZIP whose central directory will not parse: the directory it has, + // then the local-header salvage. #[cfg(feature = "zip")] Format::Zip => extract_zip(data, budget, visit), - #[cfg(feature = "bzip2")] - Format::Bzip2 => extract_bzip2(data, budget, visit), - #[cfg(feature = "xz")] - Format::Xz => extract_xz(data, budget, visit), - #[cfg(feature = "cab")] - Format::Cab => extract_cab(data, budget, visit), #[cfg(feature = "chm")] Format::Chm => extract_chm(data, budget, visit), // OLE builds a combined VBA-macro dump from all streams, so it collects - // then emits — the visitor still gets one member at a time. + // then emits; the visitor still gets one member at a time. #[cfg(feature = "ole")] Format::Ole => emit_collected(extract_ole(data, budget)?, budget, visit), #[cfg(feature = "pdf")] Format::Pdf => extract_pdf(data, budget, visit), #[cfg(feature = "email")] Format::Email => extract_email(data, budget, visit), - #[cfg(feature = "sevenz")] - Format::SevenZip => extract_sevenz(data, budget, visit), - #[cfg(feature = "iso")] - Format::Iso => extract_iso(data, budget, visit), - #[cfg(feature = "lha")] - Format::Lha => extract_lha(data, budget, visit), #[cfg(feature = "arj")] Format::Arj => extract_arj(data, budget, visit), // RAR decodes members eagerly (CRC checks, metadata-only entries for - // encrypted/unsupported members), so it collects then emits — the visitor + // encrypted/unsupported members), so it collects then emits; the visitor // still gets one member at a time and can stop early. #[cfg(feature = "rar")] Format::Rar => emit_collected(extract_rar(data, budget)?, budget, visit), #[cfg(feature = "upx")] Format::Upx => extract_upx(data, budget, visit), - #[cfg(feature = "ar")] - Format::Ar => extract_ar(data, budget, visit), - #[cfg(feature = "cpio")] - Format::Cpio => extract_cpio(data, budget, visit), #[cfg(feature = "xar")] Format::Xar => extract_xar(data, budget, visit), - #[cfg(feature = "dmg")] - Format::Dmg => extract_dmg(data, budget, visit), #[cfg(feature = "vhd")] Format::Vhd => formats::vhd::extract_vhd(data, budget, visit), - #[cfg(not(feature = "vhd"))] - Format::Vhd => not_compiled_in(fmt, data, budget, visit), - #[cfg(feature = "lzw")] - Format::Lzw => formats::lzw::extract_lzw(data, budget, visit), - #[cfg(not(feature = "lzw"))] - Format::Lzw => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "diskimage")] Format::Qcow2 => formats::qcow2::extract_qcow2(data, budget, visit), - #[cfg(not(feature = "diskimage"))] - Format::Qcow2 => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "diskimage")] Format::Vmdk => formats::vmdk::extract_vmdk(data, budget, visit), - #[cfg(not(feature = "diskimage"))] - Format::Vmdk => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "diskimage")] Format::Vhdx => formats::vhdx::extract_vhdx(data, budget, visit), - #[cfg(not(feature = "diskimage"))] - Format::Vhdx => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "wim")] Format::Wim => formats::wim::extract_wim(data, budget, visit), - #[cfg(not(feature = "wim"))] - Format::Wim => not_compiled_in(fmt, data, budget, visit), - #[cfg(feature = "lz4")] - Format::Lz4 => formats::lz4::extract_lz4(data, budget, visit), - #[cfg(not(feature = "lz4"))] - Format::Lz4 => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "arc")] Format::Arc => formats::arc::extract_arc(data, budget, visit), - #[cfg(not(feature = "arc"))] - Format::Arc => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "ace")] Format::Ace => formats::ace::extract_ace(data, budget, visit), - #[cfg(not(feature = "ace"))] - Format::Ace => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "alz")] Format::Alz => formats::alz::extract_alz(data, budget, visit), - #[cfg(not(feature = "alz"))] - Format::Alz => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "egg")] Format::Egg => formats::egg::extract_egg(data, budget, visit), - #[cfg(not(feature = "egg"))] - Format::Egg => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "hwp3")] Format::Hwp3 => formats::hwp3::extract_hwp3(data, budget, visit), - #[cfg(not(feature = "hwp3"))] - Format::Hwp3 => not_compiled_in(fmt, data, budget, visit), - // Recognised, not opened — all share one reporting path. + // Recognised, not opened; all share one reporting path. Format::IshieldMsi | Format::IshieldCab | Format::CryptFf @@ -1648,64 +2032,33 @@ fn dispatch_extract( | Format::AppleSingle => formats::reported::extract_reported(fmt, data, budget, visit), #[cfg(feature = "ext")] Format::Ext => formats::ext::extract_ext(data, budget, visit), - #[cfg(not(feature = "ext"))] - Format::Ext => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "zoo")] Format::Zoo => formats::zoo::extract_zoo(data, budget, visit), - #[cfg(not(feature = "zoo"))] - Format::Zoo => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "ishieldz")] Format::IshieldZ => formats::ishield_z::extract_ishield_z(data, budget, visit), - #[cfg(not(feature = "ishieldz"))] - Format::IshieldZ => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "stuffit")] Format::StuffIt => formats::stuffit::extract_stuffit(data, budget, visit), - #[cfg(not(feature = "stuffit"))] - Format::StuffIt => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "fat")] Format::Fat => formats::fat::extract_fat(data, budget, visit), - #[cfg(not(feature = "fat"))] - Format::Fat => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "inno")] Format::Inno => formats::inno::extract_inno(data, budget, visit), - #[cfg(not(feature = "inno"))] - Format::Inno => not_compiled_in(fmt, data, budget, visit), #[cfg(feature = "ntfs")] Format::Ntfs => formats::ntfs::extract_ntfs(data, budget, visit), - #[cfg(not(feature = "ntfs"))] - Format::Ntfs => not_compiled_in(fmt, data, budget, visit), - #[cfg(feature = "zstd")] - Format::Zstd => extract_zstd(data, budget, visit), - #[cfg(feature = "lzip")] - Format::Lzip => extract_lzip(data, budget, visit), #[cfg(feature = "uuencode")] Format::Uuencode => extract_uuencode(data, budget, visit), #[cfg(feature = "xdp")] Format::Xdp => extract_xdp(data, budget, visit), + // KWAJ; SZDD itself is decoded as it is read. #[cfg(feature = "szdd")] Format::Szdd => extract_szdd(data, budget, visit), - #[cfg(feature = "tnef")] - Format::Tnef => extract_tnef(data, budget, visit), - #[cfg(feature = "swf")] - Format::Swf => extract_swf(data, budget, visit), #[cfg(feature = "binhex")] Format::Binhex => extract_binhex(data, budget, visit), #[cfg(feature = "lnk")] Format::Lnk => extract_lnk(data, budget, visit), - #[cfg(feature = "partition")] - Format::Partition => extract_partition(data, budget, visit), - #[cfg(feature = "pyc")] - Format::Pyc => extract_pyc(data, budget, visit), #[cfg(feature = "nsis")] Format::Nsis => extract_nsis(data, budget, visit), - #[cfg(feature = "machofat")] - Format::Machofat => extract_machofat(data, budget, visit), - #[cfg(feature = "sfx")] - Format::Sfx => extract_sfx(data, budget, visit), #[cfg(feature = "autoit")] Format::Autoit => extract_autoit(data, budget, visit), - #[cfg(feature = "onenote")] - Format::OneNote => extract_onenote(data, budget, visit), #[cfg(feature = "rtf")] Format::Rtf => extract_rtf(data, budget, visit), #[cfg(feature = "pepack")] @@ -1716,9 +2069,8 @@ fn dispatch_extract( Format::AiModel => extract_aimodel(data, budget, visit), #[cfg(feature = "screnc")] Format::Screnc => extract_screnc(data, budget, visit), - // Unreachable when `all-formats` is on (every arm above exists then). - #[cfg(not(feature = "all-formats"))] - #[allow(unreachable_patterns)] + // A format whose extractor this build left out. (A format decoded as it + // is read never reaches here when its extractor is compiled in.) _ => not_compiled_in(fmt, data, budget, visit), } } @@ -1729,7 +2081,6 @@ fn dispatch_extract( /// Every disabled-format arm must route here rather than return `Ok(None)`: /// `Ok(None)` means "no members", which the caller cannot distinguish from an /// empty archive, and the file then scans clean. -#[cfg(not(feature = "all-formats"))] fn not_compiled_in( fmt: Format, data: &[u8], @@ -1763,34 +2114,67 @@ pub(crate) fn emit_collected( Ok(None) } -/// Collect the immediate members of a container into a `Vec` (buffers them all -/// at once). Prefer [`extract_each`] for scanning — it streams member-by-member -/// and can stop early. Kept for callers/tests that want the full list. +/// Collect the immediate members of a container into a `Vec`, each read into +/// memory up to what [`Budget::reserve`] allows. [`walk`] is the call for +/// scanning: it hands members over one at a time and can stop early. This is +/// for callers that want the whole list. +/// +/// A member whose decoding failed part way keeps the bytes decoded before the +/// failure and is marked `unsupported`, unless checksums are verified, when +/// the failure is an error. /// /// # Errors /// /// Returns [`LimitHit`], whose two shapes mean different things and should not /// be collapsed into one "failed" branch: /// -/// * `corrupt == true` — the container could not be decoded: malformed, +/// * `corrupt == true`: the container could not be decoded: malformed, /// truncated, an unsupported compression method, or a decoder panic caught at /// the extraction boundary. The right verdict is `Unscannable`. -/// * `corrupt == false` — a budget stopped the walk, and `kind` names which +/// * `corrupt == false`: a budget stopped the walk, and `kind` names which /// one. The right verdict is `LimitsExceeded`. /// -/// In both cases some members may already have been decoded and are discarded -/// along with the error; use [`extract_each`] if partial results matter. +/// In both cases the members already decoded are discarded along with the +/// error; use [`walk`] if partial results matter. /// /// **An error is never a reason to treat the input as clean.** A container this /// call refused is content that was not scanned, which is the one outcome the /// verdict model exists to keep distinguishable from an empty result. -pub fn extract(fmt: Format, data: &[u8], budget: &mut Budget) -> Result, LimitHit> { +pub fn extract( + fmt: Format, + src: &dyn ByteSource, + budget: &mut Budget, +) -> Result, LimitHit> { let mut out = Vec::new(); - extract_each::(fmt, data, budget, &mut |e, _| { - out.push(e); + let stopped = walk(fmt, src, budget, &mut |meta, content, budget| { + let mut entry = Entry { + name: meta.name.clone(), + data: Vec::new(), + comp_size: meta.comp_size, + encrypted: meta.encrypted, + unsupported: meta.unsupported, + }; + if let Some(content) = content { + match content.into_bytes(meta, budget) { + Ok((data, partial)) => { + entry.data = data; + if partial && entry.unsupported.is_none() { + entry.unsupported = Some( + "member failed to decode part way; the bytes before the \ + failure are kept", + ); + } + } + Err(hit) => return Some(hit), + } + } + out.push(entry); None })?; - Ok(out) + match stopped { + Some(hit) => Err(hit), + None => Ok(out), + } } /// True if `data` looks like a UPX-packed executable (a valid `PackHeader` is @@ -1801,7 +2185,7 @@ pub fn is_upx(data: &[u8]) -> bool { { // Both layouts the unpacker handles: the `l_info` chain, and a bare // PackHeader. Gating on `find_packheader` alone meant a PackHeader-only - // image — a routine shape for packed malware — reached no unpacker at + // image (a routine shape for packed malware) reached no unpacker at // all and scanned clean. find_packheader(data).is_some() || formats::has_packheader_layout(data) } @@ -1814,7 +2198,7 @@ pub fn is_upx(data: &[u8]) -> bool { /// Run the PE-packer emulator over `data` and report what it did, for the /// `pepack_emu` example. Returns a one-line summary plus the reconstructed -/// image when the stub produced one. Diagnostic surface only — the scan path +/// image when the stub produced one. Diagnostic surface only: the scan path /// goes through [`extract`]; may change in any release. #[cfg(feature = "pe-emu")] #[doc(hidden)] @@ -1839,14 +2223,14 @@ pub fn is_pepack(data: &[u8]) -> bool { } /// Ceiling for a `Vec::with_capacity` pre-allocation driven by an -/// attacker-declared size (16 MiB). The buffer still grows on demand — bounded -/// by the budget-checked reads — so this only prevents a crafted header from +/// attacker-declared size (16 MiB). The buffer still grows on demand, bounded +/// by the budget-checked reads, so this only prevents a crafted header from /// forcing a huge up-front allocation (the over-allocation DoS class; cf. the /// ClamAV 7z/InstallShield advisories and the fuzz-found delharc OOM). // Dead only in a build with none of the formats that pre-allocate (dmg, cab, // 7z, ppmd7). Listing those features here instead would have to be corrected // every time one of them starts or stops calling this, and getting that list -// wrong is a warning rather than an error — so it would rot quietly. +// wrong is a warning rather than an error, so it would rot quietly. #[allow(dead_code)] pub(crate) const PREALLOC_CAP: usize = 16 * 1024 * 1024; @@ -1863,7 +2247,7 @@ pub(crate) fn cap_prealloc(requested: usize) -> usize { /// /// Enforced only past [`RATIO_FLOOR_BYTES`] of output: below it the absolute /// caps already bound the allocation and the content is cheap to scan, while a -/// bare ratio trips on ordinary content — a 1 KB-compressed blank scanned page +/// bare ratio trips on ordinary content: a 1 KB-compressed blank scanned page /// (1 MB of one byte) has a ratio over 1000:1 without being anyone's bomb. A /// real bomb still trips as soon as its output crosses the floor, milliseconds /// into the decompression. @@ -1894,10 +2278,10 @@ pub(crate) fn ratio_guard(input: u64, output: u64, budget: &Budget) -> Result<() /// /// Sized to the evidence rather than generously: 4x the 1 MB page image that /// makes the floor necessary. Every byte of headroom past that is ratio -/// checking given up for nothing — a bomb is still caught the moment its output +/// checking given up for nothing: a bomb is still caught the moment its output /// crosses the floor, milliseconds into decompression, and real bombs overshoot /// by orders of magnitude. -const RATIO_FLOOR_BYTES: u64 = 4 * 1024 * 1024; +pub(crate) const RATIO_FLOOR_BYTES: u64 = 4 * 1024 * 1024; /// The EICAR anti-virus test string, assembled at runtime from its reverse. /// @@ -1905,7 +2289,7 @@ const RATIO_FLOOR_BYTES: u64 = 4 * 1024 * 1024; /// binary built from it. That is deliberate: a scanner is a file every other /// scanner reads. Stored as a plain literal it would travel into the compiled /// binary, the `.crate` tarballs on crates.io and the container image, and every -/// AV worth installing would quarantine all three on sight — which reads as a +/// AV worth installing would quarantine all three on sight, which reads as a /// broken release rather than as the test string it is. ClamAV keeps EICAR in a /// signature database, not in its binary, for the same reason. /// @@ -1936,14 +2320,14 @@ pub const FIXTURE_MASK: u8 = 0x5A; /// /// A committed fixture that a scanner detects is a fixture that gets quarantined /// on `git clone`, deleted by an AV-scanned CI runner, and flagged by whatever -/// watches a contributor's laptop — for files whose entire job is to be detected. +/// watches a contributor's laptop, for files whose entire job is to be detected. /// Masking them removes the archive magic and the payload in one step, so no /// scanner has anything to match, while the bytes stay one XOR away. /// /// XOR rather than the password-protected ZIP that is standard for distributing /// samples: these fixtures are the corpus for an archive extractor, so wrapping /// them in archives would make the ZIP and 7z tests depend on working ZIP and -/// decryption support to load their own inputs — and the `--no-default-features` +/// decryption support to load their own inputs, and the `--no-default-features` /// build has neither compiled in. /// /// Test support, not part of the stable API: may change in any release. @@ -2070,599 +2454,6 @@ pub(crate) fn bounded_read_salvage( }) } -// --------------------------------------------------------------------------- -// Archive — format-agnostic streaming member access -// --------------------------------------------------------------------------- - -/// State for buffered (non-seekable) format extraction. -struct BufState { - next_index: usize, - /// Lazily populated on first extraction. Once filled, members are yielded - /// one at a time via `next_index`. - cached: Option>, - /// The same members as [`MemberInfo`], so `list` can hand out a slice - /// without re-walking. Filled together with `cached`, hence complete - /// whenever it is non-empty. - info: Vec, -} - -/// The [`MemberInfo`] projection of fully-extracted members. -fn member_infos(entries: &[Entry]) -> Vec { - entries - .iter() - .enumerate() - .map(|(index, e)| MemberInfo { - name: e.name.clone(), - index, - compressed_size: e.comp_size, - uncompressed_size: e.data.len() as u64, - encrypted: e.encrypted, - }) - .collect() -} - -/// An opened archive with lazy, member-by-member extraction. -/// -/// The public API is format-agnostic: [`open`](Archive::open) detects the -/// container, [`extract_next`](Archive::extract_next) pulls members one at a -/// time under a shared [`Budget`], and [`extract`](Archive::extract) / -/// [`extract_all`](Archive::extract_all) provide random-access and collect-all -/// convenience. -/// -/// For seekable formats (ZIP today) the central directory is read once on -/// [`Archive::open`] and individual members are fetched on demand without loading -/// the whole file. For non-seekable formats the data is read into memory on -/// [`Archive::open`] and members are extracted lazily from the buffer. -pub struct Archive { - format: Format, - inner: ArchiveInner, -} - -enum ArchiveInner { - #[cfg(feature = "zip")] - Zip { - members: ZipMembers, - info: Vec, - }, - #[cfg(feature = "gzip")] - Gzip { - reader: Option, - done: bool, - }, - #[cfg(feature = "tar")] - Tar { - reader: Option, - members: Vec, - /// The same members as [`MemberInfo`], so `list` can hand out a slice. - info: Vec, - next_index: usize, - }, - Lazy { - reader: Option, - format: Format, - state: BufState, - }, - Buffered { - data: Vec, - state: BufState, - }, -} - -/// Pre-parsed tar member metadata. -#[cfg(feature = "tar")] -struct TarMember { - name: String, - size: u64, - data_offset: u64, -} - -#[cfg(feature = "tar")] -impl TarMember { - fn parse(header: &[u8; 512]) -> Option { - // End-of-archive: two consecutive all-zero blocks. - if header.iter().all(|&b| b == 0) { - return None; - } - let name_raw = &header[..100]; - let name_end = name_raw.iter().position(|&b| b == 0).unwrap_or(100); - if name_end == 0 { - return None; - } - let name = String::from_utf8_lossy(&name_raw[..name_end]).into_owned(); - - // size: bytes 124..136, octal ASCII. - // - // POSIX leaves the terminator open: the field may end in NUL, in a space, - // or in both, and writers disagree. GNU tar, python's tarfile and ustar - // all write digits then NUL; node-tar — and therefore every npm package - // tarball — writes digits, a space, then a NUL. - // - // Trimming whitespace before NULs cannot handle that second form. NUL is - // not whitespace to `trim`, so it strips nothing, and stripping the NUL - // afterwards leaves the space behind for `from_str_radix` to reject. The - // failure is silent and total: an unparseable size ends the walk, so a - // first-member failure yields ZERO members and the archive scans as an - // empty tar. Strip both characters, from both ends, in one pass. - let size_str = std::str::from_utf8(&header[124..136]).ok()?; - let size = u64::from_str_radix(size_str.trim_matches(['\0', ' ']), 8).ok()?; - - // typeflag: byte 156 — skip directories and non-regular files. - let typeflag = header[156]; - if typeflag == b'5' || typeflag == b'1' || typeflag == b'2' { - // directory, hard link, symlink — skip but still advance - } - - Some(TarMember { - name, - size, - data_offset: 0, - }) - } -} - -/// Parse all tar headers from the reader, returning member metadata with -/// correct `data_offset` values. The reader must be positioned at the start. -#[cfg(feature = "tar")] -fn parse_tar_headers(reader: &mut R) -> Result, std::io::Error> { - let mut members = Vec::new(); - let mut offset = 0u64; - loop { - reader.seek(std::io::SeekFrom::Start(offset))?; - let mut header = [0u8; 512]; - reader.read_exact(&mut header)?; - match TarMember::parse(&header) { - Some(mut m) => { - m.data_offset = offset + 512; - members.push(m); - // Advance past header + padded data. - let data_end = offset + 512 + ((members.last().unwrap().size + 511) & !511); - offset = data_end; - } - // Two different things end the walk here, and they are not the same - // fact. An all-zero block IS the archive's end marker; a header that - // will not parse is a member the walk cannot get past, and stopping - // on it silently truncates the archive at that point. When the very - // first header is the one that fails, that means ZERO members and the - // file scans as an empty tar — clean. - // - // That is not hypothetical: one mis-ordered trim in the size field - // (fixed above) made every npm package tarball do exactly this. - // - // Members already recovered are kept — surfacing the damage must not - // cost the coverage we DID get, which is the same rule the ZIP - // walkers follow for an unreadable member. The consequence is that a - // mid-archive header failure still stops quietly, since this - // signature has nowhere to carry "these members, and also a problem"; - // only the total failure is reported. Narrowing that further needs - // the return type to change and is tracked separately. - None => { - if header.iter().all(|&b| b == 0) || !members.is_empty() { - break; - } - // Nothing recovered at all: the FIRST header did not parse, so - // this is not a short archive, it is one we could not read. Say - // so. Reporting it Unscannable is the only honest answer: an - // empty member list here makes a tarball whose size field uses a - // terminator this reader mishandles scan as a clean empty - // archive. - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("tar: unparseable header at offset {offset}"), - )); - } - } - } - Ok(members) -} - -impl Archive { - /// Detect the container format, parse format-specific headers (e.g. the - /// ZIP central directory), and return an [`Archive`] ready for member - /// extraction. - /// - /// For ZIP the reader is kept alive and seeked on demand (only the central - /// directory is read up front). For all other formats the remaining bytes - /// are read into memory so the existing buffer-based extractors are reused. - pub fn open(mut reader: R) -> Result { - // Read up to 64 KiB for format detection. - let mut head = Vec::with_capacity(65536); - reader - .by_ref() - .take(65536) - .read_to_end(&mut head) - .map_err(|e| LimitHit::corrupt(format!("read head: {e}")))?; - - let format = - detect(&head).ok_or_else(|| LimitHit::corrupt("unrecognised archive format".into()))?; - - // Seekable per-format fast paths (only for the compiled formats). Each - // returns early; anything not handled falls through to the buffered - // path below, which re-reads the object and routes it through the same - // `extract_each` dispatch (so a disabled format lands on the - // "unsupported" fallback there rather than being silently skipped). - #[cfg(feature = "zip")] - if format == Format::Zip { - // Seek back to the start so the ZIP central-directory parser sees - // the full file. - reader - .seek(std::io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("seek: {e}")))?; - let mut members = ZipMembers::open(reader).map_err(|e| LimitHit::corrupt(e.reason))?; - let info = members.list_entries(); - return Ok(Self { - format, - inner: ArchiveInner::Zip { members, info }, - }); - } - #[cfg(feature = "gzip")] - if format == Format::Gzip { - reader - .seek(std::io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("seek: {e}")))?; - return Ok(Self { - format, - inner: ArchiveInner::Gzip { - reader: Some(reader), - done: false, - }, - }); - } - #[cfg(feature = "tar")] - if format == Format::Tar { - reader - .seek(std::io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("seek: {e}")))?; - let members = parse_tar_headers(&mut reader) - .map_err(|e| LimitHit::corrupt(format!("tar: {e}")))?; - // A tar's headers ARE its index: every member's name, size and - // offset is known once they are parsed, which is what lets - // `extract` seek straight to one. Reporting that through `list` is - // what makes a caller able to see what an archive holds without - // decompressing it — and an empty list would say it holds nothing. - // Tar stores members uncompressed, so the two sizes are one size. - let info = members - .iter() - .enumerate() - .map(|(index, m)| MemberInfo { - name: m.name.clone(), - index, - compressed_size: m.size, - uncompressed_size: m.size, - encrypted: false, - }) - .collect(); - return Ok(Self { - format, - inner: ArchiveInner::Tar { - reader: Some(reader), - members, - info, - next_index: 0, - }, - }); - } - if matches!(format, Format::Bzip2 | Format::Xz | Format::Zstd) { - reader - .seek(std::io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("seek: {e}")))?; - return Ok(Self { - format, - inner: ArchiveInner::Lazy { - reader: Some(reader), - format, - state: BufState { - next_index: 0, - cached: None, - info: Vec::new(), - }, - }, - }); - } - // Buffered fallback: read the rest of the reader into memory, bounded by - // the global peak-buffer limit. `Archive::open` predates the budget, so - // it uses the default limit; callers wanting a different ceiling drive - // extraction through the budgeted [`extract_each`]/[`stream_members`] APIs. - // (In-tree, exav-core reaches `open` only for ZIP, which returns above - // before this fallback — so this bounds the public API, not the scanner.) - let max_buffer = Limits::default().max_buffer_bytes; - let mut data = head; - reader - .take( - max_buffer - .saturating_add(1) - .saturating_sub(data.len() as u64), - ) - .read_to_end(&mut data) - .map_err(|e| LimitHit::corrupt(format!("read: {e}")))?; - if data.len() as u64 > max_buffer { - return Err(LimitHit::new(format!( - "container exceeds max-buffer {max_buffer}" - ))); - } - Ok(Self { - format, - inner: ArchiveInner::Buffered { - data, - state: BufState { - next_index: 0, - cached: None, - info: Vec::new(), - }, - }, - }) - } - - /// Detected container format. - pub fn format(&self) -> Format { - self.format - } - - /// Pre-parsed member metadata. For ZIP this is free (central directory); - /// for tar it comes from the headers. For lazy/buffered formats the list - /// is empty until the first extraction call populates the cache, after - /// which it reports every member (the cache always fills completely or - /// not at all, so a non-empty list is a complete one). - pub fn list(&self) -> &[MemberInfo] { - match &self.inner { - #[cfg(feature = "zip")] - ArchiveInner::Zip { info, .. } => info, - #[cfg(feature = "gzip")] - ArchiveInner::Gzip { .. } => &[], - #[cfg(feature = "tar")] - ArchiveInner::Tar { info, .. } => info, - ArchiveInner::Lazy { state, .. } => &state.info, - ArchiveInner::Buffered { state, .. } => &state.info, - } - } - - /// Extract the next member under budget. Returns `Ok(None)` when the - /// archive is exhausted. Directories and other non-file entries are - /// skipped (but still counted toward the file-count budget). - pub fn extract_next(&mut self, budget: &mut Budget) -> Result, LimitHit> { - match &mut self.inner { - #[cfg(feature = "zip")] - ArchiveInner::Zip { members, .. } => members.next_member(budget).transpose(), - #[cfg(feature = "gzip")] - ArchiveInner::Gzip { reader, done } => { - if *done { - return Ok(None); - } - let r = reader - .take() - .ok_or_else(|| LimitHit::corrupt("gzip: already extracted".into()))?; - budget.count_entry()?; - let cap = budget.reserve()?; - let s = gunzip(std::io::BufReader::new(r), cap, budget) - .map_err(|e| LimitHit::corrupt(format!("gzip: {e}")))?; - if s.over_cap { - return Err(LimitHit::new("gzip member exceeds budget".to_string())); - } - budget.commit(s.data.len() as u64); - *done = true; - Ok(Some(gzip_entry(s))) - } - #[cfg(feature = "tar")] - ArchiveInner::Tar { - reader, - members, - next_index, - .. - } => { - if *next_index >= members.len() { - return Ok(None); - } - let r = reader - .as_mut() - .ok_or_else(|| LimitHit::corrupt("tar: already consumed".into()))?; - let m = &members[*next_index]; - budget.count_entry()?; - let cap = budget.reserve()?; - r.seek(std::io::SeekFrom::Start(m.data_offset)) - .map_err(|e| LimitHit::corrupt(format!("tar seek: {e}")))?; - let mut take = r.take(m.size); - let (out, truncated) = bounded_read(&mut take, cap) - .map_err(|e| LimitHit::corrupt(format!("tar read: {e}")))?; - if truncated { - return Err(LimitHit::new(format!( - "tar member '{}' exceeds budget", - m.name - ))); - } - budget.commit(out.len() as u64); - *next_index += 1; - Ok(Some(Entry::new(m.name.clone(), out))) - } - ArchiveInner::Lazy { - reader, - format, - state, - } => { - if state.cached.is_none() { - let r = reader - .take() - .ok_or_else(|| LimitHit::corrupt("lazy: already consumed".into()))?; - let mut data = Vec::new(); - r.take(budget.limits.max_buffer_bytes.saturating_add(1)) - .read_to_end(&mut data) - .map_err(|e| LimitHit::corrupt(format!("read: {e}")))?; - if data.len() as u64 > budget.limits.max_buffer_bytes { - return Err(LimitHit::new(format!( - "container exceeds max-buffer {}", - budget.limits.max_buffer_bytes - ))); - } - let mut entries = Vec::new(); - extract_each::( - *format, - &data, - budget, - &mut |e, _| { - entries.push(e); - None - }, - )?; - state.info = member_infos(&entries); - state.cached = Some(entries); - } - let entries = state.cached.as_ref().unwrap(); - if state.next_index < entries.len() { - let e = entries[state.next_index].clone(); - state.next_index += 1; - Ok(Some(e)) - } else { - Ok(None) - } - } - ArchiveInner::Buffered { data, state } => { - // Lazy full extraction on first call. - if state.cached.is_none() { - let fmt = self.format; - let mut entries = Vec::new(); - extract_each::(fmt, data, budget, &mut |e, _| { - entries.push(e); - None - })?; - state.info = member_infos(&entries); - state.cached = Some(entries); - } - let entries = state.cached.as_ref().unwrap(); - if state.next_index < entries.len() { - let e = entries[state.next_index].clone(); - state.next_index += 1; - Ok(Some(e)) - } else { - Ok(None) - } - } - } - } - - /// Extract a specific member by index under budget. For ZIP this seeks - /// directly to the member; for buffered formats the full extraction is - /// triggered on first call and the result is returned from the cache. - pub fn extract(&mut self, index: usize, budget: &mut Budget) -> Result { - match &mut self.inner { - #[cfg(feature = "zip")] - ArchiveInner::Zip { members, .. } => members - .extract_entry(index, budget)? - .ok_or_else(|| LimitHit::corrupt(format!("index {index} is a directory"))), - #[cfg(feature = "gzip")] - ArchiveInner::Gzip { reader, done } => { - if index != 0 { - return Err(LimitHit::new(format!("index {index} out of bounds"))); - } - if *done { - return Err(LimitHit::corrupt("gzip: already extracted".into())); - } - let r = reader - .take() - .ok_or_else(|| LimitHit::corrupt("gzip: already extracted".into()))?; - budget.count_entry()?; - let cap = budget.reserve()?; - let s = gunzip(std::io::BufReader::new(r), cap, budget) - .map_err(|e| LimitHit::corrupt(format!("gzip: {e}")))?; - if s.over_cap { - return Err(LimitHit::new("gzip member exceeds budget".to_string())); - } - budget.commit(s.data.len() as u64); - *done = true; - Ok(gzip_entry(s)) - } - #[cfg(feature = "tar")] - ArchiveInner::Tar { - reader, members, .. - } => { - if index >= members.len() { - return Err(LimitHit::new(format!("index {index} out of bounds"))); - } - let r = reader - .as_mut() - .ok_or_else(|| LimitHit::corrupt("tar: already consumed".into()))?; - let m = &members[index]; - budget.count_entry()?; - let cap = budget.reserve()?; - r.seek(std::io::SeekFrom::Start(m.data_offset)) - .map_err(|e| LimitHit::corrupt(format!("tar seek: {e}")))?; - let mut take = r.take(m.size); - let (out, truncated) = bounded_read(&mut take, cap) - .map_err(|e| LimitHit::corrupt(format!("tar read: {e}")))?; - if truncated { - return Err(LimitHit::new(format!( - "tar member '{}' exceeds budget", - m.name - ))); - } - budget.commit(out.len() as u64); - Ok(Entry::new(m.name.clone(), out)) - } - ArchiveInner::Lazy { - reader, - format, - state, - } => { - if state.cached.is_none() { - let r = reader - .take() - .ok_or_else(|| LimitHit::corrupt("lazy: already consumed".into()))?; - let mut data = Vec::new(); - r.take(budget.limits.max_buffer_bytes.saturating_add(1)) - .read_to_end(&mut data) - .map_err(|e| LimitHit::corrupt(format!("read: {e}")))?; - if data.len() as u64 > budget.limits.max_buffer_bytes { - return Err(LimitHit::new(format!( - "container exceeds max-buffer {}", - budget.limits.max_buffer_bytes - ))); - } - let mut entries = Vec::new(); - extract_each::( - *format, - &data, - budget, - &mut |e, _| { - entries.push(e); - None - }, - )?; - state.info = member_infos(&entries); - state.cached = Some(entries); - } - let entries = state.cached.as_ref().unwrap(); - entries - .get(index) - .cloned() - .ok_or_else(|| LimitHit::new(format!("index {index} out of bounds"))) - } - ArchiveInner::Buffered { data, state } => { - if state.cached.is_none() { - let fmt = self.format; - let mut entries = Vec::new(); - extract_each::(fmt, data, budget, &mut |e, _| { - entries.push(e); - None - })?; - state.info = member_infos(&entries); - state.cached = Some(entries); - } - let entries = state.cached.as_ref().unwrap(); - entries - .get(index) - .cloned() - .ok_or_else(|| LimitHit::new(format!("index {index} out of bounds"))) - } - } - } - - /// Extract all members under budget and return them as a `Vec`. - pub fn extract_all(&mut self, budget: &mut Budget) -> Result, LimitHit> { - let mut out = Vec::new(); - while let Some(e) = self.extract_next(budget)? { - out.push(e); - } - Ok(out) - } -} - #[cfg(all(test, feature = "base64scan"))] mod markup_payload_tests { use super::*; @@ -2681,11 +2472,73 @@ mod markup_payload_tests { base64::engine::general_purpose::STANDARD.encode(d) } + /// A PE big enough for the executable scan's shortest run. + fn pe(marker: &[u8]) -> Vec { + let mut v = vec![0u8; 1200]; + v[..2].copy_from_slice(b"MZ"); + v[0x3c..0x40].copy_from_slice(&0x80u32.to_le_bytes()); + v[0x80..0x84].copy_from_slice(b"PE\0\0"); + v[0x100..0x100 + marker.len()].copy_from_slice(marker); + v + } + + fn wrapped(s: &str, width: usize) -> String { + s.as_bytes() + .chunks(width) + .map(|l| std::str::from_utf8(l).unwrap()) + .collect::>() + .join("\r\n ") + } + + #[test] + fn payloads_read_in_blocks_come_back_as_from_memory() { + use crate::source::{BlockCache, CHUNK}; + let mut doc = String::new(); + // Each payload placed across a chunk seam, some line-wrapped, some a + // near miss. + let payloads: Vec = vec![ + format!("", b64(&png(b"one"))), + format!("{}", b64(&png(b"two!"))), + format!("$b = \"{}\";", wrapped(&b64(&pe(b"three")), 76)), + format!("x={}=", b64(&pe(b"four"))), + // Not introduced, not an asset, too short. + format!("plain {} text", b64(&png(b"five"))), + format!("

{}

", b64(b"just some words, not an image at all, long enough to pass the run length check")), + format!("{}", b64(&png(b"")[..40])), + ]; + for (k, p) in payloads.iter().enumerate() { + let seam = (k + 1) * CHUNK; + let pad = seam.saturating_sub(doc.len() + p.len() / 2); + doc.push_str(&".".repeat(pad)); + doc.push_str(p); + } + let data = doc.as_bytes(); + let cache = + BlockCache::with_sizes(std::io::Cursor::new(data.to_vec()), 509, 8 * 509).unwrap(); + for cap in [1 << 20, 1199, 1200, 1201, 260, 64] { + let want = markup_embedded_payloads(&data, cap); + assert_eq!( + markup_embedded_payloads(&cache, cap), + want, + "markup, cap {cap}" + ); + let want_b64 = base64_payloads(&data, cap); + assert_eq!(base64_payloads(&cache, cap), want_b64, "base64, cap {cap}"); + if cap == 1 << 20 { + assert_eq!(want.len(), 2, "both introduced images"); + assert_eq!(want_b64.len(), 2, "both executables"); + } + } + // The cap is applied to what a run decodes to, before it is read. + assert_eq!(base64_payloads(&data, 1199).len(), 0); + assert_eq!(base64_payloads(&data, 1200).len(), 2); + } + #[test] fn a_data_uri_image_comes_back_byte_for_byte() { let want = png(b"marker-A"); let page = format!("", b64(&want)); - let got = markup_embedded_payloads(page.as_bytes(), 1 << 20); + let got = markup_embedded_payloads(&page.as_bytes(), 1 << 20); assert_eq!(got, vec![want], "padding must not be dropped from the tail"); } @@ -2698,7 +2551,7 @@ mod markup_payload_tests { b64(&want) ); assert_eq!( - markup_embedded_payloads(doc.as_bytes(), 1 << 20), + markup_embedded_payloads(&doc.as_bytes(), 1 << 20), vec![want] ); } @@ -2708,11 +2561,11 @@ mod markup_payload_tests { let asset = b64(&png(b"marker-C")); // Same bytes, but not introduced by a `data:` URI or an element body. let loose = format!("some text {asset} more text"); - assert!(markup_embedded_payloads(loose.as_bytes(), 1 << 20).is_empty()); + assert!(markup_embedded_payloads(&loose.as_bytes(), 1 << 20).is_empty()); // Introduced, but decodes to nothing that is an asset. let junk = "A".repeat(400); let page = format!(""); - assert!(markup_embedded_payloads(page.as_bytes(), 1 << 20).is_empty()); + assert!(markup_embedded_payloads(&page.as_bytes(), 1 << 20).is_empty()); } #[test] @@ -2720,8 +2573,8 @@ mod markup_payload_tests { let want = png(b"marker-D"); let page = format!("", b64(&want)); assert!( - markup_embedded_payloads(page.as_bytes(), 16).is_empty(), - "a truncated asset is worse than no asset — it would be scanned as \ + markup_embedded_payloads(&page.as_bytes(), 16).is_empty(), + "a truncated asset is worse than no asset: it would be scanned as \ if complete" ); } @@ -2810,7 +2663,7 @@ mod tests { /// Pins the floor at SHIPPED settings. Every other bomb test overrides /// `max_compression_ratio`, so before this one nothing exercised the guard - /// as deployed — the floor could have been any value, or ineffective, and + /// as deployed: the floor could have been any value, or ineffective, and /// the suite would have stayed green. #[test] fn ratio_guard_boundary_at_default_limits() { @@ -2849,9 +2702,9 @@ mod tests { } fn cab_of(members: &[(&str, &[u8])]) -> Vec { - let mut b = cab::CabinetBuilder::new(); + let mut b = ::cab::CabinetBuilder::new(); { - let folder = b.add_folder(cab::CompressionType::None); + let folder = b.add_folder(::cab::CompressionType::None); for (name, _) in members { folder.add_file(*name); } @@ -2867,8 +2720,8 @@ mod tests { #[test] fn cab_with_corrupted_total_size_is_recovered() { // A well-formed cabinet whose CFHEADER `cbCabinet` (bytes 8..12) is - // overwritten with 0xFFFFFFFF — an evasion that defeats strict parsers. - // `repair_cab_size` clamps it so the member is still extracted. + // overwritten with 0xFFFFFFFF, an evasion that defeats strict parsers. + // The field is not trusted, so the member is still extracted. let mut blob = cab_of(&[("payload.bin", b"INNER-CAB-PAYLOAD")]); blob[8..12].copy_from_slice(&u32::MAX.to_le_bytes()); let mut budget = Budget::new(Limits::default()); @@ -2878,22 +2731,22 @@ mod tests { } #[test] - fn extract_each_stops_early_and_threads_budget() { + fn walk_stops_early_and_threads_budget() { // A tar with three members; the visitor stops at the second. The third // must never be decoded/visited (early-exit), and the visitor sees the // shared budget so it can recurse. let blob = tar_of(&[("a", b"first"), ("b", b"second"), ("c", b"third")]); let mut budget = Budget::new(Limits::default()); let mut seen: Vec = Vec::new(); - let stopped = extract_each(Format::Tar, &blob, &mut budget, &mut |e, b| { - // The budget is real (a member was just committed against it). - assert!(b.total_out > 0); - seen.push(String::from_utf8_lossy(&e.data).into_owned()); - if e.data == b"second" { - Some("hit-b") - } else { - None + let stopped = walk(Format::Tar, &blob, &mut budget, &mut |_, content, b| { + // The budget is real (the member was just counted against it). + assert!(b.files > 0); + let mut data = Vec::new(); + if let Some(Member::Stream(r)) = content { + r.read_to_end(&mut data).unwrap(); } + seen.push(String::from_utf8_lossy(&data).into_owned()); + (data == b"second").then_some("hit-b") }) .unwrap(); assert_eq!(stopped, Some("hit-b")); @@ -2924,7 +2777,7 @@ mod tests { fn gzip_multi_member_concatenated() { // A gzip file can be several concatenated members; the payload may live // in a later one. The extractor must decode ALL members (MultiGzDecoder), - // not just the first — a real FN source for some packagers. + // not just the first: a real FN source for some packagers. let mut blob = gz(b"header-member-only"); blob.extend_from_slice(&gz(b"PAYLOAD-with-eicar-marker-X5O!")); let mut budget = Budget::new(Limits::default()); @@ -2991,7 +2844,7 @@ mod tests { 255, 109, 184, ]; let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Bzip2, blob, &mut budget).unwrap(); + let entries = extract(Format::Bzip2, &blob, &mut budget).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].data, b"hello exav inside bzip2"); } @@ -3002,9 +2855,9 @@ mod tests { let real: &[u8] = &[ 66, 90, 104, 57, 49, 65, 89, 38, 83, 89, 213, 127, 182, 220, 0, 0, ]; - assert_eq!(detect(real), Some(Format::Bzip2)); + assert_eq!(detect(&real), Some(Format::Bzip2)); // ...but a coincidental `BZh#` run in binary data (as carved out of an ISO - // at offset 343598: `BZh3` then `1h1H…`, not the block magic) is not — it + // at offset 343598: `BZh3` then `1h1H…`, not the block magic) is not. It // must not be treated as bzip2 and reported UNSCANNABLE. assert_eq!(detect(b"BZh31h1HDataHere....."), None); assert_eq!(detect(b"BZh4kUPHgo........."), None); @@ -3023,6 +2876,21 @@ mod tests { assert_eq!(detect(b"MSCF"), None); } + /// `60 EA` is two bytes, so about one object in 65,536 starts with it by + /// chance, and reading such an object as a damaged ARJ reports it + /// `UNSCANNABLE`. The main header carries a CRC-32 that settles it. + #[cfg(feature = "arj")] + #[test] + fn arj_magic_rejects_false_positives() { + let real: &[u8] = include_bytes!("../tests/fixtures/sample.arj"); + assert_eq!(detect(&real), Some(Format::Arj)); + let mut bad = real.to_vec(); + bad[10] ^= 0xff; // inside the main header + assert_eq!(detect(&bad.as_slice()), None); + assert_eq!(detect(b"\x60\xea\x10\x00random bytes after it"), None); + assert_eq!(detect(b"\x60\xea"), None); + } + #[cfg(feature = "base64scan")] #[test] fn base64_payloads_extracts_embedded_executable() { @@ -3037,14 +2905,14 @@ mod tests { let b64 = base64::engine::general_purpose::STANDARD.encode(&pe); // Embedded as a PowerShell-style string assignment inside script text. let carrier = format!("$PEBytes = \"{b64}\"\nInvoke-Something $PEBytes\n"); - let got = base64_payloads(carrier.as_bytes(), u64::MAX); + let got = base64_payloads(&carrier.as_bytes(), u64::MAX); assert_eq!(got.len(), 1, "should recover the one embedded PE"); assert!(got[0].starts_with(b"MZ")); assert_eq!(&got[0][0x40..0x44], b"PE\x00\x00"); // A long base64 run that decodes to plain text (no exec magic) is ignored. let txt = base64::engine::general_purpose::STANDARD.encode(vec![b'A'; 2048]); - assert!(base64_payloads(format!("x=\"{txt}\"").as_bytes(), u64::MAX).is_empty()); + assert!(base64_payloads(&format!("x=\"{txt}\"").as_bytes(), u64::MAX).is_empty()); // Too-short a run is never trial-decoded. assert!(base64_payloads(b"var x = \"aGVsbG8gd29ybGQ=\"", u64::MAX).is_empty()); } @@ -3164,8 +3032,8 @@ mod tests { #[test] fn cab_roundtrip() { // Build a small uncompressed cabinet, then extract it. - let mut builder = cab::CabinetBuilder::new(); - let folder = builder.add_folder(cab::CompressionType::None); + let mut builder = ::cab::CabinetBuilder::new(); + let folder = builder.add_folder(::cab::CompressionType::None); folder.add_file("payload.txt"); let mut blob = Vec::new(); let mut writer = builder.build(Cursor::new(&mut blob)).unwrap(); @@ -3260,7 +3128,7 @@ mod tests { Content-Transfer-Encoding: base64\r\n\r\n{b64}\r\n--BB--\r\n" ); let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Email, msg.as_bytes(), &mut budget).unwrap(); + let entries = extract(Format::Email, &msg.as_bytes(), &mut budget).unwrap(); assert!(has_eicar(&entries), "EICAR not found in email parts"); } @@ -3362,273 +3230,21 @@ mod tests { assert_eq!(entries[0].data, payload); } - // ----------------------------------------------------------------------- - // Archive tests - // ----------------------------------------------------------------------- - - #[test] - fn archive_zip_extract_next() { - let zip_blob = zip_of(&[("a.txt", b"alpha"), ("b.txt", b"beta")]); - let mut archive = Archive::open(Cursor::new(zip_blob)).unwrap(); - assert_eq!(archive.format(), Format::Zip); - - let info = archive.list(); - assert_eq!(info.len(), 2); - assert_eq!(info[0].name, "a.txt"); - assert_eq!(info[1].name, "b.txt"); - - let mut budget = Budget::new(Limits::default()); - let e1 = archive.extract_next(&mut budget).unwrap().unwrap(); - assert_eq!(e1.name, "a.txt"); - assert_eq!(e1.data, b"alpha"); - - let e2 = archive.extract_next(&mut budget).unwrap().unwrap(); - assert_eq!(e2.name, "b.txt"); - assert_eq!(e2.data, b"beta"); - - assert!(archive.extract_next(&mut budget).unwrap().is_none()); - } - - #[test] - fn archive_tar_extract_next() { - let blob = tar_of(&[("x", b"one"), ("y", b"two"), ("z", b"three")]); - let mut archive = Archive::open(Cursor::new(blob)).unwrap(); - assert_eq!(archive.format(), Format::Tar); - - let mut budget = Budget::new(Limits::default()); - let mut names = Vec::new(); - while let Some(e) = archive.extract_next(&mut budget).unwrap() { - names.push(e.name); - } - assert_eq!(names, vec!["x", "y", "z"]); - } - - // Helper: build a GNU ar archive in memory (short `/`-terminated names). - fn ar_of(members: &[(&str, &[u8])]) -> Vec { - let mut out = b"!\n".to_vec(); - for (name, data) in members { - let mut hdr = format!("{name}/"); - while hdr.len() < 16 { - hdr.push(' '); - } - out.extend_from_slice(hdr.as_bytes()); - out.extend_from_slice( - format!( - "{:<12}{:<6}{:<6}{:<8}{:<10}`\n", - 0, - 0, - 0, - 0o100644, - data.len() - ) - .as_bytes(), - ); - out.extend_from_slice(data); - if data.len() % 2 == 1 { - out.push(b'\n'); - } - } - out - } - - /// A buffered format has no index, so `list` is empty on `open` — but the - /// first extraction call fills the whole cache, and from then on `list` - /// reports every member. An empty list after extraction would tell a caller - /// the archive holds nothing, which is the opposite of what just happened. - #[test] - fn archive_buffered_lists_members_after_first_extraction() { - let blob = ar_of(&[("a.txt", b"alpha"), ("b.txt", b"beta")]); - assert_eq!(detect(&blob), Some(Format::Ar)); - let mut archive = Archive::open(Cursor::new(blob)).unwrap(); - assert!(archive.list().is_empty(), "no index before extraction"); - - let mut budget = Budget::new(Limits::default()); - let e1 = archive.extract_next(&mut budget).unwrap().unwrap(); - assert_eq!(e1.name, "a.txt"); - - let names: Vec<&str> = archive.list().iter().map(|m| m.name.as_str()).collect(); - assert_eq!( - names, - vec!["a.txt", "b.txt"], - "cache fills completely or not at all" - ); - assert_eq!(archive.list()[1].uncompressed_size, 4); - assert_eq!(archive.list()[1].index, 1); - } - - /// A tar's headers ARE its index, so `list` must report them. - /// - /// Returning an empty slice here tells a caller the archive holds NOTHING, - /// which is a different claim from "this format has no directory" and the - /// wrong one — the names, sizes and offsets are all parsed at `open`, and - /// `extract` already seeks straight to a member by them. - #[test] - fn archive_tar_lists_its_members_without_extracting_them() { - let blob = tar_of(&[("x", b"one"), ("y", b"two"), ("z", b"three")]); - let archive = Archive::open(Cursor::new(blob)).unwrap(); - - let names: Vec<&str> = archive.list().iter().map(|m| m.name.as_str()).collect(); - assert_eq!(names, vec!["x", "y", "z"]); - assert_eq!(archive.list()[2].uncompressed_size, 5); - // Stored uncompressed, so the two sizes are the same size rather than - // one of them being left at zero. - assert_eq!(archive.list()[2].compressed_size, 5); - assert_eq!(archive.list()[1].index, 1); - } - - /// The two ways into a ZIP must find the same members. - /// /// A member with a local header and NO central-directory entry is the /// classic way to hide one: the directory is what most readers walk, and - /// the target extracts it anyway. `extract` scans for those; `Archive` walks - /// the directory through the `zip` crate and does not. Whichever door a - /// caller comes through, the archive holds what it holds. + /// the target extracts it anyway. #[test] - fn archive_finds_the_same_members_as_extract() { + fn a_member_hidden_from_the_central_directory_is_extracted() { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("tests/fixtures/zip/orphan_local.zip"); let blob = std::fs::read(&path).expect("fixture reads"); - let mut budget = Budget::new(Limits::default()); - let buffered: Vec = extract(Format::Zip, &blob, &mut budget) + let names: Vec = extract(Format::Zip, &blob, &mut budget) .unwrap() .into_iter() .map(|e| e.name) .collect(); - - let mut archive = Archive::open(Cursor::new(blob)).unwrap(); - let mut budget = Budget::new(Limits::default()); - let mut streamed = Vec::new(); - while let Some(e) = archive.extract_next(&mut budget).unwrap() { - streamed.push(e.name); - } - - assert_eq!(buffered, streamed, "the two paths disagree about members"); - } - - /// Listing must account for a member hidden from the central directory, and - /// must be able to do so without decompressing anything. - /// - /// A listing that reports only what the directory admits to is the polite - /// version of the wrong answer: it is exactly what the archive was built to - /// obtain. Finding the member and reading it are separate jobs, so the first - /// costs a header parse — and the index it reports must be the index - /// `extract` takes, or a caller acting on the listing reaches a different - /// member than the one it was told about. - #[test] - fn archive_lists_a_member_hidden_from_the_central_directory() { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/zip/orphan_local.zip"); - let blob = std::fs::read(&path).expect("fixture reads"); - - let mut archive = Archive::open(Cursor::new(blob)).unwrap(); - let listed: Vec<(usize, String)> = archive - .list() - .iter() - .map(|m| (m.index, m.name.clone())) - .collect(); - assert_eq!( - listed, - vec![(0, "benign.txt".to_string()), (1, "hidden.txt".to_string())], - ); - - // The index from the listing addresses the member the listing named. - let mut budget = Budget::new(Limits::default()); - let e = archive.extract(1, &mut budget).unwrap(); - assert_eq!(e.name, "hidden.txt"); - } - - #[test] - fn archive_extract_by_index() { - let zip_blob = zip_of(&[("first", b"111"), ("second", b"222"), ("third", b"333")]); - let mut archive = Archive::open(Cursor::new(zip_blob)).unwrap(); - let mut budget = Budget::new(Limits::default()); - - let e = archive.extract(1, &mut budget).unwrap(); - assert_eq!(e.name, "second"); - assert_eq!(e.data, b"222"); - } - - #[test] - fn archive_extract_all_matches_next() { - let zip_blob = zip_of(&[("a", b"aa"), ("b", b"bb")]); - let mut budget1 = Budget::new(Limits::default()); - let mut budget2 = Budget::new(Limits::default()); - - // extract_all via extract_next loop - let mut a1 = Archive::open(Cursor::new(zip_blob.clone())).unwrap(); - let mut all_next = Vec::new(); - while let Some(e) = a1.extract_next(&mut budget1).unwrap() { - all_next.push((e.name, e.data)); - } - - // extract_all - let mut a2 = Archive::open(Cursor::new(zip_blob)).unwrap(); - let all = a2.extract_all(&mut budget2).unwrap(); - let all_collected: Vec<_> = all.into_iter().map(|e| (e.name, e.data)).collect(); - - assert_eq!(all_next, all_collected); - } - - #[test] - fn archive_gzip_extract_next() { - let blob = gz(b"hello from gzip"); - let mut archive = Archive::open(Cursor::new(blob)).unwrap(); - assert_eq!(archive.format(), Format::Gzip); - - let mut budget = Budget::new(Limits::default()); - let e = archive.extract_next(&mut budget).unwrap().unwrap(); - assert_eq!(e.data, b"hello from gzip"); - assert!(archive.extract_next(&mut budget).unwrap().is_none()); - } - - #[test] - fn archive_bzip2_extract_next() { - // Same hardcoded bzip2 stream as bzip2_roundtrip. - let blob: &[u8] = &[ - 66, 90, 104, 57, 49, 65, 89, 38, 83, 89, 213, 127, 182, 220, 0, 0, 5, 25, 128, 64, 0, - 16, 0, 54, 101, 201, 80, 32, 0, 49, 76, 0, 19, 66, 154, 105, 163, 77, 168, 242, 145, - 94, 233, 129, 65, 248, 112, 129, 150, 100, 114, 190, 46, 228, 138, 112, 161, 33, 170, - 255, 109, 184, - ]; - let mut archive = Archive::open(Cursor::new(blob.to_vec())).unwrap(); - assert_eq!(archive.format(), Format::Bzip2); - - let mut budget = Budget::new(Limits::default()); - let e = archive.extract_next(&mut budget).unwrap().unwrap(); - assert_eq!(e.data, b"hello exav inside bzip2"); - assert!(archive.extract_next(&mut budget).unwrap().is_none()); - } - - #[test] - fn archive_xz_extract_next() { - let payload = b"hello exav inside xz"; - let blob = include_bytes!("../tests/fixtures/xz/simple.xz"); - - let mut archive = Archive::open(Cursor::new(blob.as_slice())).unwrap(); - assert_eq!(archive.format(), Format::Xz); - - let mut budget = Budget::new(Limits::default()); - let e = archive.extract_next(&mut budget).unwrap().unwrap(); - assert_eq!(e.data, payload); - assert!(archive.extract_next(&mut budget).unwrap().is_none()); - } - - // Helper: build a ZIP in memory (stored, no compression). - fn zip_of(members: &[(&str, &[u8])]) -> Vec { - use std::io::Write; - let mut buf = Cursor::new(Vec::new()); - { - let mut zip = ::zip::ZipWriter::new(&mut buf); - let opts = ::zip::write::SimpleFileOptions::default() - .compression_method(::zip::CompressionMethod::Stored); - for (name, data) in members { - zip.start_file(*name, opts).unwrap(); - zip.write_all(data).unwrap(); - } - zip.finish().unwrap(); - } - buf.into_inner() + assert_eq!(names, ["benign.txt", "hidden.txt"]); } fn zstd_of(data: &[u8]) -> Vec { @@ -3642,18 +3258,6 @@ mod tests { out } - #[test] - fn archive_zstd_extract_next() { - let blob = zstd_of(b"hello from zstd"); - let mut archive = Archive::open(Cursor::new(blob)).unwrap(); - assert_eq!(archive.format(), Format::Zstd); - - let mut budget = Budget::new(Limits::default()); - let e = archive.extract_next(&mut budget).unwrap().unwrap(); - assert_eq!(e.data, b"hello from zstd"); - assert!(archive.extract_next(&mut budget).unwrap().is_none()); - } - #[test] fn lzip_roundtrip() { let payload = b"hello exav inside lzip"; @@ -3669,7 +3273,7 @@ mod tests { /// An LZMA dictionary size clamped to what the caller's budget allows. /// /// The declared size comes straight out of the file and the decoder allocates it -/// UP FRONT, before decompressing a byte — so an attacker chooses how much +/// UP FRONT, before decompressing a byte, so an attacker chooses how much /// memory exav commits. Measured: a 766 KB NSIS installer declaring a 1.5 GB /// dictionary, which aborted the process under the daemon's per-job address-space /// limit. Under the daemon that abort closes the client connection with no reply, diff --git a/crates/exav-unpack/src/source.rs b/crates/exav-unpack/src/source.rs new file mode 100644 index 00000000..b53baa3d --- /dev/null +++ b/crates/exav-unpack/src/source.rs @@ -0,0 +1,719 @@ +//! Random access to the object being scanned. +//! +//! An object is either held in memory, where every consumer reads the slice +//! directly, or read from a seekable source through a [`BlockCache`]: fixed +//! blocks, the least recently used evicted past a byte budget. The cache bounds +//! the memory an object costs, not how far a consumer may look: any offset can +//! be read, and a block evicted is read again from the source when needed. +//! +//! Consumers are written once, generic over [`ByteSource`]. Instantiated for +//! `[u8]` they compile to the slice code they replaced; instantiated for a +//! `dyn ByteSource` they reach the source through the cache. + +use std::borrow::Cow; +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::io::{Read, Seek, SeekFrom}; + +/// Bytes a [`BlockCache`] reads at a time. The HTTP range reader fetches the +/// same size, so a block is one request. +pub const BLOCK_SIZE: usize = 64 * 1024; + +/// Bytes a [`BlockCache`] holds by default. +pub const CACHE_BYTES: usize = 8 * 1024 * 1024; + +/// Bytes handed to a chunk visitor at a time when the source is not in memory. +/// Small in tests, so a small object crosses many seams. +pub const CHUNK: usize = if cfg!(any(test, feature = "small-chunks")) { + 4096 +} else { + 1024 * 1024 +}; + +/// An object's bytes, read at any offset. +pub trait ByteSource { + fn len(&self) -> usize; + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Up to `len` bytes from `off`, fewer where the object ends. Borrowed + /// when the object is in memory. + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]>; + + /// The whole object, when it is held in memory. + fn as_slice(&self) -> Option<&[u8]> { + None + } + + /// Hand `f` consecutive, non-overlapping chunks covering `[from, to)` with + /// the offset each starts at, until it returns `false`. + fn chunks(&self, from: usize, to: usize, f: &mut dyn FnMut(usize, &[u8]) -> bool) { + let to = to.min(self.len()); + let mut at = from; + while at < to { + let w = self.window(at, (to - at).min(CHUNK)); + if w.is_empty() || !f(at, &w) { + return; + } + at += w.len(); + } + } + + /// Start of the first occurrence of `needle` lying wholly inside + /// `[from, to)`. + fn find(&self, needle: &[u8], from: usize, to: usize) -> Option { + let to = to.min(self.len()); + if needle.is_empty() { + return (from <= to).then_some(from); + } + if from >= to || to - from < needle.len() { + return None; + } + let finder = memchr::memmem::Finder::new(needle); + let mut at = from; + // Small first, so a search that ends close by copies little, then + // doubling. Each window runs `needle.len() - 1` bytes into the next, + // so an occurrence across the seam is seen whole. + let mut span = 4096; + while at + needle.len() <= to { + let w = self.window(at, (to - at).min(span + needle.len() - 1)); + if w.len() < needle.len() { + return None; + } + if let Some(p) = finder.find(&w) { + return Some(at + p); + } + at += w.len() - (needle.len() - 1); + span = (span * 2).min(CHUNK); + } + None + } + + /// The whole object, read into memory, when it is at most `limit` bytes. + fn materialize(&self, limit: usize) -> Option> { + if let Some(s) = self.as_slice() { + return Some(Cow::Borrowed(s)); + } + if self.len() > limit { + return None; + } + let w = self.window(0, self.len()); + (w.len() == self.len()).then_some(w) + } + + /// Why a read from the source failed, if one did. Everything read after it + /// came back short, so a scan that saw one did not see the whole object. + fn read_error(&self) -> Option { + None + } + + /// Identifies this object for as long as it lives, so a decision made + /// about it cannot be applied to another. + fn identity(&self) -> (usize, usize); +} + +impl ByteSource for [u8] { + fn len(&self) -> usize { + <[u8]>::len(self) + } + + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]> { + let start = off.min(<[u8]>::len(self)); + let end = start.saturating_add(len).min(<[u8]>::len(self)); + Cow::Borrowed(&self[start..end]) + } + + fn as_slice(&self) -> Option<&[u8]> { + Some(self) + } + + fn chunks(&self, from: usize, to: usize, f: &mut dyn FnMut(usize, &[u8]) -> bool) { + let to = to.min(<[u8]>::len(self)); + if from < to { + f(from, &self[from..to]); + } + } + + fn find(&self, needle: &[u8], from: usize, to: usize) -> Option { + let to = to.min(<[u8]>::len(self)); + if from > to { + return None; + } + memchr::memmem::find(&self[from..to], needle).map(|p| from + p) + } + + fn identity(&self) -> (usize, usize) { + (self.as_ptr() as usize, <[u8]>::len(self)) + } +} + +impl ByteSource for Vec { + fn len(&self) -> usize { + self.as_slice().len() + } + + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]> { + ByteSource::window(self.as_slice(), off, len) + } + + fn as_slice(&self) -> Option<&[u8]> { + Some(self) + } + + fn chunks(&self, from: usize, to: usize, f: &mut dyn FnMut(usize, &[u8]) -> bool) { + ByteSource::chunks(self.as_slice(), from, to, f) + } + + fn find(&self, needle: &[u8], from: usize, to: usize) -> Option { + ByteSource::find(self.as_slice(), needle, from, to) + } + + fn identity(&self) -> (usize, usize) { + ByteSource::identity(self.as_slice()) + } +} + +impl ByteSource for [u8; N] { + fn len(&self) -> usize { + N + } + + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]> { + ByteSource::window(&self[..], off, len) + } + + fn as_slice(&self) -> Option<&[u8]> { + Some(self) + } + + fn chunks(&self, from: usize, to: usize, f: &mut dyn FnMut(usize, &[u8]) -> bool) { + ByteSource::chunks(&self[..], from, to, f) + } + + fn find(&self, needle: &[u8], from: usize, to: usize) -> Option { + ByteSource::find(&self[..], needle, from, to) + } + + fn identity(&self) -> (usize, usize) { + ByteSource::identity(&self[..]) + } +} + +/// A slice by reference, so bytes held in memory pass as a `&dyn ByteSource` +/// (`&[u8]` itself is unsized and cannot). +impl ByteSource for &[u8] { + fn len(&self) -> usize { + <[u8]>::len(self) + } + + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]> { + ByteSource::window(*self, off, len) + } + + fn as_slice(&self) -> Option<&[u8]> { + Some(self) + } + + fn chunks(&self, from: usize, to: usize, f: &mut dyn FnMut(usize, &[u8]) -> bool) { + ByteSource::chunks(*self, from, to, f) + } + + fn find(&self, needle: &[u8], from: usize, to: usize) -> Option { + ByteSource::find(*self, needle, from, to) + } + + fn identity(&self) -> (usize, usize) { + ByteSource::identity(*self) + } +} + +/// Blocks held, by index, with the tick each was last used at. +type Blocks = HashMap, u64)>; + +/// A seekable source read through fixed blocks, of which at most +/// `capacity` bytes are held. +pub struct BlockCache { + src: RefCell, + len: usize, + block: usize, + blocks: RefCell, + max_blocks: usize, + tick: Cell, + error: RefCell>, +} + +impl BlockCache { + /// A cache over `src`, whose length is taken now. + pub fn new(src: R) -> std::io::Result { + Self::with_sizes(src, BLOCK_SIZE, CACHE_BYTES) + } + + pub fn with_sizes(mut src: R, block: usize, capacity: usize) -> std::io::Result { + let len = src.seek(SeekFrom::End(0))?; + let block = block.max(1); + Ok(BlockCache { + src: RefCell::new(src), + len: usize::try_from(len).unwrap_or(usize::MAX), + block, + blocks: RefCell::new(HashMap::new()), + max_blocks: (capacity / block).max(2), + tick: Cell::new(0), + error: RefCell::new(None), + }) + } + + /// Copy block `idx`'s bytes from `skip` into `out`, up to `want` bytes. + fn copy_block(&self, idx: usize, skip: usize, want: usize, out: &mut Vec) -> bool { + let tick = self.tick.get() + 1; + self.tick.set(tick); + let mut blocks = self.blocks.borrow_mut(); + if let Some((data, used)) = blocks.get_mut(&idx) { + *used = tick; + let end = (skip + want).min(data.len()); + if skip < end { + out.extend_from_slice(&data[skip..end]); + } + return end == skip + want; + } + if self.error.borrow().is_some() { + return false; + } + let start = idx * self.block; + let size = self.block.min(self.len.saturating_sub(start)); + let mut data = vec![0u8; size]; + let got = { + let mut src = self.src.borrow_mut(); + match src.seek(SeekFrom::Start(start as u64)) { + Ok(_) => read_full(&mut *src, &mut data), + Err(e) => Err(e), + } + }; + match got { + Ok(n) if n == size => {} + Ok(n) => { + *self.error.borrow_mut() = Some(format!( + "the source ended at {} of its {} bytes", + start + n, + self.len + )); + data.truncate(n); + } + Err(e) => { + *self.error.borrow_mut() = Some(format!("read at {start} failed: {e}")); + return false; + } + } + let end = (skip + want).min(data.len()); + if skip < end { + out.extend_from_slice(&data[skip..end]); + } + let whole = end == skip + want; + if blocks.len() >= self.max_blocks { + if let Some(&old) = blocks + .iter() + .min_by_key(|(_, (_, used))| *used) + .map(|(k, _)| k) + { + blocks.remove(&old); + } + } + blocks.insert(idx, (data.into_boxed_slice(), tick)); + whole + } +} + +/// Read until `buf` is full or the source ends. +fn read_full(src: &mut R, buf: &mut [u8]) -> std::io::Result { + let mut n = 0; + while n < buf.len() { + match src.read(&mut buf[n..]) { + Ok(0) => break, + Ok(k) => n += k, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(n) +} + +impl ByteSource for BlockCache { + fn len(&self) -> usize { + self.len + } + + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]> { + let end = off.saturating_add(len).min(self.len); + if off >= end { + return Cow::Owned(Vec::new()); + } + let mut out = Vec::with_capacity(end - off); + let mut at = off; + while at < end { + let idx = at / self.block; + let skip = at - idx * self.block; + let want = (self.block - skip).min(end - at); + if !self.copy_block(idx, skip, want, &mut out) { + break; + } + at += want; + } + Cow::Owned(out) + } + + fn read_error(&self) -> Option { + self.error.borrow().clone() + } + + fn identity(&self) -> (usize, usize) { + (self as *const Self as usize, self.len) + } +} + +/// `[base, base + len)` of another source, as an object of its own: a region +/// carved out of its carrier. +pub struct Sub<'a> { + inner: &'a dyn ByteSource, + base: usize, + len: usize, +} + +impl<'a> Sub<'a> { + pub fn new(inner: &'a dyn ByteSource, base: usize, len: usize) -> Self { + let base = base.min(inner.len()); + Sub { + inner, + base, + len: len.min(inner.len() - base), + } + } +} + +impl ByteSource for Sub<'_> { + fn len(&self) -> usize { + self.len + } + + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]> { + let off = off.min(self.len); + let len = len.min(self.len - off); + self.inner.window(self.base + off, len) + } + + fn as_slice(&self) -> Option<&[u8]> { + self.inner + .as_slice() + .map(|s| &s[self.base..self.base + self.len]) + } + + fn find(&self, needle: &[u8], from: usize, to: usize) -> Option { + let to = to.min(self.len); + if from > to { + return None; + } + self.inner + .find(needle, self.base + from, self.base + to) + .map(|p| p - self.base) + } + + fn read_error(&self) -> Option { + self.inner.read_error() + } + + fn identity(&self) -> (usize, usize) { + let (id, _) = self.inner.identity(); + (id.wrapping_add(self.base), self.len) + } +} + +/// Another source with ASCII letters lowercased: the haystack the +/// case-insensitive matchers search. +pub struct Lower<'a, B: ?Sized>(pub &'a B); + +impl ByteSource for Lower<'_, B> { + fn len(&self) -> usize { + self.0.len() + } + + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]> { + let mut w = self.0.window(off, len).into_owned(); + w.make_ascii_lowercase(); + Cow::Owned(w) + } + + fn identity(&self) -> (usize, usize) { + self.0.identity() + } +} + +/// A `Read + Seek` cursor over a source, for the extractors that take one. +pub struct Reader<'a> { + src: &'a dyn ByteSource, + pos: u64, + /// Where in `src` the reader's offset 0 is, and how many bytes it covers. + base: u64, + len: u64, +} + +impl<'a> Reader<'a> { + pub fn new(src: &'a dyn ByteSource) -> Self { + Self::range(src, 0, src.len()) + } + + /// A reader over `[from, to)` of `src` only, its offsets counted from `from`. + pub fn range(src: &'a dyn ByteSource, from: usize, to: usize) -> Self { + let to = to.min(src.len()); + let from = from.min(to); + Reader { + src, + pos: 0, + base: from as u64, + len: (to - from) as u64, + } + } +} + +impl Read for Reader<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let len = self.len; + if self.pos >= len { + return Ok(0); + } + let want = buf.len().min((len - self.pos) as usize).min(CHUNK); + let w = self.src.window((self.base + self.pos) as usize, want); + if w.is_empty() { + return Err(std::io::Error::other( + self.src + .read_error() + .unwrap_or_else(|| "the source ended early".to_string()), + )); + } + buf[..w.len()].copy_from_slice(&w); + self.pos += w.len() as u64; + Ok(w.len()) + } +} + +impl Seek for Reader<'_> { + fn seek(&mut self, to: SeekFrom) -> std::io::Result { + let len = self.len as i128; + let pos = match to { + SeekFrom::Start(p) => p as i128, + SeekFrom::End(d) => len + d as i128, + SeekFrom::Current(d) => self.pos as i128 + d as i128, + }; + if pos < 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "seek before the start", + )); + } + self.pos = pos as u64; + Ok(self.pos) + } +} + +/// An object's bytes by index, for scans written as index loops: a slice +/// ([`Indexed`]), or a source read through a sliding window ([`Stepper`]). +pub trait Bytes { + fn len(&self) -> usize; + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The byte at `i`, which is below `len()`. 0 where the source could not + /// be read. + fn at(&mut self, i: usize) -> u8; + + /// `[from, to)`, clamped to the object. + fn range(&mut self, from: usize, to: usize) -> Cow<'_, [u8]>; +} + +pub struct Indexed<'a>(pub &'a [u8]); + +impl Bytes for Indexed<'_> { + fn len(&self) -> usize { + self.0.len() + } + + #[inline] + fn at(&mut self, i: usize) -> u8 { + self.0[i] + } + + fn range(&mut self, from: usize, to: usize) -> Cow<'_, [u8]> { + let to = to.min(self.0.len()); + Cow::Borrowed(&self.0[from.min(to)..to]) + } +} + +/// A source read a chunk at a time, keeping [`Stepper::BEHIND`] bytes before +/// the chunk's start so a short look back needs no read. +pub struct Stepper<'a> { + src: &'a dyn ByteSource, + buf: Cow<'a, [u8]>, + base: usize, +} + +impl<'a> Stepper<'a> { + pub const BEHIND: usize = 128; + + pub fn new(src: &'a dyn ByteSource) -> Self { + Stepper { + src, + buf: Cow::Borrowed(&[]), + base: 0, + } + } +} + +impl Bytes for Stepper<'_> { + fn len(&self) -> usize { + self.src.len() + } + + #[inline] + fn at(&mut self, i: usize) -> u8 { + if i < self.base || i - self.base >= self.buf.len() { + self.base = i.saturating_sub(Self::BEHIND); + self.buf = self.src.window(self.base, CHUNK + Self::BEHIND); + } + self.buf.get(i - self.base).copied().unwrap_or(0) + } + + fn range(&mut self, from: usize, to: usize) -> Cow<'_, [u8]> { + let to = to.min(self.src.len()); + let from = from.min(to); + if from >= self.base && to - self.base <= self.buf.len() { + return Cow::Borrowed(&self.buf[from - self.base..to - self.base]); + } + self.src.window(from, to - from) + } +} + +/// Every byte in `[from, to)`, in order, read a chunk at a time. +pub fn bytes<'a, B: ByteSource + ?Sized>( + src: &'a B, + from: usize, + to: usize, +) -> impl Iterator + 'a { + let to = to.min(src.len()); + let mut at = from; + let mut buf: Cow<'a, [u8]> = Cow::Owned(Vec::new()); + let mut i = 0; + std::iter::from_fn(move || { + if i == buf.len() { + if at >= to { + return None; + } + buf = src.window(at, (to - at).min(CHUNK)); + if buf.is_empty() { + return None; + } + at += buf.len(); + i = 0; + } + i += 1; + Some(buf[i - 1]) + }) +} + +/// A source that claims `claimed` bytes and delivers only `data`, for tests of +/// what a failed read does. +#[cfg(test)] +pub(crate) fn short_source(data: &[u8], claimed: u64) -> BlockCache { + struct Short(std::io::Cursor>, u64); + impl Read for Short { + fn read(&mut self, b: &mut [u8]) -> std::io::Result { + self.0.read(b) + } + } + impl Seek for Short { + fn seek(&mut self, p: SeekFrom) -> std::io::Result { + match p { + SeekFrom::End(_) => Ok(self.1), + p => self.0.seek(p), + } + } + } + BlockCache::with_sizes(Short(std::io::Cursor::new(data.to_vec()), claimed), 64, 256).unwrap() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn data(n: usize) -> Vec { + (0..n).map(|i| (i * 7 % 251) as u8).collect() + } + + #[test] + fn a_cached_source_reads_what_the_slice_holds() { + let d = data(10_000); + // Blocks small enough that reads cross them, a cache small enough to + // evict. + let c = BlockCache::with_sizes(Cursor::new(d.clone()), 64, 256).unwrap(); + assert_eq!(c.len(), d.len()); + for (off, len) in [ + (0, 10), + (60, 10), + (63, 2), + (500, 1000), + (9990, 50), + (10_000, 5), + (20_000, 5), + ] { + assert_eq!(c.window(off, len), d.window(off, len), "({off}, {len})"); + } + assert!(c.blocks.borrow().len() <= 4); + assert_eq!(c.read_error(), None); + } + + #[test] + fn find_agrees_with_the_slice_across_seams() { + let mut d = vec![b'x'; 3 * CHUNK]; + for at in [5, CHUNK - 2, 2 * CHUNK - 1, 3 * CHUNK - 4] { + d[at..at + 4].copy_from_slice(b"abcd"); + } + let c = BlockCache::with_sizes(Cursor::new(d.clone()), 4096, 64 * 1024).unwrap(); + let mut from = 0; + while let Some(p) = d.find(b"abcd", from, d.len()) { + assert_eq!(c.find(b"abcd", from, d.len()), Some(p)); + from = p + 1; + } + assert_eq!(c.find(b"abcd", from, d.len()), None); + // The range end is honoured. + assert_eq!(c.find(b"abcd", 0, 8), None); + assert_eq!(c.find(b"abcd", 0, 9), Some(5)); + } + + #[test] + fn a_short_source_is_reported() { + let c = short_source(&[1; 100], 1000); + assert_eq!(c.window(0, 1000).len(), 100); + assert!(c.read_error().is_some()); + } + + #[test] + fn views_and_readers_agree_with_the_slice() { + let d = data(5000); + let c = BlockCache::with_sizes(Cursor::new(d.clone()), 64, 256).unwrap(); + let s = Sub::new(&c, 1000, 2000); + assert_eq!(s.window(10, 100), d.window(1010, 100)); + assert_eq!( + s.find(&d[1500..1504], 0, 2000), + d.find(&d[1500..1504], 1000, 3000).map(|p| p - 1000) + ); + let mut all = Vec::new(); + Reader::new(&c).read_to_end(&mut all).unwrap(); + assert_eq!(all, d); + assert_eq!(bytes(&c, 100, 4000).collect::>(), d[100..4000]); + let l = Lower(&c); + assert_eq!(l.window(0, 5000).into_owned(), d.to_ascii_lowercase()); + assert_eq!(c.materialize(4999), None); + assert_eq!(c.materialize(5000).as_deref(), Some(&d[..])); + } +} diff --git a/crates/exav-unpack/src/stream.rs b/crates/exav-unpack/src/stream.rs index 07726668..f4dcdb2a 100644 --- a/crates/exav-unpack/src/stream.rs +++ b/crates/exav-unpack/src/stream.rs @@ -1,16 +1,17 @@ -//! Reader-based extraction: the low-level streaming API. +//! The walk over a container's members: [`walk`]. //! -//! Unlike [`crate::extract_each`] — which decodes each member fully into an -//! [`crate::Entry`]'s `Vec` — this API hands the caller a **reader** per -//! member. The member is decoded on demand as the caller reads it, so a member -//! of any size (multi-gigabyte) is never materialized in RAM. Bomb defenses are -//! enforced *as bytes flow* through [`BudgetReader`], not by pre-buffering. +//! Every format goes through it. A format whose decoder can produce a member +//! as it is read hands the visitor a [`Member::Stream`], so a member of any +//! size is never held here; the rest decode each member whole and hand it +//! over as [`Member::Bytes`]. A format whose parser needs the whole container +//! gets it from the source, without a copy when the container is in memory, +//! and is refused as over the buffer limit when it is larger. Bomb defenses +//! are enforced as bytes flow, through [`BudgetReader`]. //! -//! The source is `Read + Seek` and nothing here allocates a member-sized buffer; -//! any buffering a caller needs (e.g. to run a slice-based structural matcher on -//! a small member) is the caller's decision, made in `exav-core` — deliberately -//! as high up the stack as possible. +//! Any buffering of a streamed member (to run the checks that need it whole) +//! is the caller's decision, made in `exav-core`. +use crate::source::ByteSource; use crate::{Budget, Format, LimitHit}; use std::io::{self, Read, Seek}; @@ -24,7 +25,7 @@ const BUDGET_OVERFLOW: &str = "exav:member-budget-overflow"; /// still has data at the cap, the next `read` fails with `BUDGET_OVERFLOW` /// rather than returning EOF — so an over-budget (bomb) member is reported, not /// silently cut short and treated as fully scanned. -pub struct BudgetReader<'a> { +pub(crate) struct BudgetReader<'a> { inner: &'a mut dyn Read, cap: u64, count: u64, @@ -44,7 +45,8 @@ impl<'a> BudgetReader<'a> { } /// Bytes actually delivered to the caller so far. - pub fn count(&self) -> u64 { + #[cfg(test)] + fn count(&self) -> u64 { self.count } } @@ -81,391 +83,366 @@ impl Read for BudgetReader<'_> { } } -/// Metadata for a member handed to a [`stream_members`] visitor. The member's -/// bytes are read separately via the reader argument (or absent for a member -/// whose content can't be produced — encrypted / undecodable). +/// Metadata for a member handed to a [`walk`] visitor. #[derive(Debug, Clone)] pub struct MemberMeta { pub name: String, /// Compressed size within the container (for `.cdb` matching), or the /// decompressed length when the extractor can't report it. pub comp_size: u64, + /// The decoded size the container declares for the member, when it + /// declares one: what an index lists without decoding anything. Declared, + /// not measured, so a hostile container can make it anything. + pub size: Option, pub encrypted: bool, /// `Some(reason)` when the member is recognised but its content can't be - /// decoded (unsupported method / encryption with no matching password). The - /// reader argument to the visitor is then `None`. + /// decoded, or not all of it (unsupported method, encryption with no + /// matching password, damage part way). What was decoded, if anything, + /// still comes with it. pub unsupported: Option<&'static str>, } -/// Visitor for [`stream_members`]. Receives each member's metadata and, when its -/// content is decodable, a budgeted reader over the (on-demand decoded) bytes. -/// Returning `Some(T)` stops the walk immediately — later members are never -/// touched. The `Budget` is threaded for recursion / counting. -pub type StreamVisit<'a, T> = - &'a mut dyn FnMut(&MemberMeta, Option<&mut dyn Read>, &mut Budget) -> Option; +/// A member's content. +pub enum Member<'a> { + /// Decoded in full. + Bytes(Vec), + /// Decoded as it is read, up to what the scan budget has left. Reading + /// past that fails with an error [`is_budget_overflow`] recognises. + Stream(&'a mut dyn Read), +} -/// True for the formats [`stream_members`] can walk with each member's decoded -/// bytes produced on demand — no decompressed member is ever materialized -/// whole. Some arms (bzip2, xz, 7z among them) do buffer the COMPRESSED -/// container first, bounded by the peak-buffer limit; the win is on the output -/// side, where one stream can decompress to orders of magnitude more than the -/// file occupies. A decoding step that inherently needs a whole object — an -/// encrypted ZIP member's decryption needs the full ciphertext — is likewise -/// bounded by the peak-buffer limit. -pub fn is_streamable(fmt: Format) -> bool { - match fmt { - Format::Gzip | Format::Tar => true, - // Same shape as gzip: one stream whose output can be orders of - // magnitude larger than the file, so it must not be materialized. - #[cfg(feature = "bzip2")] - Format::Bzip2 => true, - #[cfg(feature = "xz")] - Format::Xz => true, - #[cfg(feature = "sevenz")] - Format::SevenZip => true, - #[cfg(feature = "cab")] - Format::Cab => true, - #[cfg(feature = "zip")] - Format::Zip => true, - #[cfg(feature = "zstd")] - Format::Zstd => true, - #[cfg(feature = "lzip")] - Format::Lzip => true, - #[cfg(feature = "lha")] - Format::Lha => true, - #[cfg(feature = "ar")] - Format::Ar => true, - #[cfg(feature = "cpio")] - Format::Cpio => true, - #[cfg(feature = "machofat")] - Format::Machofat => true, - #[cfg(feature = "pyc")] - Format::Pyc => true, - #[cfg(feature = "sfx")] - Format::Sfx => true, - #[cfg(feature = "tnef")] - Format::Tnef => true, - #[cfg(feature = "partition")] - Format::Partition => true, - #[cfg(feature = "iso")] - Format::Iso => true, - #[cfg(feature = "onenote")] - Format::OneNote => true, - #[cfg(feature = "swf")] - Format::Swf => true, - #[cfg(feature = "szdd")] - Format::Szdd => true, - _ => false, +impl Member<'_> { + /// The member's bytes, a streamed one read whole under the bound a member + /// decoded whole has ([`Budget::reserve`]). The flag is set when decoding + /// failed part way; the bytes before the failure are kept. + pub fn into_bytes( + self, + meta: &MemberMeta, + budget: &mut Budget, + ) -> Result<(Vec, bool), LimitHit> { + let reader = match self { + Member::Bytes(data) => return Ok((data, false)), + Member::Stream(reader) => reader, + }; + let name = &meta.name; + let cap = budget.reserve()?; + let mut data = Vec::new(); + let partial = match reader.take(cap.saturating_add(1)).read_to_end(&mut data) { + Ok(_) => false, + Err(e) if is_budget_overflow(&e) => { + return Err(LimitHit::new(format!("member '{name}': {e}"))); + } + Err(e) if budget.should_verify_checksums() => { + return Err(LimitHit::corrupt(format!("member '{name}': {e}"))); + } + Err(e) => crate::decode_error_hides_content(&e), + }; + if data.len() as u64 > cap { + return Err(LimitHit::new(format!("member '{name}' exceeds budget"))); + } + budget.commit(data.len() as u64); + Ok((data, partial)) } } -/// Walk a container's members, handing each to `visit` as a reader that decodes -/// on demand. The container is read straight from the seekable `source`; no -/// member-sized buffer is ever allocated here. Only [`is_streamable`] formats -/// are supported — the caller checks first and routes other formats to the -/// buffered [`crate::extract_each`] path. +/// Visitor for [`walk`]: each member's metadata and its content, `None` when +/// none could be produced. Returning `Some(T)` stops the walk; later members +/// are never touched. +pub type Visit<'a, T> = + &'a mut dyn FnMut(&MemberMeta, Option>, &mut Budget) -> Option; + +/// Walk the members of `src`, a container of format `fmt`, handing each to +/// `visit`. `Ok(Some(t))` when the visitor stopped the walk, `Err` when a +/// budget ran out or the container could not be read. /// -/// Bomb defenses: each member is counted (`count_entry`) and its decoded bytes -/// are capped by [`BudgetReader`] at the remaining scan budget; exceeding it ends -/// the walk with a [`LimitHit`]. +/// Every member is counted toward [`crate::Limits::max_members`], and its +/// decoded bytes are charged to the scan budget as they are produced. /// -/// Panic containment: some decoders (e.g. `delharc` for LHA) can panic on crafted -/// input. As in [`crate::extract_each`], the whole dispatch runs inside a -/// `catch_unwind` so a decoder panic becomes a clean `LimitHit` (→ `Unscannable`) -/// rather than aborting the process. This crate is `#![forbid(unsafe_code)]`, so -/// a panic is the only failure mode to contain. -pub fn stream_members( +/// Panic containment: some third-party decoders panic on crafted input instead +/// of returning an error. The whole dispatch runs inside `catch_unwind`, so a +/// panic becomes undecodable content rather than a crash. This crate is +/// `#![forbid(unsafe_code)]`, so there is no undefined behaviour to leak +/// across the boundary. What it does not contain: an allocation large enough +/// to abort, stack exhaustion, and a loop that neither allocates nor returns. +/// The daemon bounds those with process limits; a one-shot run and a library +/// embedding have no equivalent (see `SECURITY.md`). +pub fn walk( fmt: Format, - source: R, + src: &dyn ByteSource, budget: &mut Budget, - visit: StreamVisit, + visit: Visit, ) -> Result, LimitHit> { + // A format the caller excluded at runtime is reported, not skipped: the + // container is there, exav declined to open it, and the scan has to be + // able to say so. Every walk, nested ones included, passes through here. + if !budget.limits().allows(fmt) { + let meta = MemberMeta { + name: format!("{fmt:?}"), + comp_size: src.len() as u64, + size: None, + encrypted: false, + unsupported: Some("format excluded by the caller's allowed_formats"), + }; + return Ok(visit(&meta, None, budget)); + } let caught = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - dispatch_stream(fmt, source, budget, visit) + dispatch(fmt, src, budget, visit) })); match caught { + // A decoder that met a failed read sees it as the end of its input, so + // a walk that finished may still have missed the rest of the container. + Ok(Ok(None)) => match src.read_error() { + Some(e) => Err(LimitHit::corrupt(format!("{fmt:?}: {e}"))), + None => Ok(None), + }, Ok(r) => r, Err(_) => Err(LimitHit::corrupt(format!( - "{fmt:?} streaming decoder panicked on malformed input" + "{fmt:?} decoder panicked on malformed input" ))), } } -/// Format dispatch for [`stream_members`]; kept separate so the panic-containment -/// boundary wraps every streaming decoder uniformly. -// Every arm of the match below is behind a format feature, so in a build with -// none of them the body is `match fmt {}` and no parameter is read. -#[allow(unused_variables, unused_mut)] -fn dispatch_stream( +/// How far into a container the testing-faults marker is looked for. +#[cfg(feature = "testing-faults")] +const PROVOKE_REACH: usize = 64 * 1024 * 1024; + +/// Format dispatch for [`walk`], inside its panic boundary. A format whose +/// members can come out as they are read has its own `walk` (7z's reads the +/// container whole, then streams its members); every other one is read whole +/// ([`whole`]). +// In a build with no streamed format, every format takes the last arm. +#[allow(unreachable_code, unused_variables)] +fn dispatch( fmt: Format, - mut source: R, + src: &dyn ByteSource, budget: &mut Budget, - visit: StreamVisit, + visit: Visit, ) -> Result, LimitHit> { - // Fail on demand, to test the boundary rather than the decoders — the - // streaming half of the hook in `dispatch_extract`. The top-level scan - // routes every natively-streaming container here, never through the - // buffered dispatch, so a hook only there is unreachable from a real - // `exav ` run. The source is a reader, not a slice, so the marker is - // looked for in a bounded prefix of the raw container bytes; a marker - // carried in a stored member or a header field near the start of a - // well-formed container lands inside it. + // Fail on demand, to test the containment boundary rather than the + // decoders. The marker is looked for anywhere in the container's first + // bytes, so it can be carried inside a well-formed one. #[cfg(feature = "testing-faults")] - { - let mut prefix = vec![0u8; 64 * 1024]; - if source.seek(io::SeekFrom::Start(0)).is_ok() { - let mut n = 0; - while n < prefix.len() { - match source.read(&mut prefix[n..]) { - Ok(0) => break, - Ok(k) => n += k, - Err(_) => break, - } - } - prefix.truncate(n); - // Each format arm seeks to where it needs to be; rewinding here - // keeps the probe invisible to arms that read without seeking. - let _ = source.seek(io::SeekFrom::Start(0)); - crate::provoke(&prefix); - } - } - match fmt { + crate::provoke(&src.window(0, PROVOKE_REACH)); + #[allow(unused_imports)] + use crate::formats as f; + type WalkFn = fn(&dyn ByteSource, &mut Budget, Visit) -> Result, LimitHit>; + let walk: WalkFn = match fmt { #[cfg(feature = "gzip")] - Format::Gzip => stream_gzip(&mut source, budget, visit), + Format::Gzip => f::gzip::walk, #[cfg(feature = "bzip2")] - Format::Bzip2 => { - // Same trade as 7z below: the COMPRESSED input is buffered (bounded - // by the peak-buffer limit — it is the file on disk, not its - // output), and the decompressed content then streams. That is the - // win, since a `.bz2` decompresses to far more than it occupies. - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("bzip2: {e}")))?; - let (buf, truncated) = crate::bounded_read(&mut source, budget.limits.max_buffer_bytes) - .map_err(|e| LimitHit::corrupt(format!("bzip2: {e}")))?; - if truncated { - return Err(LimitHit::new( - "bzip2 container exceeds max-buffer".to_string(), - )); - } - budget.count_entry()?; - let mut dec = - crate::formats::bzip2::content_reader(&buf, budget.limits.max_buffer_bytes)?; - let meta = MemberMeta { - name: "bzip2-content".to_string(), - comp_size: buf.len() as u64, - encrypted: false, - unsupported: None, - }; - visit_member(&meta, &mut dec, budget, visit) - } + Format::Bzip2 => f::bzip2::walk, + #[cfg(feature = "lzw")] + Format::Lzw => f::lzw::walk, + #[cfg(feature = "lz4")] + Format::Lz4 => f::lz4::walk, #[cfg(feature = "xz")] - Format::Xz => { - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("xz: {e}")))?; - let (buf, truncated) = crate::bounded_read(&mut source, budget.limits.max_buffer_bytes) - .map_err(|e| LimitHit::corrupt(format!("xz: {e}")))?; - if truncated { - return Err(LimitHit::new("xz container exceeds max-buffer".to_string())); - } - budget.count_entry()?; - let mut dec = crate::formats::xz::content_reader(&buf); - let meta = MemberMeta { - name: "xz-content".to_string(), - comp_size: buf.len() as u64, - encrypted: false, - unsupported: None, - }; - visit_member(&meta, &mut dec, budget, visit) - } + Format::Xz => f::xz::walk, + #[cfg(feature = "zstd")] + Format::Zstd => f::zstd::walk, + #[cfg(feature = "lzip")] + Format::Lzip => f::lzip::walk, #[cfg(feature = "sevenz")] - Format::SevenZip => { - // 7z's header is at the end (random access), so the *compressed* - // container is buffered (bounded by the peak-buffer limit); each - // member's *decompressed* solid-block output then streams — that is - // the win, since a block can be far larger than the packed archive. - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("7z: {e}")))?; - let (buf, truncated) = crate::bounded_read(&mut source, budget.limits.max_buffer_bytes) - .map_err(|e| LimitHit::corrupt(format!("7z: {e}")))?; - if truncated { - return Err(LimitHit::new("7z container exceeds max-buffer".to_string())); - } - crate::formats::sevenz::stream_sevenz(&buf, budget, visit) - } + Format::SevenZip => f::sevenz::walk, #[cfg(feature = "cab")] - Format::Cab => crate::formats::stream_cab(&mut source, budget, visit), + Format::Cab => f::cab::walk, #[cfg(feature = "tar")] - Format::Tar => stream_tar(&mut source, budget, visit), + Format::Tar => f::tar::walk, #[cfg(feature = "zip")] - Format::Zip => stream_zip(source, budget, visit), - #[cfg(feature = "zstd")] - Format::Zstd => stream_single(&mut source, budget, visit, "zstd-content", |r| { - ruzstd::decoding::StreamingDecoder::new(r) - .map(|d| Box::new(d) as Box) - .map_err(|e| LimitHit::corrupt(format!("zstd: {e}"))) - }), - // `LzipReader` takes any failure to read a member header, an I/O error - // included, for the end of the stream, so errors are caught underneath. - #[cfg(feature = "lzip")] - Format::Lzip => { - let mut src = ErrorLatch { - inner: &mut source, - failed: None, - }; - let out = stream_single(&mut src, budget, visit, "lzip-content", |r| { - Ok(Box::new(lzma_rust2::LzipReader::new(r)) as Box) - })?; - match (out, src.failed) { - (None, Some(e)) => Err(LimitHit::corrupt(format!("lzip: read failed: {e}"))), - (out, _) => Ok(out), - } - } + Format::Zip => f::zip::walk, #[cfg(feature = "lha")] - Format::Lha => stream_lha(&mut source, budget, visit), + Format::Lha => f::lha::walk, #[cfg(feature = "ar")] - Format::Ar => { - let members = - crate::formats::ar::stream_offsets(&mut source, budget.limits.max_buffer_bytes)?; - stream_stored(&mut source, budget, visit, &members) - } + Format::Ar => f::ar::walk, #[cfg(feature = "cpio")] - Format::Cpio => { - let members = crate::formats::cpio::stream_offsets(&mut source)?; - stream_stored(&mut source, budget, visit, &members) - } + Format::Cpio => f::cpio::walk, #[cfg(feature = "machofat")] - Format::Machofat => { - let members = crate::formats::machofat::stream_offsets(&mut source)?; - stream_stored(&mut source, budget, visit, &members) - } + Format::Machofat => f::machofat::walk, #[cfg(feature = "pyc")] - Format::Pyc => { - let members = crate::formats::pyc::stream_offsets(&mut source)?; - stream_stored(&mut source, budget, visit, &members) - } + Format::Pyc => f::pyc::walk, #[cfg(feature = "sfx")] - Format::Sfx => { - let members = - crate::formats::sfx::stream_offsets(&mut source, budget.limits.max_buffer_bytes)?; - stream_stored(&mut source, budget, visit, &members) - } + Format::Sfx => f::sfx::walk, #[cfg(feature = "tnef")] - Format::Tnef => { - let members = - crate::formats::tnef::stream_offsets(&mut source, budget.limits.max_buffer_bytes)?; - stream_stored(&mut source, budget, visit, &members) - } + Format::Tnef => f::tnef::walk, #[cfg(feature = "partition")] - Format::Partition => { - let members = crate::formats::partition::stream_offsets(&mut source)?; - stream_stored(&mut source, budget, visit, &members) - } + Format::Partition => f::partition::walk, #[cfg(feature = "iso")] - Format::Iso => { - let (members, mut seen) = - crate::formats::iso::stream_offsets(&mut source, budget.limits.max_buffer_bytes)?; - if let Some(t) = stream_stored(&mut source, budget, visit, &members)? { - return Ok(Some(t)); - } - // The same order as `extract_iso`: the ISO 9660 trees, then UDF, - // skipping the files the first already emitted. - crate::formats::udf::stream_udf(&mut source, budget, visit, &mut seen) - } + Format::Iso => f::iso::walk, #[cfg(feature = "onenote")] - Format::OneNote => { - let members = crate::formats::onenote::stream_offsets( - &mut source, - budget.limits.max_buffer_bytes, - )?; - stream_stored(&mut source, budget, visit, &members) - } + Format::OneNote => f::onenote::walk, #[cfg(feature = "swf")] - Format::Swf => stream_swf(&mut source, budget, visit), + Format::Swf => f::swf::walk, + #[cfg(feature = "dmg")] + Format::Dmg => f::dmg::walk, #[cfg(feature = "szdd")] - Format::Szdd => stream_szdd(&mut source, budget, visit), - _ => Err(LimitHit::corrupt(format!( - "{fmt:?} is not stream-extractable" + Format::Szdd => f::szdd::walk, + _ => return whole(fmt, src, budget, visit), + }; + walk(src, budget, visit) +} + +/// The member of a single-stream compressor. +#[allow(dead_code)] +pub(crate) fn single_meta( + name: &str, + src: &dyn ByteSource, + unsupported: Option<&'static str>, +) -> MemberMeta { + MemberMeta { + name: name.to_string(), + comp_size: src.len() as u64, + size: None, + encrypted: false, + unsupported, + } +} + +/// All of `src`, for a format whose parser needs the whole container: the +/// bytes themselves when they are in memory, else read into memory up to the +/// buffer limit. +pub(crate) fn read_whole<'a>( + fmt: Format, + src: &'a dyn ByteSource, + budget: &Budget, +) -> Result, LimitHit> { + let max = budget.limits.max_buffer_bytes; + match src.materialize(usize::try_from(max).unwrap_or(usize::MAX)) { + Some(data) => Ok(data), + None if src.len() as u64 > max => Err(LimitHit::new(format!( + "{fmt:?} container is {} bytes, over the {max}-byte buffer limit \ + (--max-object-bytes), and its format is read whole: its members \ + were not scanned", + src.len() + ))), + None => Err(LimitHit::corrupt(format!( + "{fmt:?}: {}", + src.read_error() + .unwrap_or_else(|| "the source ended early".to_string()) ))), } } +/// A format whose parser needs the whole container and decodes each member +/// whole. +pub(crate) fn whole( + fmt: Format, + src: &dyn ByteSource, + budget: &mut Budget, + visit: Visit, +) -> Result, LimitHit> { + let data = read_whole(fmt, src, budget)?; + let stopped = crate::dispatch_extract(fmt, &data, budget, &mut |e: crate::Entry, + budget: &mut Budget| + -> Option< + Result, + > { + emit_entry(e, budget, visit).transpose() + })?; + stopped.transpose() +} + +/// Hand the visitor a member an extractor decoded whole. +pub(crate) fn emit_entry( + e: crate::Entry, + budget: &mut Budget, + visit: Visit, +) -> Result, LimitHit> { + let meta = MemberMeta { + name: e.name, + comp_size: e.comp_size, + size: e.unsupported.is_none().then_some(e.data.len() as u64), + encrypted: e.encrypted, + unsupported: e.unsupported, + }; + // An unsupported member may still carry what was decoded of it. + let content = (e.unsupported.is_none() || !e.data.is_empty()).then_some(e.data); + emit_bytes(&meta, content, budget, visit) +} + +/// Hand the visitor a member decoded whole. Its bytes are charged to the scan +/// budget, as a streamed member's are while it is read. +pub(crate) fn emit_bytes( + meta: &MemberMeta, + content: Option>, + budget: &mut Budget, + visit: Visit, +) -> Result, LimitHit> { + if let Some(data) = &content { + budget.charge_scan(data.len() as u64)?; + } + Ok(visit(meta, content.map(Member::Bytes), budget)) +} + /// Shared driver for a single-member sequential compressor: seek to the start, /// build a streaming `Read` decoder over the source via `mk`, and hand it to the /// visitor under a [`BudgetReader`]. The decoder pulls compressed bytes on /// demand, so a member decompressing to any size is never materialized here. #[allow(dead_code)] -fn stream_single( +pub(crate) fn stream_single( source: &mut R, budget: &mut Budget, - visit: StreamVisit, + visit: Visit, name: &str, mk: impl for<'a> FnOnce(&'a mut R) -> Result, LimitHit>, ) -> Result, LimitHit> { - source - .seek(io::SeekFrom::Start(0)) + let comp_size = source + .seek(io::SeekFrom::End(0)) + .and_then(|len| source.seek(io::SeekFrom::Start(0)).map(|_| len)) .map_err(|e| LimitHit::corrupt(format!("{name} seek: {e}")))?; budget.count_entry()?; let mut dec = mk(source)?; let meta = MemberMeta { name: name.to_string(), - comp_size: 0, + comp_size, + size: None, encrypted: false, unsupported: None, }; - visit_member(&meta, &mut *dec, budget, visit) -} - -/// Passes reads and seeks through, keeping the first read error for a caller -/// whose decoder does not report it. -#[cfg(feature = "lzip")] -struct ErrorLatch { - inner: R, - failed: Option, -} - -#[cfg(feature = "lzip")] -impl Read for ErrorLatch { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.inner.read(buf).inspect_err(|e| { - if e.kind() != io::ErrorKind::Interrupted && self.failed.is_none() { - self.failed = Some(io::Error::new(e.kind(), e.to_string())); - } - }) - } + emit_stream(&meta, &mut *dec, budget, visit) } -#[cfg(feature = "lzip")] -impl Seek for ErrorLatch { - fn seek(&mut self, to: io::SeekFrom) -> io::Result { - self.inner.seek(to) +/// The most a member `comp_size` bytes long in its container may decode to +/// before its ratio is a bomb's ([`crate::ratio_guard`]); `None` when its size +/// in the container is unknown. +fn ratio_cap(comp_size: u64, budget: &Budget) -> Option { + if comp_size == 0 { + return None; } + let ratio = budget.limits.max_compression_ratio.saturating_add(1); + Some( + comp_size + .saturating_mul(ratio) + .saturating_sub(1) + .max(crate::RATIO_FLOOR_BYTES - 1), + ) } -/// Run one member's reader through the visitor under a [`BudgetReader`], mapping -/// an over-budget overflow to a [`LimitHit`]. +/// Hand the visitor a member decoded as it is read, under a [`BudgetReader`], +/// mapping an over-budget overflow to a [`LimitHit`]. /// -/// A streamed member is **never retained**, so it is bounded by the cumulative +/// A streamed member is not held here, so it is bounded by the cumulative /// *scan* budget ([`Budget::remaining_scan`] = `max_scanned_bytes − scanned`), a -/// processing limit — not by the per-member *buffer* cap (`max_buffer_bytes`) -/// that governs buffered extraction. The caller (`exav-core::member_stream_scan`) -/// holds only a bounded prefix (`deep_analysis_max`) in RAM and streams the rest -/// through the constant-memory matcher, so a member far larger than the RAM -/// buffer is scanned in full without ever being materialized. Consumed bytes are -/// charged against the scan budget so a container of many large members is still -/// bounded cumulatively. -pub(crate) fn visit_member( +/// processing limit, not by the per-member *buffer* cap (`max_buffer_bytes`) +/// that bounds what is held whole. What the visitor reads is charged against +/// the scan budget, so a container of many large members is still bounded +/// cumulatively. +pub(crate) fn emit_stream( meta: &MemberMeta, inner: &mut dyn Read, budget: &mut Budget, - visit: StreamVisit, + visit: Visit, ) -> Result, LimitHit> { - let cap = budget.remaining_scan(); + let scan_cap = budget.remaining_scan(); + let ratio_cap = ratio_cap(meta.comp_size, budget); + let cap = ratio_cap.map_or(scan_cap, |r| r.min(scan_cap)); let mut br = BudgetReader::new(inner, cap); - let out = visit(meta, Some(&mut br), budget); + let out = visit(meta, Some(Member::Stream(&mut br)), budget); if br.overflowed { + if ratio_cap.is_some_and(|r| r < scan_cap) { + return Err(LimitHit::new(format!( + "member '{}': compression ratio > {}", + meta.name, budget.limits.max_compression_ratio + ))); + } return Err(LimitHit::new(format!( "member '{}' exceeds scan budget {}", meta.name, budget.limits.max_scanned_bytes @@ -475,632 +452,24 @@ pub(crate) fn visit_member( Ok(out) } -#[cfg(feature = "gzip")] -fn stream_gzip( - source: &mut R, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("gzip seek: {e}")))?; - budget.count_entry()?; - let mut dec = crate::inflate::Gunzip::new(io::BufReader::new(source)); - let meta = MemberMeta { - name: "gzip-content".to_string(), - comp_size: 0, - encrypted: false, - unsupported: None, - }; - visit_member(&meta, &mut dec, budget, visit) -} - -/// Stream a ZIP member-by-member off a seekable source. Only the central -/// directory and the members actually scanned are read (a range-backed reader -/// fetches nothing more). A cleartext member is handed to the visitor as the -/// crate's decompressing reader — so a member decompressing to any size is -/// scanned without being materialized. An encrypted member must be decrypted -/// whole (decryption needs the full ciphertext), bounded by the peak-buffer -/// limit; that buffered plaintext is then presented as a reader. -#[cfg(feature = "zip")] -fn stream_zip( - mut source: R, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - // Borrow the source for the central-directory parse so it survives a parse - // failure. A forged/corrupt central directory (bad CDFH offset in the EOCD, - // etc.) makes the seekable member walk impossible — but the members' Local - // File Headers are still in the file. Fall back to buffering (bounded by - // `max-buffer`) and the local-header salvage the buffered extractor performs, - // then present each recovered member to the streaming visitor, so a malformed - // archive still gets its members scanned instead of being written off whole - // (matching clamd). Without this fallback, such an archive is written off as - // `LimitsExceeded`, masking the detection inside. - let mut zip = match ::zip::ZipArchive::new(source.by_ref()) { - Ok(z) => z, - Err(_) => { - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("zip seek: {e}")))?; - let max = budget.limits.max_buffer_bytes; - let mut data = Vec::new(); - source - .by_ref() - .take(max.saturating_add(1)) - .read_to_end(&mut data) - .map_err(|e| LimitHit::corrupt(format!("zip read: {e}")))?; - if data.len() as u64 > max { - return Err(LimitHit::corrupt(format!( - "corrupt zip exceeds max-buffer {max}" - ))); - } - // Collect salvaged members (local-header scan), then hand each to the - // streaming visitor. Collecting first keeps the visitor's `Result` - // error path clean (the buffered `Sink` can't propagate errors). - let mut entries = Vec::new(); - crate::formats::zip::extract_zip::( - &data, - budget, - &mut |e, _| { - entries.push(e); - None - }, - )?; - // `extract_zip` already counted each of these toward `max_members`. - return visit_entries(entries, budget, visit); - } - }; - for i in 0..zip.len() { - budget.count_entry()?; - // Peek metadata with the RAW reader — it never invokes the crate - // decryptor, so it succeeds for encrypted members too. - // A member whose header will not parse costs only itself — see the same - // guard in `formats::zip::extract_zip_from`. Abandoning the walk here - // strands every later member: this pass drops them and the salvage pass - // skips them as already covered, so a payload behind one bad directory - // pointer is never scanned and the file reports unreadable, not infected. - if let Err(e) = zip.by_index_raw(i).map(|_| ()) { - let hit = crate::formats::zip::zip_entry_error(i, &e); - if !hit.is_corrupt() { - return Err(hit); - } - let meta = MemberMeta { - name: format!("zip entry {i}"), - comp_size: 0, - encrypted: false, - unsupported: Some("ZIP member header will not parse"), - }; - if let Some(t) = visit(&meta, None, budget) { - return Ok(Some(t)); - } - continue; - } - let (name, is_file, encrypted, comp) = { - let f = zip - .by_index_raw(i) - .map_err(|e| crate::formats::zip::zip_entry_error(i, &e))?; - ( - f.name().to_string(), - f.is_file(), - f.encrypted(), - f.compressed_size(), - ) - }; - // `is_file()` is false purely because the name ends in '/'. A JAR packer - // buys exactly that: `kingDavid/9.class/` holds a real deflate-compressed - // class that the JVM loads by name, while every ZIP tool discards it as a - // folder. Skip only what carries nothing at all — a "directory" with - // content is content. (The same guard lives in `formats/zip.rs`; this is - // the streaming walker the top-level scan actually takes.) - if !is_file && comp == 0 { - continue; // directory — counted toward the file budget above, skipped - } - if encrypted { - if let Some(t) = stream_zip_encrypted(&mut zip, i, name, comp, budget, visit)? { - return Ok(Some(t)); - } - continue; - } - // Cleartext member: stream the decompressing reader on demand. - // - // A codec the `zip` crate lacks fails here rather than in our own - // decoder, so fall back to the raw bytes and decode them ourselves — - // otherwise a member exav CAN decode (Deflate64, LZMA, bzip2, zstd, XZ) - // is reported unsupported purely because of which walker reached it. - // The buffered walker already does this; the top-level scan takes this - // one, so a fix in only one of the two never reaches the scanner. - // Probe first so the borrow of `zip` ends before the fallback needs it. - let decodable = zip.by_index(i).is_ok(); - let mut file = match decodable { - true => crate::formats::zip::member_reader(&mut zip, i) - .map_err(|e| crate::formats::zip::zip_entry_error(i, &e))?, - false => { - if let Some(t) = stream_zip_raw_decode(&mut zip, i, name, comp, budget, visit)? { - return Ok(Some(t)); - } - continue; - } - }; - let meta = MemberMeta { - name, - comp_size: comp, - encrypted: false, - unsupported: None, - }; - if let Some(t) = visit_member(&meta, &mut file, budget, visit)? { - return Ok(Some(t)); - } - } - // Members left out of the central directory while their local headers and - // data stay in the file, which is what the target extracts. The buffered - // extractor has always looked for them; this is the walk every scanned ZIP - // takes, so without it a hidden member is never read. - let hidden = crate::formats::zip::hidden_members(zip, budget)?; - visit_entries(hidden, budget, visit) -} - -/// Hand already-decoded members to a streaming visitor, stopping when it does. -#[cfg(feature = "zip")] -fn visit_entries( - entries: Vec, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - for entry in entries { - let meta = MemberMeta { - name: entry.name, - comp_size: entry.comp_size, - encrypted: entry.encrypted, - unsupported: entry.unsupported, - }; - let hit = if entry.unsupported.is_some() { - visit(&meta, None, budget) - } else { - let mut cur = io::Cursor::new(&entry.data[..]); - visit_member(&meta, &mut cur, budget, visit)? - }; - if let Some(t) = hit { - return Ok(Some(t)); - } - } - Ok(None) -} - -/// Decode a member the `zip` crate refused, from its RAW bytes, using exav's own -/// codec set. The member is visited once: decoded, or as a marker saying why -/// it could not be. Too large for the per-object buffer is a limit. -#[cfg(feature = "zip")] -fn stream_zip_raw_decode( - zip: &mut ::zip::ZipArchive, - i: usize, - name: String, - comp: u64, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - let cap = budget.limits.max_buffer_bytes; - let mut meta = MemberMeta { - name, - comp_size: comp, - encrypted: false, - unsupported: None, - }; - let too_big = |name: &str| { - LimitHit::new(format!( - "zip member '{name}' is larger than --max-object-bytes {cap}" - )) - }; - let decoded = match zip.by_index_raw(i) { - Err(_) => Err("ZIP member header will not parse"), - Ok(f) => { - if f.compressed_size() > cap { - return Err(too_big(&meta.name)); - } - let method = crate::formats::zip::zip_method_code(&f.compression()); - let usz = f.size(); - let mut raw = Vec::new(); - match f.take(cap).read_to_end(&mut raw) { - Err(_) => Err("ZIP member data could not be read"), - Ok(_) => crate::formats::zip::decode_zip_raw(method, &raw, usz, cap) - .ok_or("unsupported zip compression method"), - } - } - }; - match decoded { - Err(reason) => { - meta.unsupported = Some(reason); - Ok(visit(&meta, None, budget)) - } - Ok((_, true)) => Err(too_big(&meta.name)), - Ok((out, false)) => { - budget.commit(out.len() as u64); - visit_member(&meta, &mut io::Cursor::new(out), budget, visit) - } - } -} - -/// Handle one encrypted ZIP member: try the password pool, present the decrypted -/// plaintext as a reader on success, else a metadata-only unsupported member so -/// the caller surfaces PasswordProtected without masking later members. -#[cfg(feature = "zip")] -fn stream_zip_encrypted( - zip: &mut ::zip::ZipArchive, - i: usize, - name: String, - comp: u64, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - let unsupported_meta = MemberMeta { - name: name.clone(), - comp_size: comp, - encrypted: true, - unsupported: Some("encrypted ZIP member"), - }; - // Disprove the flag BEFORE any `decrypt` gating: the check decrypts nothing, - // it proves by CRC-32 that these bytes were never ciphertext. - let max_buffer = budget.limits.max_buffer_bytes; - let (crc, enc) = { - let mut f = zip - .by_index_raw(i) - .map_err(|e| crate::formats::zip::zip_entry_error(i, &e))?; - let crc = f.crc32(); - match crate::formats::zip::read_encrypted_member(&mut f, max_buffer) { - Ok(e) => (crc, e), - Err(_) => return Ok(visit(&unsupported_meta, None, budget)), - } - }; - // The bit can lie — an APK packer sets it on every member because - // Android's ZIP reader ignores it, buying a PASSWORD-PROTECTED report on - // an archive the platform installs happily. A CRC-32 match over a plain - // decode proves the bytes were never encrypted. (Same guard as the - // buffered walker in `formats/zip.rs`; this is the path the top-level - // scan takes, and a fix in only one of the two does not reach the - // scanner.) - if let Some(plain) = crate::formats::zip::cleartext_despite_flag(&enc, crc) { - budget.commit(plain.len() as u64); - let meta = MemberMeta { - name, - comp_size: comp, - encrypted: false, - unsupported: None, - }; - let mut cur = io::Cursor::new(plain); - return Ok(visit(&meta, Some(&mut cur), budget)); - } - // Actually encrypted. Without the `decrypt` feature there is no cipher - // stack compiled in, so report it — never decrypted, never silently clean. - #[cfg(not(feature = "decrypt"))] - { - Ok(visit(&unsupported_meta, None, budget)) - } - #[cfg(feature = "decrypt")] - { - match crate::formats::zip::decrypt_zip_member(&enc, crc, budget)? { - Some(plain) => { - budget.commit(plain.len() as u64); - // Decrypted: the plaintext is scanned AND the member is still - // reported as having been encrypted. Clearing the flag on success - // is what made cracking a password erase the report of one. - let meta = MemberMeta { - name, - comp_size: comp, - encrypted: true, - unsupported: None, - }; - let mut cur = io::Cursor::new(plain); - Ok(visit(&meta, Some(&mut cur), budget)) - } - None => Ok(visit(&unsupported_meta, None, budget)), - } - } -} - -/// Stream an LHA/LZH archive: `delharc`'s per-member decoder is already a -/// `Read`, so each member is handed to the visitor and decoded on demand rather -/// than drained into a `Vec`. Members with an unsupported compression method are -/// skipped (matching the buffered extractor), and directories are ignored. -#[cfg(feature = "lha")] -fn stream_lha( - source: &mut R, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("lha seek: {e}")))?; - let mut dec = delharc::LhaDecodeReader::new(&mut *source) - .map_err(|e| LimitHit::new(format!("lha: {e}")))?; - loop { - let supported = !dec.header().is_directory() && dec.is_decoder_supported(); - if supported { - let name = dec.header().parse_pathname_to_str(); - budget.count_entry()?; - let meta = MemberMeta { - name, - comp_size: 0, - encrypted: false, - unsupported: None, - }; - if let Some(t) = visit_member(&meta, &mut dec, budget, visit)? { - return Ok(Some(t)); - } - } - match dec.next_file() { - Ok(true) => {} - Ok(false) => break, - Err(e) => return Err(LimitHit::new(format!("lha: {e}"))), - } - } - Ok(None) -} - -/// Incremental SZDD (MS-Compress) LZSS decoder as a `Read`: the 4 KiB ring -/// window is driven byte-by-byte from the source and decompressed output is -/// produced on demand, so a large expanded file is scanned without ever being -/// buffered. Mirrors `formats::szdd`'s `lzss_decompress` exactly. -#[cfg(feature = "szdd")] -struct SzddReader<'a> { - src: &'a mut dyn Read, - window: [u8; 4096], - wpos: usize, - declared: u64, - produced: u64, - out: [u8; 18], // one token emits ≤ 18 bytes (backref len 3..18) - out_len: usize, - out_pos: usize, - flags: u32, - bits_left: u8, - done: bool, -} - -#[cfg(feature = "szdd")] -impl<'a> SzddReader<'a> { - fn new(src: &'a mut dyn Read, declared: u64) -> Self { - Self { - src, - window: [0x20u8; 4096], - wpos: 4096 - 16, - declared, - produced: 0, - out: [0u8; 18], - out_len: 0, - out_pos: 0, - flags: 0, - bits_left: 0, - done: false, - } - } - fn next_byte(&mut self) -> io::Result> { - let mut b = [0u8; 1]; - match self.src.read(&mut b)? { - 0 => Ok(None), - _ => Ok(Some(b[0])), - } - } - fn emit(&mut self, b: u8) { - self.out[self.out_len] = b; - self.out_len += 1; - self.window[self.wpos] = b; - self.wpos = (self.wpos + 1) & 4095; - self.produced += 1; - } -} - -#[cfg(feature = "szdd")] -impl Read for SzddReader<'_> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let mut w = 0; - while w < buf.len() { - if self.out_pos < self.out_len { - buf[w] = self.out[self.out_pos]; - self.out_pos += 1; - w += 1; - continue; - } - if self.done || self.produced >= self.declared { - break; - } - self.out_len = 0; - self.out_pos = 0; - if self.bits_left == 0 { - match self.next_byte()? { - Some(f) => { - self.flags = f as u32; - self.bits_left = 8; - } - None => { - self.done = true; - break; - } - } - } - let is_literal = (self.flags & 1) == 1; - self.flags >>= 1; - self.bits_left -= 1; - if is_literal { - match self.next_byte()? { - Some(b) => self.emit(b), - None => { - self.done = true; - break; - } - } - } else { - let (b0, b1) = match (self.next_byte()?, self.next_byte()?) { - (Some(a), Some(b)) => (a, b), - _ => { - self.done = true; - break; - } - }; - let mut mpos = (b0 as usize) | (((b1 as usize) & 0xF0) << 4); - let len = ((b1 as usize) & 0x0F) + 3; - for _ in 0..len { - if self.produced >= self.declared { - break; - } - let b = self.window[mpos & 4095]; - self.emit(b); - mpos = mpos.wrapping_add(1); - } - } - } - Ok(w) - } +/// A member stored as it stands in its container, `(name, offset, size)`, or +/// a part of the container the walk stopped short of (too many partitions, +/// too many ISO directories, a table it would not walk) and why. +#[allow(dead_code)] +pub(crate) enum Region { + Member(String, u64, u64), + Unwalked(String, &'static str), } -/// Stream SZDD (`expanded.bin` via the incremental LZSS `Read` above). KWAJ — the -/// other MS-Compress variant, with several compression methods — is decoded by -/// the buffered extractor over a bounded read and its members re-emitted, since -/// it is not a single LZSS stream. -#[cfg(feature = "szdd")] -fn stream_szdd( - source: &mut R, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - const SZDD_MAGIC: &[u8; 8] = b"SZDD\x88\xF0\x27\x33"; - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("szdd: {e}")))?; - let mut hdr = [0u8; 14]; - let n = crate::read_full(source, &mut hdr)?; - if n >= 8 && &hdr[0..8] == SZDD_MAGIC { - if n < 14 { - return Err(LimitHit::corrupt("szdd: truncated header".to_string())); - } - let declared = u32::from_le_bytes([hdr[10], hdr[11], hdr[12], hdr[13]]) as u64; - budget.count_entry()?; - source - .seek(io::SeekFrom::Start(14)) - .map_err(|e| LimitHit::corrupt(format!("szdd: {e}")))?; - let mut rdr = SzddReader::new(&mut *source, declared); - let meta = MemberMeta { - name: "expanded.bin".to_string(), - comp_size: 0, - encrypted: false, - unsupported: None, - }; - return visit_member(&meta, &mut rdr, budget, visit); - } - // KWAJ / other: buffered decode, bounded, members re-emitted. - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("szdd: {e}")))?; - let (buf, truncated) = crate::bounded_read(&mut *source, budget.limits.max_buffer_bytes) - .map_err(|e| LimitHit::corrupt(format!("szdd: {e}")))?; - if truncated { - return Err(LimitHit::new("szdd input exceeds max-buffer".to_string())); - } - for e in crate::extract(Format::Szdd, &buf, budget)? { - let meta = MemberMeta { - name: e.name, - comp_size: e.comp_size, - encrypted: e.encrypted, - unsupported: e.unsupported, - }; - let r = if e.unsupported.is_some() { - visit(&meta, None, budget) - } else { - let mut cur = io::Cursor::new(e.data); - visit(&meta, Some(&mut cur), budget) - }; - if r.is_some() { - return Ok(r); - } +impl From<(String, u64, u64)> for Region { + fn from((name, offset, size): (String, u64, u64)) -> Self { + Region::Member(name, offset, size) } - Ok(None) } -/// Stream an SWF movie: `CWS` (zlib) / `ZWS` (LZMA) bodies decode through a `Read` -/// adapter, so a large decompressed movie is scanned without being buffered. The -/// rebuilt `FWS` header is chained ahead of the decoded body (matching -/// [`crate::formats::extract_swf`]). `FWS` (already uncompressed) yields no member -/// — its bytes are covered by the caller's raw-container scan. -#[cfg(feature = "swf")] -fn stream_swf( - source: &mut R, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - let total = source - .seek(io::SeekFrom::End(0)) - .map_err(|e| LimitHit::corrupt(format!("swf: {e}")))?; - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("swf: {e}")))?; - let mut hdr = [0u8; 17]; - let n = crate::read_full(source, &mut hdr)?; - if n < 8 || (&hdr[0..3] != b"CWS" && &hdr[0..3] != b"ZWS") { - return Ok(None); // FWS / non-SWF: covered by the raw-container scan - } - budget.count_entry()?; - let mut fws = Vec::with_capacity(8); - fws.extend_from_slice(b"FWS"); - fws.extend_from_slice(&hdr[3..8]); - let meta = MemberMeta { - name: "movie.swf".to_string(), - comp_size: 0, - encrypted: false, - unsupported: None, - }; - let unsupported = MemberMeta { - name: "movie.swf".to_string(), - comp_size: total.saturating_sub(8), - encrypted: false, - unsupported: Some("SWF LZMA decode unsupported"), - }; - if &hdr[0..3] == b"CWS" { - source - .seek(io::SeekFrom::Start(8)) - .map_err(|e| LimitHit::corrupt(format!("swf: {e}")))?; - let dec = flate2::read::ZlibDecoder::new(&mut *source); - let mut reader = io::Cursor::new(fws).chain(dec); - visit_member(&meta, &mut reader, budget, visit) - } else { - // ZWS / LZMA: 8 header + 4 comp-length + 5 LZMA props before the stream. - if n < 17 { - return Ok(visit(&unsupported, None, budget)); - } - let file_length = u32::from_le_bytes([hdr[4], hdr[5], hdr[6], hdr[7]]) as u64; - let want = file_length.saturating_sub(8); - let props = hdr[12]; - let dict_size = swf_dict_size( - u32::from_le_bytes([hdr[13], hdr[14], hdr[15], hdr[16]]), - want, - budget.limits.max_buffer_bytes, - ); - source - .seek(io::SeekFrom::Start(17)) - .map_err(|e| LimitHit::corrupt(format!("swf: {e}")))?; - match lzma_rust2::LzmaReader::new_with_props(&mut *source, want, props, dict_size, None) { - Ok(dec) => { - let mut reader = io::Cursor::new(fws).chain(dec); - visit_member(&meta, &mut reader, budget, visit) - } - Err(_) => Ok(visit(&unsupported, None, budget)), - } - } -} - -/// Pick the LZMA dictionary size for a `ZWS` movie. Both `declared` (the props -/// header's dictionary field) and `want` (the movie header's own `FileLength`, -/// less the 8-byte header) are attacker-controlled, and the dictionary is -/// allocated up front — so `want` is no ceiling on its own: a movie declaring -/// 4 GiB would buy itself a 4 GiB dictionary. `max_buffer` is the real bound; -/// `want` only ever tightens it, since a dictionary larger than the bytes it -/// will be used to look back into cannot be consulted. -#[cfg(feature = "swf")] -fn swf_dict_size(declared: u32, want: u64, max_buffer: u64) -> u32 { - crate::bounded_dict(declared, want.min(max_buffer)) -} - -/// Emit a set of pre-parsed stored (uncompressed) members `(name, offset, size)` -/// by seeking to each and handing the visitor a bounded window — the shared tail -/// of every STORED-OFFSET format (ar/cpio/machofat/…). No member data is ever -/// buffered here. +/// Emit pre-parsed stored (uncompressed) members by seeking to each and handing +/// the visitor a bounded window: the shared tail of every format whose members +/// are stored as they are (ar, cpio, machofat...). No member data is buffered. // Dead only in a build with none of the STORED-OFFSET formats (ar, cpio, // machofat, pyc, sfx, tnef, partition, iso, onenote); see // `crate::cap_prealloc` for why the feature list is not spelled out. @@ -1108,91 +477,43 @@ fn swf_dict_size(declared: u32, want: u64, max_buffer: u64) -> u32 { pub(crate) fn stream_stored( source: &mut R, budget: &mut Budget, - visit: StreamVisit, - members: &[(String, u64, u64)], + visit: Visit, + members: impl IntoIterator>, ) -> Result, LimitHit> { - for (name, offset, size) in members { + for region in members { budget.count_entry()?; - // A zero-length `<…>` region is a walker saying it stopped short — too - // many partitions, too many ISO directories. It names content that was - // never enumerated, so it has to arrive as UNSUPPORTED. Emitting it as - // an ordinary empty member would make "we did not look" indistinguishable - // from "we looked and it was empty", which is the shape of a silent - // clean. The buffered walkers say this with `Entry::unsupported`; this - // is the streamed half of the same statement. - if *size == 0 && name.starts_with('<') && name.ends_with('>') { - let meta = MemberMeta { - name: name.clone(), - comp_size: 0, - encrypted: false, - unsupported: Some("container walk stopped at its limit"), - }; - if let Some(t) = visit(&meta, None, budget) { - return Ok(Some(t)); + let (name, offset, size) = match region.into() { + Region::Member(name, offset, size) => (name, offset, size), + // Content that was never enumerated arrives as UNSUPPORTED. + // Emitting it as an ordinary empty member would make "we did not + // look" indistinguishable from "we looked and it was empty", which + // is the shape of a silent clean. + Region::Unwalked(name, reason) => { + let meta = MemberMeta { + name, + comp_size: 0, + size: None, + encrypted: false, + unsupported: Some(reason), + }; + if let Some(t) = visit(&meta, None, budget) { + return Ok(Some(t)); + } + continue; } - continue; - } + }; source - .seek(io::SeekFrom::Start(*offset)) + .seek(io::SeekFrom::Start(offset)) .map_err(|e| LimitHit::corrupt(format!("stored member seek: {e}")))?; - let mut window = source.take(*size); - let meta = MemberMeta { - name: name.clone(), - comp_size: *size, - encrypted: false, - unsupported: None, - }; - if let Some(t) = visit_member(&meta, &mut window, budget, visit)? { - return Ok(Some(t)); - } - } - Ok(None) -} - -#[cfg(feature = "tar")] -fn stream_tar( - source: &mut R, - budget: &mut Budget, - visit: StreamVisit, -) -> Result, LimitHit> { - // The SAME reader the buffered `extract` path uses. Two readers of one - // format can disagree, and when the scan path holds the weaker one, a header - // form it mishandles makes the scanner see nothing while `exav-unpack list` - // shows every member — the diagnostic tool contradicting the defect. One - // reader cannot disagree with itself. - // - // No cost to memory: `entries()` needs only `Read`, and each entry IS a - // reader bounded to its member, so nothing is buffered. - source - .seek(io::SeekFrom::Start(0)) - .map_err(|e| LimitHit::corrupt(format!("tar seek: {e}")))?; - let mut archive = ::tar::Archive::new(source); - let entries = archive - .entries() - .map_err(|e| LimitHit::corrupt(format!("tar: {e}")))?; - for entry in entries { - budget.count_entry()?; - let mut entry = match entry { - Ok(e) => e, - // A truncated archive: entries before the cut were already scanned, - // and the missing tail is absent rather than hidden. exav scans for - // malware, it is not an integrity validator — the buffered path makes - // the same call for the same reason. - Err(e) if crate::formats::tar::is_truncation(&e) => return Ok(None), - Err(e) => return Err(LimitHit::corrupt(format!("tar entry: {e}"))), - }; - let name = entry - .path() - .map(|p| p.to_string_lossy().into_owned()) - .unwrap_or_else(|_| "tar-entry".to_string()); - let size = entry.size(); + let mut window = source.take(size); let meta = MemberMeta { name, comp_size: size, + size: Some(size), encrypted: false, unsupported: None, }; - if let Some(t) = visit_member(&meta, &mut entry, budget, visit)? { + if let Some(t) = emit_stream(&meta, &mut window, budget, visit)? { return Ok(Some(t)); } } @@ -1213,69 +534,103 @@ mod tests { use crate::Limits; use std::io::Cursor; - /// Collect every member `(name, bytes)` the streaming API yields for `blob`. + /// Every member `(name, bytes)` a visitor reading each one whole sees in + /// `blob`, leaving out those reported unsupported. /// /// Every test using it is behind a format feature, so it is dead in a build /// with none of them compiled in. #[allow(dead_code)] - fn streamed_members(fmt: Format, blob: &[u8]) -> Vec<(String, Vec)> { - let mut budget = Budget::new(Limits::default()); + fn members(fmt: Format, blob: &[u8]) -> Vec<(String, Vec)> { + members_with(fmt, blob, Budget::new(Limits::default())) + } + + #[allow(dead_code)] // see `members` + fn members_with(fmt: Format, blob: &[u8], mut budget: Budget) -> Vec<(String, Vec)> { let mut out: Vec<(String, Vec)> = Vec::new(); - let mut visit = |m: &MemberMeta, r: Option<&mut dyn Read>, _b: &mut Budget| -> Option<()> { - let mut data = Vec::new(); - if let Some(r) = r { - let _ = r.read_to_end(&mut data); + let mut visit = |m: &MemberMeta, content: Option>, _: &mut Budget| { + if m.unsupported.is_some() { + return None::<()>; } + let data = match content { + None => Vec::new(), + Some(Member::Bytes(d)) => d, + Some(Member::Stream(r)) => { + let mut d = Vec::new(); + let _ = r.read_to_end(&mut d); + d + } + }; out.push((m.name.clone(), data)); None }; - let _ = stream_members(fmt, Cursor::new(blob.to_vec()), &mut budget, &mut visit); + let _ = walk(fmt, &blob, &mut budget, &mut visit); out } - /// The streaming path must yield the same member `(name, bytes)` as the - /// buffered [`crate::extract`] path — the correctness contract for every - /// STORED-OFFSET conversion. - #[allow(dead_code)] // see `streamed_members` - fn assert_stream_matches_buffered(fmt: Format, blob: &[u8]) { - let mut budget = Budget::new(Limits::default()); - let buffered: Vec<(String, Vec)> = crate::extract(fmt, blob, &mut budget) - .unwrap() - .into_iter() - .filter(|e| e.unsupported.is_none()) - .map(|e| (e.name, e.data)) + /// `blob`'s members are exactly `want`. + #[allow(dead_code)] // see `members` + fn assert_members(fmt: Format, blob: &[u8], want: &[(&str, &[u8])]) { + let want: Vec<(String, Vec)> = want + .iter() + .map(|(n, d)| (n.to_string(), d.to_vec())) .collect(); - let streamed = streamed_members(fmt, blob); - assert_eq!( - streamed, buffered, - "{fmt:?}: streamed members differ from buffered" - ); + assert_eq!(members(fmt, blob), want, "{fmt:?}"); } - // A `ZWS` header carries two attacker-chosen sizes, and the LZMA dictionary - // is allocated before a single byte is decoded. Neither may set that size. - #[cfg(feature = "swf")] + /// An LHA member compressed with a method there is no decoder for is + /// reported, not skipped, and the walk goes on to the next one. + #[cfg(feature = "lha")] #[test] - fn swf_dictionary_is_bounded_by_the_buffer_limit() { - let max_buffer = Limits::default().max_buffer_bytes; - // The sizes from a movie that asked for a 2.7 GiB dictionary by declaring - // a ~4 GiB FileLength: neither number may be believed. - assert_eq!( - swf_dict_size(0xA1A1_C32B, 0xF04A_0957 - 8, max_buffer), - max_buffer as u32, - "a huge declared dictionary is clamped to the buffer limit" - ); - // A movie small enough to be honest keeps its own (smaller) dictionary. - assert_eq!( - swf_dict_size(1 << 16, 1 << 20, max_buffer), - 1 << 16, - "a dictionary under both bounds is used as-is" - ); - // `want` still tightens: no point holding more history than output. + fn an_lha_member_with_an_unknown_method_is_reported() { + // Level-0 headers: size, checksum, method, sizes, time, attribute, + // level, name, CRC-16 of the content. + fn member(method: &[u8; 5], name: &[u8], body: &[u8]) -> Vec { + let crc = body.iter().fold(0u16, |mut crc, &b| { + crc ^= b as u16; + for _ in 0..8 { + crc = if crc & 1 != 0 { + (crc >> 1) ^ 0xA001 + } else { + crc >> 1 + }; + } + crc + }); + let mut h = method.to_vec(); + h.extend_from_slice(&(body.len() as u32).to_le_bytes()); + h.extend_from_slice(&(body.len() as u32).to_le_bytes()); + h.extend_from_slice(&[0, 0, 0x21, 0x5a, 0x20, 0]); + h.push(name.len() as u8); + h.extend_from_slice(name); + h.extend_from_slice(&crc.to_le_bytes()); + let sum = h.iter().fold(0u8, |s, &b| s.wrapping_add(b)); + [&[h.len() as u8, sum][..], &h, body].concat() + } + let blob = [ + member(b"-lh0-", b"a.txt", b"hello"), + member(b"-lh9-", b"b.txt", b"12345"), + vec![0], + ] + .concat(); + let mut seen = Vec::new(); + let mut budget = Budget::new(Limits::default()); + walk( + Format::Lha, + &blob.as_slice(), + &mut budget, + &mut |m, content, b| { + let data = content.map(|c| c.into_bytes(m, b).unwrap().0); + seen.push((m.name.clone(), m.unsupported.is_some(), data)); + None::<()> + }, + ) + .unwrap(); assert_eq!( - swf_dict_size(u32::MAX, 1 << 20, max_buffer), - 1 << 20, - "the declared output bounds the dictionary when it is the smaller" + seen, + [ + ("a.txt".to_string(), false, Some(b"hello".to_vec())), + ("b.txt".to_string(), true, None), + ] ); } @@ -1307,12 +662,12 @@ mod tests { ..Limits::default() }); let mut seen = 0; - let mut visit = |_: &MemberMeta, _: Option<&mut dyn Read>, _: &mut Budget| -> Option<()> { + let mut visit = |_: &MemberMeta, _: Option>, _: &mut Budget| -> Option<()> { seen += 1; None }; - let walk = stream_members(Format::Zip, Cursor::new(blob), &mut budget, &mut visit); - assert!(walk.is_ok(), "{:?}", walk.err()); + let walked = walk(Format::Zip, &blob, &mut budget, &mut visit); + assert!(walked.is_ok(), "{:?}", walked.err()); assert_eq!(seen, 2); } @@ -1351,7 +706,7 @@ mod tests { let eocd = blob.len() - 22; blob[eocd + 16..eocd + 20].copy_from_slice(&((cd + orphan.len()) as u32).to_le_bytes()); - let names: Vec = streamed_members(Format::Zip, &blob) + let names: Vec = members(Format::Zip, &blob) .into_iter() .map(|(n, _)| n) .collect(); @@ -1373,7 +728,7 @@ mod tests { #[cfg(feature = "ar")] #[test] - fn stream_ar_matches_buffered() { + fn ar_members() { let mut blob = Vec::new(); blob.extend_from_slice(b"!\n"); let member = |name: &str, data: &[u8], out: &mut Vec| { @@ -1391,12 +746,12 @@ mod tests { }; member("a.txt/", b"abc", &mut blob); // odd → padded member("b.bin/", b"xy", &mut blob); - assert_stream_matches_buffered(Format::Ar, &blob); + assert_members(Format::Ar, &blob, &[("a.txt", b"abc"), ("b.bin", b"xy")]); } #[cfg(feature = "cpio")] #[test] - fn stream_cpio_newc_matches_buffered() { + fn cpio_newc_members() { // SVR4 "new ASCII" cpio: 110-byte hex header, header+name and data each // padded to 4 bytes. fn member(name: &str, data: &[u8], out: &mut Vec) { @@ -1425,12 +780,19 @@ mod tests { member("b.bin", b"\x00\x01\x02\x03\x04", &mut blob); // Trailer. member("TRAILER!!!", b"", &mut blob); - assert_stream_matches_buffered(Format::Cpio, &blob); + assert_members( + Format::Cpio, + &blob, + &[ + ("dir/a.txt", b"hello world"), + ("b.bin", b"\x00\x01\x02\x03\x04"), + ], + ); } #[cfg(feature = "machofat")] #[test] - fn stream_machofat_matches_buffered() { + fn machofat_members() { let mut blob = Vec::new(); blob.extend_from_slice(&0xCAFE_BABEu32.to_be_bytes()); // FAT_MAGIC blob.extend_from_slice(&2u32.to_be_bytes()); // nfat_arch = 2 @@ -1447,12 +809,16 @@ mod tests { } blob.extend_from_slice(a0); blob.extend_from_slice(a1); - assert_stream_matches_buffered(Format::Machofat, &blob); + assert_members( + Format::Machofat, + &blob, + &[("macho-arch-0", a0), ("macho-arch-1", a1)], + ); } #[cfg(feature = "onenote")] #[test] - fn stream_onenote_matches_buffered() { + fn onenote_members() { let mut blob = Vec::new(); blob.extend_from_slice(&crate::formats::onenote::ONENOTE_HEADER_GUID); blob.extend_from_slice(&[0u8; 16]); // filler @@ -1466,12 +832,12 @@ mod tests { blob.extend_from_slice(&0u32.to_le_bytes()); // unused blob.extend_from_slice(&0u64.to_le_bytes()); // reserved blob.extend_from_slice(payload); - assert_stream_matches_buffered(Format::OneNote, &blob); + assert_members(Format::OneNote, &blob, &[("onenote-embedded-0", payload)]); } #[cfg(feature = "szdd")] #[test] - fn stream_szdd_matches_buffered() { + fn szdd_member() { let payload = &b"HELLO SZDD WORLD -- literal-encoded body for the equivalence test"[..]; let mut blob = Vec::new(); blob.extend_from_slice(b"SZDD\x88\xF0\x27\x33"); // magic @@ -1484,48 +850,184 @@ mod tests { blob.push(flag); blob.extend_from_slice(chunk); } - assert_stream_matches_buffered(Format::Szdd, &blob); + assert_members(Format::Szdd, &blob, &[("expanded.bin", payload)]); } - // 7z pattern-A streaming must yield exactly the same files (as a set) as the - // buffered extractor — including the solid multi-file case, where each file is - // carved from one decompressed block by skip+take without buffering the block. - #[cfg(feature = "sevenz")] + /// Every `compress` fixture, at both code widths, decodes to the file it + /// was made from, and cut short to a prefix of it. The fixtures cross width + /// changes and table resets, and are larger than a source chunk. + #[cfg(feature = "lzw")] + #[test] + fn lzw_member_is_the_compressed_file() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/lzw"); + let mut seen = 0; + for e in std::fs::read_dir(dir).unwrap().flatten() { + let path = e.path().to_string_lossy().into_owned(); + let Some(stem) = path.split(".b1").next().filter(|_| path.contains(".Z")) else { + continue; + }; + // `read_fixture` unmasks `.xor` itself. + let blob = crate::read_fixture(path.trim_end_matches(".xor")).unwrap(); + let plain = crate::read_fixture(&format!("{stem}.txt")).unwrap(); + assert!(blob.starts_with(&[0x1f, 0x9d]), "{path}"); + assert!(blob.len() > 4 * crate::source::CHUNK || !path.contains("bigreset")); + assert_members(Format::Lzw, &blob, &[("lzw-content", &plain)]); + let cut = &members(Format::Lzw, &blob[..blob.len() * 2 / 3])[0].1; + assert!(!cut.is_empty() && plain.starts_with(cut), "{path}"); + seen += 1; + } + assert_eq!(seen, 10); + } + + /// A linked frame whose matches reach back tens of KiB into earlier blocks, + /// across a skippable frame and into a legacy one, decodes to what its + /// sequences say; cut short, to a prefix of that. + #[cfg(feature = "lz4")] + #[test] + fn lz4_matches_reach_back_across_blocks_and_frames() { + // One LZ4 length field: the nibble, then 255s and the remainder. + fn len_field(n: usize, nibble: &mut u8, tail: &mut Vec) { + if n < 15 { + *nibble = n as u8; + return; + } + *nibble = 15; + let mut rest = n - 15; + while rest >= 255 { + tail.push(255); + rest -= 255; + } + tail.push(rest as u8); + } + fn sequence(lits: &[u8], offset: u16, mlen: usize, out: &mut Vec) { + let (mut hi, mut lo) = (0u8, 0u8); + let (mut lit_tail, mut match_tail) = (Vec::new(), Vec::new()); + len_field(lits.len(), &mut hi, &mut lit_tail); + len_field(mlen - 4, &mut lo, &mut match_tail); + out.push(hi << 4 | lo); + out.extend_from_slice(&lit_tail); + out.extend_from_slice(lits); + out.extend_from_slice(&offset.to_le_bytes()); + out.extend_from_slice(&match_tail); + } + let mut seed = 0x2545_f491_u32; + let mut noise = |n: usize| -> Vec { + (0..n) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed as u8 + }) + .collect() + }; + // What the sequences decode to, built alongside them. + let mut want: Vec = Vec::new(); + let copy_back = |want: &mut Vec, offset: usize, len: usize| { + for _ in 0..len { + let b = want[want.len() - offset]; + want.push(b); + } + }; + let mut linked = vec![0x04, 0x22, 0x4D, 0x18, 0x40, 0x70, 0xDF]; + // A stored block first, so the matches after it have 70 KB behind them. + let first = noise(70_000); + linked.extend_from_slice(&(first.len() as u32 | 0x8000_0000).to_le_bytes()); + linked.extend_from_slice(&first); + want.extend_from_slice(&first); + for _ in 0..40 { + let mut body = Vec::new(); + let lits = noise(1000); + sequence(&lits, 60_000, 5000, &mut body); + want.extend_from_slice(&lits); + copy_back(&mut want, 60_000, 5000); + // A block ends with literals only. + let tail = noise(5); + body.push(5 << 4); + body.extend_from_slice(&tail); + want.extend_from_slice(&tail); + linked.extend_from_slice(&(body.len() as u32).to_le_bytes()); + linked.extend_from_slice(&body); + } + linked.extend_from_slice(&0u32.to_le_bytes()); + // A skippable frame, then a legacy one reaching back into the first. + linked.extend_from_slice(&[0x5A, 0x2A, 0x4D, 0x18, 3, 0, 0, 0, 1, 2, 3]); + linked.extend_from_slice(&[0x02, 0x21, 0x4C, 0x18]); + let mut body = Vec::new(); + sequence(b"legacy", 65_535, 3000, &mut body); + want.extend_from_slice(b"legacy"); + copy_back(&mut want, 65_535, 3000); + body.push(1 << 4); + body.push(b'!'); + want.push(b'!'); + linked.extend_from_slice(&(body.len() as u32).to_le_bytes()); + linked.extend_from_slice(&body); + + assert_members(Format::Lz4, &linked, &[("lz4-content", &want)]); + let cut = &members(Format::Lz4, &linked[..linked.len() * 2 / 3])[0].1; + assert!(!cut.is_empty() && want.starts_with(cut)); + } + + /// Every DMG fixture, HFS+ and APFS, plain, compressed and encrypted, + /// yields its one file. + #[cfg(feature = "dmg")] #[test] - fn stream_sevenz_matches_buffered() { - use std::collections::BTreeMap; - fn as_map(v: Vec<(String, Vec)>) -> BTreeMap> { - v.into_iter().collect() + fn dmg_members() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/dmg"); + let mut seen = 0; + for e in std::fs::read_dir(dir).unwrap().flatten() { + let path = e.path().to_string_lossy().into_owned(); + if !path.ends_with(".dmg") { + continue; + } + let blob = std::fs::read(&path).unwrap(); + let budget = Budget::with_passwords(Limits::default(), vec!["test123".to_string()]); + let files = members_with(Format::Dmg, &blob, budget); + seen += 1; + if cfg!(not(feature = "decrypt")) && path.contains("encrypted") { + assert!(files.is_empty(), "{path}"); + continue; + } + assert_eq!( + files, + [("/test.txt".to_string(), b"hello hfs+\n".to_vec())], + "{path}" + ); } - for name in ["lzma2.7z", "lzma.7z", "copy.7z", "deflate.7z", "bzip2.7z"] { + assert_eq!(seen, 7); + } + + // Every codec decodes, including the solid multi-file case, where each file + // is carved from one decompressed block by skip+take without buffering + // the block. + #[cfg(feature = "sevenz")] + #[test] + fn sevenz_codecs_decode() { + for name in [ + "lzma2.7z", + "lzma.7z", + "copy.7z", + "deflate.7z", + "bzip2.7z", + "lzma2_solid.7z", + ] { let p = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("tests/fixtures/7z") .join(name); - let blob = std::fs::read(&p).unwrap(); - let mut budget = Budget::new(Limits::default()); - let buffered: Vec<(String, Vec)> = - crate::extract(Format::SevenZip, &blob, &mut budget) - .unwrap() - .into_iter() - .filter(|e| e.unsupported.is_none()) - .map(|e| (e.name, e.data)) - .collect(); - let streamed = streamed_members(Format::SevenZip, &blob); - assert_eq!( - as_map(streamed), - as_map(buffered), - "{name}: streamed 7z files differ" + let files = members(Format::SevenZip, &std::fs::read(&p).unwrap()); + assert!( + !files.is_empty() && files.iter().all(|(_, d)| !d.is_empty()), + "{name}" ); } } - // Cab pattern-A streaming must yield the same files as the buffered - // extractor. No valid multi-file cab fixture ships (only fuzz cases), so we - // build a minimal STORED (uncompressed) cab with two files in one folder — - // exercising FolderReader + the skip/window/offset logic in stream_cab. + // No valid multi-file cab fixture ships (only fuzz cases), so we build a + // minimal STORED (uncompressed) cab with files in one folder, exercising + // FolderReader and the skip/window/offset logic in stream_cab. #[cfg(feature = "cab")] #[test] - fn stream_cab_matches_buffered() { + fn cab_stored_members() { fn build_stored_cab(files: &[(&str, &[u8])]) -> Vec { let folder_data: Vec = files.iter().flat_map(|(_, d)| d.iter().copied()).collect(); let cffile_start = 36u32 + 8; // header + 1 CFFOLDER @@ -1572,15 +1074,15 @@ mod tests { out.extend_from_slice(&folder_data); out } - let blob = build_stored_cab(&[ + let two: [(&str, &[u8]); 2] = [ ("readme.txt", b"hello cab world"), ("data.bin", b"\x00\x01\x02\x03\x04\x05"), - ]); - assert_stream_matches_buffered(Format::Cab, &blob); - // Two files in one folder with a gap-free layout also exercises the + ]; + assert_members(Format::Cab, &build_stored_cab(&two), &two); + // Three files in one folder with a gap-free layout also exercises the // per-file skip/window/offset bookkeeping. - let three = build_stored_cab(&[("x", b"AAAA"), ("y", b"BBBBBBBB"), ("z", b"C")]); - assert_stream_matches_buffered(Format::Cab, &three); + let three: [(&str, &[u8]); 3] = [("x", b"AAAA"), ("y", b"BBBBBBBB"), ("z", b"C")]; + assert_members(Format::Cab, &build_stored_cab(&three), &three); } // MSZIP cab (via the `cab` crate) with a multi-block folder: exercises the @@ -1588,7 +1090,7 @@ mod tests { // previous block's window (the MSZIP dictionary-priming fix). #[cfg(feature = "cab")] #[test] - fn stream_cab_mszip_matches_buffered() { + fn cab_mszip_members() { let mut builder = cab::CabinetBuilder::new(); let folder = builder.add_folder(cab::CompressionType::MsZip); folder.add_file("big.bin"); @@ -1605,7 +1107,11 @@ mod tests { i += 1; } writer.finish().unwrap(); - assert_stream_matches_buffered(Format::Cab, &blob); + assert_members( + Format::Cab, + &blob, + &[("big.bin", &big), ("tail.txt", bodies[1])], + ); } // A member past the cap: never delivers a byte beyond `cap`, and the read diff --git a/crates/exav-unpack/tests/fixtures/zip/bzip2_large_then_small.zip b/crates/exav-unpack/tests/fixtures/zip/bzip2_large_then_small.zip new file mode 100644 index 0000000000000000000000000000000000000000..77f58a5cc9a314bbd8cd92a6879de0f342a128c2 GIT binary patch literal 20699 zcmV)9K*hgMO9KQh0000C0000X04A{>F7ZzQ03c2P00#g70AgupE@EkJLRx4!F+o`- zQ(16T_1*vgdjJ3b|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0|NsBsG9P{Ck8``~ z?%nO(e6(%tsk`rc=ihUAx39kU*S_WI-*39}W8L@N*NeBl_t#rg_kHfD-)nvMp7%cA zea<-dbo+hxqu+0Dm)~}`ozAbV-rIXyT|V9S*LBZc-rcuvwe{^Z_dfI6-M!sH`|j@> z-S2zvm(K3{-&ytD*LB$aOqw+G5s8U{r|G69OcOK$X@IAVCYYF* zm_}+~CQMA20v?3Z34k;Kp@?D{4KPgxrpVP%3a9v*Vq!8gNv3E3m1$Qm><3_t)+NXdYV1j1!c(Tbu~Kgo=Z0E|iD7^i?n zCMF2NU;#2I?3e%)@i3Ssj1bI5Oqm*CG}AyD1Y%^<34t)0WX6*K05qAR7!47HQ3|K{ zBPIb1OiWCe)WI1Zso*Bm!5TeGm`@Ng88J^rhD?TrcD4cG@Bz+B=VTRm?j#S znqW+tHBZSK3Oq^Vz#3^WQ}o(tGGGk_9%`AIi1IO{Vq$6OF)-07?NJJ!nrWs@G#N0H z(^Jr5X*AKM)Wpf8+Ik9~jS~$CrcE0p(;z)l!ZghU^lD--r-3jO%~SE9NsIwK048V< z$vjP_gwqofPe2holWLa^TU&P=|4>oOE7GpWl4w>1=5gj!lXcffL^Zl7+sSPXd5`Ts zo*s&m3jpRva7b2nmkC`JZQ=IVxBoFMF)}#1Io&%Ctsbwrwt=vti2gk^T8NW`>W$HZ z;FMZ;OUgO9BOPzgrpfDS19M$hQgs))OI^UA4*a~dmjdM1U|(Op)wfR|gc8P4)0O>x z^T5}aqG15`4K?R08(Xxko@HM{aI$jE_b<9;gESw8C!{fjx*v>5p4xxTSYkxefSYO= z{Y~P0r-BV|0rohbi_No3M`7V?aF8`nY%5HUN4WiD@w9uk(o9n8Lqt!{on&z6t2UzrQQlxl(@$s%q z+tbS4d;YDabzO-y5Rye!w})-l`N)ylU01X6Kh}G9O!`-iF1OV3=}KF*c60!>qNg^I zcI#-3vkeNa6jL1mwj)iO+WDYC0h7nS_TYLmV(ODm6-0CK-gmD!uCNi_xTm!zm39F` zx#fhFDexq!-<*a>9Bywz?uq@txrQy(>+6J(tFW%RR#o=%tnHp0jVGlOU75IZ;s3$V z0>~>=u@(2ZP|v6_fh8Dk<=A3G;uKe6A%?x@v+MX$<-rY1ph<8xG15wv*+bDBofLY@ zA>1Bi4Cb(k(h(9BK)e9u{cOY@Ao_nD1FZB*(|O-w#WVP6`QrVtoe-a(AK+QvzMP0L zXA!Gu9opg2^n!z2#mur+`gHEBJU0tY_iw3JC_vCtZn%>S1J5CSg3l)P|J~Y!L_Bot#}m zFsQAjL`xgy^3ZhTYPNzS>fZ-T`NhZNI9B3>=%+6diAYX{%)F z_z;^#W2(U^Z6hc^TB?evat+zk2W0-d(QI>r|9fgzuHB}wH_`@&Jp3%b%w!=+4+wgm zqLZ2~=JE%x)gt+R*{i)tjSnx*@yP{;BNgre@+gtjt&O&w5W~0BanH=M2~3Ohn+2Au zJ7QHGgK8d2O;RWgMVu8orQ3QhV0@4i&YIG=I?KP2Ookfx{7ScxGi%Qbk_&74Ja^-q zwPenAl=PM0zhRL*yKBS6VbFzcq6=mJl@7?kB5t3%uD!^u)1+4Zfw^Q-sTKgnhIh%o zxZY&ePkJ=#pg_4jI@R~5d(kQZu%15to#V(87$Sdu6jTVrC+yaoV$m-?QRw>RKOOwW zQNDcK*CjS*#u-aZTn)w~!*eJj@ljjlJ^B2MjKTU-QnV}50Q(mtortcaD~~E-`!Eb- zG#h*Bi}4bw$LK%MkH%<;y`hD>7y;r9*lwCL8$ukAeMux4act_CYb~kPga+jb_~*)Q zt!_2kJ}>AU>71s^wq(+y9#wJYPT1-Pq-}F^qvw$LTs&j8@k`jnEC`j@b>H9ae9Qz$ zaV1YA<8}ItLGrxqYOYuiRi1k8AeIgA_%b7jgRP8Lg1tt_Q{pwr9`To^WAqk*M-I5j&k>RWyxMI?_@f*!k1NR321So6P^)26k~P3RML-EePK_*u`p7{TjlT} zbvxgq!`~gctR`p?hLZs)zh$s|*?T9qaVVynquxbeG)fii{q1gpv(8Wftos_9trRHU z{ud)|+8~M1zda=%NV`ee63!O4R#s(;xd4rd%nstX>2+(nsJlZK>cpV4_fa2#58XZ# z0khR(!vQDUk1Fhs#FsIdY=Y!{HpwxhYb+FuFnIaiA$0#`#wWpUZdS5o{vNOM9r*$p zpfYpX2@?m$Pp2lS!8;me4m(Q;)hpuHur(LCwmH7U+-~QWAcJc0sR0Yht3LK*gL^|; zT+|r&-f)5|erOe1%EtfDTP2L{*NAfD=#%R$v%aM{Kx0NeM&13cuHS;vVExQ_52}yt zqXRd~6cU}BpjjTiANK4$Xv`^!&kld6XCV3ePmK@NJneAG1M1rsT6_H8CjeqJV4y5_ zdtQy~=u%E&NkaR=@T+m1KQF!&E^e}5)Hr9Mxw@E_yE>T{Ic=(!EjA8~E1z8nw}Li+ zj%{Y@cH*EYV$E?4GP`7s-NY@LX5TbNMTIn?a`x(`V-r&`VZ#okDe9Aq@^M<)vRH3s zJq_uL7FGzR@B?*EO*;4WeW=_YlADDBRVw#1n65+ZJUqS%!r-F+O1--&f6joc3E-1*Ms_-ysmKaTJT$Npw{I<1+FgatEXcCw{t6|)*DYU zMhjA4WxX%v4Fg_%;R9@Kb98_$Ub~cwiDm{pR@1iHv)5O=mKK9Dy;B<(NNxCOPRwZY z%FArvf4w@X>L$(xiGIrPIxwWh{Ls(<^K}S9!UkK!1fJPNX`F^ z#qW26LV#q0pO4$pGFi?dBx+UW9!{NJVuQAJAp;1k;-GH0xrP1e`f6qLJ!!FFP@=ED z_s3LHFLh7$Bot{tvLY>T0ua@Lvb`NJ&$FtH)DA7axV@_E!W=L3l})YJU#nRk0q`>` zUb}%hGgWZ+&DbXC_=^f~?&-Bzo?k1TYJ;KF2CV8D&`!7c$lVmDe6&kN&Wr3n{$ryO z5xflHA!oTNCv0_*2~i z`9bQ%5c#9!BTovlonqfuefWuVE?wU-dDqBFy88}S!>(qrhvN^A%sbQjb9UyM=*sG) zMHFP7_~|305d}x5_!E_iW48*IjQhdzFY(%tki>iHwO){n4E|de9OuyB8cI=#&l)vx zFFwX|*MB>5W#5k3CGrILc8E|0`lQumH(8B%14=)O&5~-Q7=Jn%?3Rh__e8Hs|9}?9 z0GvO0ODXdmn0-%n@sG6BX6Xht+Fpe2w`H&$8u{bOn&LH^TgS?P*gGL_sxzIMO`fvX ztJD`IB`OMyI=bxDljd&*=>vYJpTSz-?65f>14VPePFx(_#&XC0J_k#;!QLVOG8V{2 zVr*f8fy0Uml5xFq-<0h`)J!%?8KpiQH$!E}%#FGH8oCN@r`b%HPmi5)F$Ih(z!UjT

5of3ac3HgLVd%x##E+s0~nvw~)_*cW+DH%u|5 zt8gY+qyH4~v3_3<7Z9Zy~kIU_F z-%$h9bGq$#ExV)sbO8d>*Lq#k-W^`krqNUfy?mKA>eJ^PGr)EGt_X11D7~gm^}GHb zD(gOyR3l7h4y;05IO|>TgPUsl;K#OlOB>vX74r#TW=Nmao^YYoc^(;8Rj(G$AYD~? z)k_GSD~(%ivX%55Cn|vFO*bt)gvx(GzGF8>2FWQ%)}T zG{9|jnm+|Ap_`36mD8b<*m_WY`2#)6Epf$e7|XLl2= zt9y+V2|2PxqsDf9+fU<;4P3vMW&xGLdEo{D@fsBE5glI3bYoH6YhNYsb$UH2!5B<3 zlTzvzdDj&d^cl+Ak;%1`M8!d7=wTy*P z`E6JMjAY#hMMGcA+2!(NG+Z-$q#@rD@vX*WW$>v8j%nl(4sCD2=f&6FXj$H1$1$UI z4eqX^91a@a&mRkQz%AzCHGeuk`@l_{PWlY=zkNEDjjnF&&$(s!XcG!CX%UzAu~VJ` z3ne`&erbQ2=(qlXq32k2;1w*`Hl}TQLW)cxOwZy`xDT^(05xkiyU<@U_@vN}l>ou9 zDHSyO4@oHsZ-EXZMJ%U$uYztHxooO}tw!uefITFXo0OP?QH zlZlt^Bz6Q14UzfZ)K$|;kM|sXoM2?{{R3avX-$qIhJ=0XM^Ng16@w_ZqzcPxP`orq zgz^HbU1@##wY9yk6S@a^rgBD6u8ywEB^|R?Mu+OiZoAD!xtvh<(U|x$Ez)?6?LM6O ztA1>W6@|%pAhDB)tDzN8oO=Wj+U@O%tbAJp&&2E{f3xa+%&k@;qQ%AP)9e~9Kz7J0 zi@vj0?gkX*O2g_4YA)9y?OZ*vJlA+DV^aD^>>ad-j24xIUP$3Ctz#74?75tB)kNE% z8{U?h+!N7W8VTGpfihv?USmGr2kU2#lQS*3;f18+lL>NNWG3axjuC7h&i_Uit^@bD zq|Pt`FEUunYcGESjcy8fP|$dXexx#aLm8#2Fr3C&AZ>2+hwTGZ4uf0Kw!6h zDxz6;T==b`{y{(ID`akNgd71W52d=3R2E?qF}F<9aG^*iHLg%%?pgUfnbzzQ8Pq`H zLGw2TK%}uCt*nBZdk908=qY@hShG*`wjdgJopv?cDA~15ranDN2fOJp^EFW^gCX*wbb?n)#3euz5+dA@xtv_tb-u8k6va zGM^gnNXnF?HF+LYy2S<-rb7{c>zGS|QnjQVRl4cL`)MI+PWX3_@Sa9orUx`6LKAS% zeT9C)W_cC&;yIJw#lTv(W-qAPeR9&a%rrbj2bi`DKCPu;h)k;8+PBQSBX{lC#7^bp z5<36yD|&YcX!i{fKKO~?XY+{$`5YzJ%7D&9?AM3|bRZ6%$^Sd?&ZN>lE#5Px;@>V1 z($Dplw3cm`SL-g^LC@mrrEzlJ;T;Vo4`MFZtE|E*qhs! zoovZwrQe056-c5YwzuVdc1J1_i4NCuemOk@UaXBb9kPq9UrN;KGbtZwt#Jk+5@w3O zBgiuV5nt$!E{O>}iYb15(TlKq33T;q!EGFXNA0uH|KkrjO-6`q7eQFJD}$5-mFwd{ zuGZQ`1;T&Xt_?IqMFzckMS0FTbQN_2G^@Xrf`m&8>&_5Ja^?uR!qdinTrG@yL(8}^ z%I{*fF<$JhQSdvq@e#R{)+eJn>cRBXl>!)y7bcpo4Nf7yn6{)@wmME$qld?W-jTw& zzG3GZ=8{_7F0lM-Lwe8Zz%|RTO>@a|X;GIr}WU<6z0!`pumSx!W6e3DOtzkjP3VT6ZFAL6dtqg$z$kVDVCZOOKK z-L?Q7SPH-mYw^^p(`Z`{PT4mr8?~7gxH+LDYp@77LCT`(z_UB@^ zZnvA5A}_ynjOS8iec+|{+y_k6U-x>HXqh13IOl1502Dz-;MRnfq@ktzuo|ugd$s_x z@C$tMx|@3_dXI=RqrRUQ224;lY0>enV^cMSwnf%YKOO+R+c&#=vhxpHV6s+9e2PR9rb5ptQ}{}mvSq)E;`lx}hs?ckuk6376d*(YwR&LtdFToaA;2s$XGpE?#j$gWprcbETqLwWNWl~X z8rQ*n_p?$;;oAQD@oeH3(z_F-uA6`5)PBdQLSj12ENQhi#Ie%Dd*GWn#So>qvq$>4IxqBg_w` zAvR^qB-F0n7%f*J*Pn$_QMCXMOTzo~9`7oI#{>s6*lv?!1)kqoI`%4iTAgaE$IDm* z&jh!YW^RaHqEi8cWuJVJy)oI95ECDK`k4ZKKXjgmdPwHtXDPup`;UnY))b-N|DkuXC5b_O5aaea35>=I0DGu32nc9Gc)$izzJ1AGa=#3R5hHq&( zy9aPfI^9|{j(Eey%rKBZHk$_Tbs%2X+P(DDaWYUv^IKx&l6$hu^+I^$^05V^yTwxk zT;|Sqo6~@jO#@1Zvz5rV;M>$1cH1+x92(2=4dUaEMAq+*v?x5eJJCYQ! z+q1nN4JMsZv|DVmY9Q3?dW!_*Dd3XtP4l;-{5IZIBV0au%{yL40$M8LXpfDt#9k_B zzmK4F|GVExFTy=z)X2oDMKeZ0m*~_|!*epC@?p}V{Mb)%ORXa8!;vO9nM|L1GPPr{ zWDt`gdOO-X-lyY=^(ppO`0f9eA10x>t^Psnkx3tK3br>`oje&^$@fu)F3e1On9x#i z@3pFfEFB3JpYnv_Kx_|fDR@m6+h+b!PR{GULeK*kqoO8^ebgkV8t$Gt!SV>H>#vsoGQudk-1Gq& zz9G1@m-_cS1ux5Zv2;4&J2rZ98cwG<{^yQ?uA94=1M!RU;$=9XSGpjk`?C%*;miyjj7e_ zdhQf_Mr!DOvm&~ymyz7rD#^u95!SgZwLh!>l*V-S2nmFc3naJp6&17YLHXYI3}Djq zq5uwvN)gH-VEeE_&Q^RZa`apO$yJFxQ_T<}@NcK-XI36|jG{s+$mMm<3lQvkxo;SS z=St`u{38}>GX2P7MjqRbk*@8Fd6@s79(y92tde!o+-eW?Wln&YBnTROsyO)jjsC*I z=3)7)(6Cyh0MfbIv{^gQF3Qd^IdhQ^BhSE-?T=j>f>a#%z`ahbMiMXm{P0u5Y@_c) z>UBm~KZuU9DS6&Z(O-roQ-66vg0s}9xhIZ- zUreUXm17NL?YQ!k5?b)83g}2uxtnjO$jK~u?&A^ACXn!LNCn^dMZdQiz3+c;xeEMh z>S#;K#p{MnctHt=ubbh!*ukhHM^zVQ*qPDI^uyT*QK@&kGoWI~p|Ind&~Hly-yX@x zHXaHvF`40q-nFSU3p357TvxA;;qBP8T_Vf6{N*JaNIx5Ba~Ct-x%7ohc%?sA?MgK* z5KhS!q=efjO*i=>%)G0ss1p+q_tvp-2YLQg0TvL++9zHJbB(W6i*OIJszbtP&TFQC z=xsI0Q(&}~U}*biRl8={BK8nlG~%hbhY~R0Hno0X%EN@L`<{sN#t67r_`={`-c~N5 z&s<)o6Ol#a~S!Ee)$SQpfOg^`nKJs&78MA{T zeB1`T4DCgM{Z;cu2n4RcVpPJItWrJT?9lx1?`T`1`)5mJ$H6#3cB=u?FJAE7I8T-# zU(j|I$d?=o0Fl1QpA^5v@K9U?Kg($gWa+*uK36+Y*^8I;tX8%53}1h4;Zs+sTO=FG zN{xeoUy-fUkBf%C9f8B&!{J?gEs8|GSIPDE!Pj)uxPyMmk?e?4$)MGm@R{ru;>au- zXAdjm&{(6EHDJpfFGIwpsP+49jQF|SBW-O}i6%ZUv1mAq8#|({#)b+^9b>!&oVq^! z7^d3@-C|Pp?eslG)@E{Xa;cze0l_nU)%pR^NUHYT)Fcs=J6#KZR=C}3rTgm)(?AW! zu@IZk6li@V7L2*?cp{t0CQd#(S%IYoO~g!b`)b2`4buXi_G>10tFFAby^>WAXJ7>CwKcJ)anRu?;fHZ8Se zmZmIy41arEQ%}8X`Mf5H>1GeMH7+CG>{m9(39?{K75z~TiwjRqcYre&Tx8|MBPqLN zy5AYOIQZG9?_D#Ke?+p9SdO?6i=m#veh5Mza+(c;pg_9vyC zuQQmS$?KwgO!8T#a4h*zYw+Q@zM`_SgAIvcM&DJ*fOoO@^^&S!ZP*Fm6S&)ROO+AB zWD^8kWPi|A3tf1MTz1y>MKAU-)?G-3JcdSs!KSCnWak z7b0=>DyhHBm1K2lA#w6~9Y3Yd2O3igLhCF?sJc0j1ZN1sm&({=ykYkW?K)2 zWau3$;N%C)ct%~!$@``wEcOn>iO|D`rlPRIQ48U6O zs~B!oFkBULq*KFFj(*OXXKXeyE(hD*sK!Td4_OCx&~D=E$Wk!J5OO=1Alby{h;EaG zp}u@ZFP zC*H}6iHthri7_dm>LEt;2g4;rs&5-Zr1l;1}L^QpQu%IixTj2xgCvb7Cg z4lyl1#V8Ctl)?{?&Ur+P_kgkVizDIE{;PBuG>aT-<<*17XHa2|qs#VAdcZcov$LaX zAkslWw9-6bMwB^11NEguo98)PmYIKOi$vn_sEAGRBcA#0&dht`7?O(cg;5)=QdLL8 z(-z?$e}S&!>9z)vP^^7?o9T{Bsj14Oj3wKsOCDS+%Gw(KHqGAilgxvJt}9RO79Tla zeJ?SExAFXFbBuli0MI&l?}R7)QS8j)XKAYWHEdqtv3Wx&Tyizfsc=+^7Y5&h-~+~N zZSTwIkjT*M{KG;@eP&7i;jxN1mu$2^wm5uo`t&@vuqEjkn-UtBkpA_-CoeDhRV1NE zo+DQ0lT=}8;m^zf3UK(s5jafS(8(0_Fu1>JZ@CHp<7|sahVS`s5ma>>?#xB1Me=pb zYmX5n0lGt?z6bJYO;snxSqhE!UIl+8i&%9Oujb0@qIgGtG}Ufuf&OUPBhi{#Pa{E0 z`gFz6Q57>*KPIPqFY7_c)y%Zg#5;Q2i+{q7*VyVfL*N`+-%S%z$yj`#kxl=A!?0Ny zv@5d2N9~aQZ8!5;L7V-{1ZVqR*g(vfQHz=MwB^i9{$!$07h4}VoluCX(0$9 zXC=mL0f(?gkb*)3SlD8&Zz5QM5Vk$?hknM!17DXW47#di6M9W3|;&=rb zWK=KJc+Z$PH0;oZq-iFGf)ED*Ps%%f5-7Zv-?^R+gQ>&^@Kkv`O!lbmrp>9z2ep=v z@y4N8Bgz~yRjON+)c@RGb-_Qb>=#q}>Xa|lyIGc~j3{E}VlNp1L(O^SsscOlu=OO` z?md&nmB*1Z*ZChV z6MG1|S-l&18=wZ#t$THZ1zN|hIl!UWPNulejp{$WX?F}z{}X-%)dVKR(mC?kRG#&+ z_0;tAWdq*=2$d_TA%XUi$Moyi#k_RQyqYBRqnC)!xI~O)z}cg?9ZALDo|+H>65a34 zqHuwvt#@=)Y!0C>%Ll_V7EiY^xOc7h-ZU2C zFbMkS$kAM>gFp7#-)KPLd$<@xdgIj1_bE08xW6gHVF2EJTNC>%%|&LykZ-{Zs=32v z2W>F=X*M%kR(}Gd)xI7uj8uwIr=`$9F(q3TUFxA0s!(@5elLeu#;zGLpN%aT1)T!1 z@6CUGAoUGpz5d!O zno-DSB<14G0S0y1b+RnC{Gha;6Ooggk*O)l1vqMqM0j(z)A^TZsmF!q$tf@)QEg~| zvNx})yw7UrpAc!7RLEjJZ_E7Xp5^l3S9*}=VN+sS*$5hoD=FC!1 zXOY4R{=dugcMRTASnoOEbU7u3stQRh*n#qJjM0ZY662CAETt{hS)zKs(JskhjjO?) zT){s1R($ez_b>5ar#cn^d2|XX#fpxR!Yr_kPAKjqIs}ee0SijDrBGn7KMFm-jJ5{^iruVc#MExqAx>!PrYk+C*z3P!JEC8e>cmU3$z2|&jGUZa+)5S= zs@uOuX1n+G8>oHqdAL9!ZDLO2Yxc1#lpC`=))UG!%&Q|QjodDA11Tm~C+SR-mZ)|% zs@zN2=e$*v#&OX%^rm&${aWYr`E?dMz3m7)z_spi;Jy)Tnm!+^DkAGAKaA=TAnSyL zrJxJk0R2cWM{a9ai*ZucMKE^%)+b$hGw4i8Ec>-Q`^V&;U1+s6O00W*Y$E3OFDl<5 z)nKJjP9Y5P^%AQl>9JrW&7VjmNB?syv?#VN zD!Zxr?6ZbZf5^bvujVDqOc#e&eNjrH0^b#>$OAHLsCT>@odd}j;Bs#9+B<)0@3Q!Q zwe>Nm6^N#xv!|CKrZ|4ONq(a8YF*fx{Q46Q2uaDQ{h{VhZD!$Ob`{KCQznt0Db%+#@{+-TgeUj3D(uRI!?%;ZYc za<(pSTN58jRS?hMQ$dQZ^-aPJ5?Jc}{LXbntjKD{*rbN~qPbEISzveNtR{e2?F&qSo;lcB7;gU5{9jXI{{Qxc_HhPx$+H>cgm`cQ!U{bfSvJ z0Tfhubv-nMG%9)9WTgbz_m6}AaacEHOrP5L9cTi$QvdRp*BDfHu5#Wj4KC+*?)}9u zhKbuBtcv<4h>q6>Al)z}YMfMG$%FSd+#f^m59eGS@7@S2$`ao}8rK8OUFTarU|J(ez@bIdNvNKgN_Ai<0m`@l#l4C4XN)8y6glFD6KCKg|1}M)XS=Qh3VE;6hM(YYMx=s)b z)9(d|D+1(#RqpCPyc3vGygr(pUzk(@INf91^PmEQ*Oun*Q9a-a*0UoK*QCnGabd6| zpui;5hgV#)|2-lFE~;9Cho4IMg2+l8ja8<1{gknGfZ7bSx|B17I1P4JRqL?2ZT99Y zromb0a$l~$M}3`#^TkL(^yq`VoT}G6bglMzbzU-P({9sJSvtm96&=>>M6QX6$Z#- zfxt|Yzy}XO+uTGAYyE=0%V`-&JNNlCTcW=KyYv?ag=JK6cxJCdjLH{p=EV4@H7;as7{&rIf1U@%sQnj6n1AZk8a8t(13Yk&>tUt8Dd>oUc%s=^rzPU2bS(*Ij^FUiz_T}-TYC&YJ`31T$FBm(S zjUG;Y21&}o-V4BdRvG&#hSIU!J6!f{m&{6rFHhY&AmaF{#*RpM4s2v4-`&HwhI%)< z-ES+-De3dHvjdS?G+VO*tB^Tq-jh(z(|yY#@kCmlN<`a+R?)PDeIM>@8CmjjZ7qK-E{hdMHxpHl=hrhD&mpVOI>bTY)zJcK2N7w%ugY7aqb%32wy!&l zVG<=djc1%Tz%Lq(K z+udlI`%*DiN~eKHnO}LesQ)F|#{+2Ez6|&(9c@nIb)uoVKH90%M{8#+zcC^KH?wRr z1#1$3LD6Srld2TmQL66wZvb2s&Rtbyz+uOJW zmpHLbsUSuOlIg(d5<(*0wXn@J!)%n~u?)}dor14=|5z5{FY!vP+0CDQi(NXE6j(si zhcyK*h7WSZUW~OMFF9jrHtXqX1U%K~Sec8Rqu>x9OgQJSQ~o1+OCiLoElm=xh&AWH zoVuGhPCQ#OC%SwAjS_$TCqC^0Zsj^}hnZp6yxHsX4{=Yn zA|1%k`lF43yw=CMKC90;$~o!AlB5e%ytaWlRLPj5SRX}cP=z~Cn~4|64!G6YimLHj z3)}yZbWx3VFjS8?xK3ZSw!A=Pf?uiVn`dRYCN$JgVoWvTbQxenbiXRyQGnKo2@=rj5 z)e{Ayau`vAsAuFJiYtU-Q~;>C-09=|*2_uZF%ezQECP3M?#z-nYjl7I&|=Z+%SM2& z7V@S$8DtNHMd>vQF7@-o`2IRq^MtPvq}dGnxC#nTX3s9Qa?(&3cxK&*hkNz51$` zU)kI<&{b3->1knDo`XXzX!BjU^dlt1kkO5q4wppQEzvi`JJw6$%!CI0P&FWg3HJ2v zBJLAHCw00pw+*`hnjh?IQA3ha=lJC(G;qL9HziqHE9Yy#-Ekj$@k=t!i4D>T^vngFG17H+D8gajtIo z13+Gr=hIORO3*#JSDR3Z7 zVuBP%N!X&K`|ZdW!Jo7QpIfdA>B3;+QNCvft>+0<%P(Z1Qbx91Y%;51h@UX_@qeG@ zmS46vzhQ_iSy25nhj1Cr`Lm^gyG&)=s~v^N)W0hso=|y%6RK%5#;IXbHTweEZykVM zEL@qGu!88nzS#U_cykz=!hW;*#A1aos>Uw31+?Q)vln^m*G9B~tGn% zLq^lvrnhT~qPM2JE>7~0t*V+n(EcQ55DHp?)LV{Bp1>?&A`NfJb%u1I4SE2s1aX((hM5FIwjU zTy{d%RMeLS$4$~HJG9M3pWAO8sS@id^xb@-M+c7Y-3c0CKFiicD8YR;pju@QUvUa2 zx_qHJUv%;xMJvur(PlFG-a-^JpXO1958`HXTeM>Ms`h04a|1NOK2K?I%{!peyLs;D zl{Nx$vL}?7$;?Uzb!nF(#N(#uPxGoHa}j1fnhwbjrtv zb2K?hL{6x&0$J*fYoJ#5DX1fjBCQ-a-dHq1G>c&B8mkQlukIE<-H6&}pp|>X$-VFH zT0wdShIFK`DHzB3BgX-VLo|pea?EbV)vC?!AaSUAtE&^k1q^QX{=uJmh>Y-RtyT>lrB5V=A(j+xaj3 z^pJG8g|p1)!5jzxkF6&NdgP^ii2K9FjD3rSD9SWzwB}c~PI>YXPZ@-d>5#8tg5UUn zVTYX@@oVs;noYlHNh#{zPHUqoYBPY@|H<{jmuH6bvjI6PUo3<>tL_s^ zbv0Z-G|w-m2ZvoW4&oO_u31Z~_GU2nsWO^mzRdpK<9T6seg2DJg0H*)si%g}=F3dp9o)ef2e>je4GO-0ewux!tV)Hw zufxB7Urx?TwH%rf!C-xUKQsXnM?X%0Q)TvhC3U_Bsuqeb2voQ@^3JEw4nZPNHnp{4 z)U4&8S3x;e+K$*nF-p&XYyCEjTEOXG#ZBZAlkId&>pP%ko=8pEfI#*@T%2Flu(V0+)il?l$PSEEH|AS1@x}gMK1YTp>zdpS(2LvwK?wH$Oho0;o~;4gD#_ zUEZR5Y>O9RK%ZzQk{%JTEp(&g0mcv>)`|Y$@3puWCYa={S^OBTcQ+$A^uQ1N4&RJ| z+8foGyDar6@$?fT8sS>?MkfDRSs6tccnrQRn|NCgoWl``K>6?MN6SQVC4jZEpuylG zl481E_p&VTB%6+7vEJ&?9>f*)v(v8WO6SZag{c;~fL8X?`Q3LBH!GBV3XoDhBYgFt zlKo6r7Aa7W^*|T>inSUxm}M+FT#@1r#xC1(x(~3OMMbVYj>!_atx1zrB8xYqQGDG8 zKv8f#+qy{)go-|(?+kXdH^kdyRdyJbc_wy#C)jC0Ue=7m(iqO0{lr(L6UBl1Y?!9^ zYXbf=jmlo(csI1EVi|#<^cv$%7o{)qrx#-=wjhbh!kY+7mO;7S(Sn8ClRCYJb*k9^KalTdOQ!>OKPN0z&*hbZcZEjUrZ}#cQ_lYJA zv+Gt_#`gqV{vZ_-Drg{mm~m1-Y!rJVNmzLU;Cq;e+Wf?`OhB>u?%jCT>(Q~`uBd)< zjUhj3uIyje(@MmI!v~EY8N-?{&7{CC7rGF%MTm%f5n19--QO7f3CG?yJHOBAG-#ve zkC0Cthg+SRE3u9*Y=YMk^TYrM5{@!w4iN^ZXQ&Qto>rCYmMjz-7{0o` zBZEz(EyjMfn84=cbYn5BqkHTcOw#*cimia1%U;N)im+&)PExa< zni7zqi)dfsjJvqhRvToUz3|%aU%qJn9RQ$y`f#;m(;XD94l<^G?@z?w^H&U(NL=-V zD3dHeh!k8T49u15*ZCWGreU)fIS`(d&J1Oj6C)7a!kDF`Y)m`-+~oHi2^6gpghv*) z@5|bc)wxRqMx(m%&ta*?=VsR|n&@hdnQ7nd{iBEhKvLz}TJ!mc573DEh=xd6ea?(& z^e--m+Farg%-u4H@~2-@4{+>AxvR}@>VR&si$;T!LFlf}mO&anKoi)1dHoVF^+0i{ zYqxG&dP?2J2}0EIJ2*aT7lO_+XE|-$t97^SQ>KWy%j?vLG@If)Q7Jrw>X%g&Zl83P zqH%<4dE}XF;cga#fHFHZ$!vQ$g-7qjM3u1+)0aaq`I)(CChqD%N zZNEeC<;V|Lp-So_e}dGRn4(4&RsIVuO{0@1(&;dB1+6BU)`G~!Gy7AY`OOc&^_M*d z=d*;5<=GW)$IB6Skt~~m5r|Hc@a));rSIEV4gIf_{%`c9SrP{Y5&2uJlhHE(u_M2~%JU2OrJ+f<`zKkkSf05PY=sjA#cCW1ey!-o#3ye5zMn!^wo&_I{hpFOxUl?A(uSAEC_RT zJ{$rnaFVE^ z;C4i>oCWi)3A0S~UGuOgRzY~;m?cPP7S^TjozqMz#`|UDy|z$nLiaNUTm{R2aSkpg zMS}l+4&rvz_>6%{M${-u3X$~kdUvYl!^}B8E9^h@k6Hu3VptD%;UBvj$>S5MWj`F+ zzwYA_!phABm4~BeO9Av{t$-*O$v`G8hqxf5&vC-aqLIf3LDt=iy+>o)c#5; z;1`B$XS=SP;a%pyz#?<+K^Q8%WCA&?M%gd2M~yqib@*Nf3Wl%6K%Ah2_(U2*T#;xd z1UPUqcUIC+1$G3~`JiS`S$oLU2?Al0q4TEKGr~y!2+?IGa1C9Ce%j<=fEJ&UCJ`o) zd&jXbDLz-V-1*Kx-u+xgF4g$E{m?AhjW_M8AvtwQo2}i$&S(>gk8V;^pm*xd;=GPH zZ}g208QW~rsN*g2Es*P}X3Dwm>1JWR`S4Qh_*EkJg(yRn*RbW%1h#V+AyiQm{m430>!Pwrr<@{ZJ(d$K8#IP(xi zU&BJUEd7x&u&SAaI8G6SLYQ3^t>}S-ZJzpR|B(FPsbG>s|2LbEBG|_3|HN@{z58Rr7uRej&)pBR1^TSMpC*P>5y<0B$e*21*DdSB?MfMl#p0z>5vXV zi3JuAlm-E%V+mzJLPA+uVu4-i^51(O?#rDy-5i?hkJ;!DDr-m{qX zyX(##Xlp!QvnQRI|IjL5= zH7@faWSsX!=5TL@C*-r7r1{;lyZ4#7+<6w*8RC@$9}^F~J1mtDaq_Z}*V#xpNlEsY z%z3Du?L7&{aMa4bC^K`Ar;*u8XTIseM6t_sAgcJNR2~1W1iex$8O?V&>y&XVufu0Wj5V%)KcM*rk4!UwmUGLhy#S5Fp`e;u(Cy!}A{-&|6^6UyT8TYziRk|v_w~0HLsQ0!r7?zMa=ij$u+z%mO)v> zsc7yQQ*djSR&s&<;K;F;^k>6wdzqv?!E6Me z(J&J4za%Vn0`?W`v(= zdtnQO%9DCXAKf?$?)zZ%e}=q|M!fPR)a0;A3)(ef9$v>RN7eY;juEvE=cEeVpTOv9 zSwB%s+=;J+!#6aZD`wgZe;a`59L|9ePaOWCpU74jO1`7Tt#O`g&WC58X)Ga zdW)`ow_`8LzG&VIhKok30)2jlmV2eH>&VhY{9sEhebfs1`rC7b-$23+GDwc^tC{7z zucF`n#tqideI*jM!llcfk$c&ExPL#X=vaJxhDpm> zWw%V@+KNu+^OO1yjja^ILw|}MoK!AlNyDuXQI+_4%wZs`A{*7Og>=vv%{w9nle6hp zd9xZvChs10d+6r-l35G0$3xK3!PptbA;+&-dj5EvnAX0nzeHhR>g8(4@V+)BLz|BCoLVvTv-e5YX+PyTKnz1^O#vq@BNK)N5fg6noXANK0AnPz zpptMjhZ?Tm+Q26k@(KQVvWNK+2sH&6mUGII-?%(%E^r1XXg10&>M^o(JyX?Cuh%RM zK*)999W~Z+XA`zJ$g@Hu7lySLG$lVs)`1J0PNRu(Uon$3(L9q^z7T>>ucQ=2`R4Kr zZEKiJumK3|4fp($IoTHXgk8&z`2{p~asf*1NVfj&e(5DlI-vb9+!M@f{Ud0b9ShO4!G{oAcy_VfGZujjCnP^Fb~An z1K6s|RdwpYRFZwNJf&pz=PZPsGX3I+W(1`ZnW;xT(B{@n{lqAG;LmQ0Qz~$>ukVPC z>)fB7Zc-b6?%XXSZ*adH3t)Su13^8BtXF)d-4{+^gjs2=bD+nxhZ9GY85F+kQ(FlF9^ z=)1eQSf1wLUO$~UCKoZjo<9f9T0Y&Hrwrz~Sn6{y6S6+Si6L2{HMVeX^eyastRwp! z#iDct+a7oPBHi!k!@kXWKJ?F0wA-J|t1KWWf?e$(^=G%BhAzz`W{2;R>u#mkNJ8f1 z=5p5E)>c{?z+`JjdGz9k*EzcpHW=n;#W_GVad(F^y@*R9CNo6nyLr43@{h0h_9s%- zG?>XNhsR#*ZEuNCloWOk8r|AESPB-69m{-syEH;5*ed29oCd+@%7tf#5vQKg;k z%hRnBePUZXI+RT=)^x&`$Gfkg7I#~d^5r8)APlLZ7~ek4P-Q07h#3c!Vj9llAEixM z`;vbFt?)=dNQf9v6+HTqcdu1Ga8;uH6t|LzEE4% z3X7q#ysn+E;3rvuN3HJ9xJ@`+UI?+ zKRY5Ve10>++y{|6jg@V47To$Md~ed4CqL%+(V(j@I)CP7Q96SA=y~<~TfXqcHysI? zz~;GAK?}AoeWuoadJP|A32VkxrtH_!0~2f2?f<^%1ZS1KT!=Az9*e1fsHv4g>nETV z*Q1P?!~Gg+S|>d)LPpt&%f4s$YwSKj+%7p_Ei*)ImKiQ$hMS5m#5PgC4=8Viy6v=a z=i@f(p^7xUs>#&S3SJy+;?bV>X>U4yp0RQ{Cb*H)7e_EFtd<}hg|@60e*MEJzOWd)e7GZV(%@mY zuPU-fSdU zn1RcVz*Hm^yjBa4Q4@I<#P2In7NN8&rT)2rkPyvCmJ+KE2DW467(73&$ewLxk||-; zBQi&TqYE51M08~n^k5If!^e{65TQxTM+9M>`{=+M_sIQ4>-kFQFBJUUVDcMr+)2}W zuRt7}e^5f{rWs!Gy`e&-tUu0X<)r96EYa`N=zhri(Ej{O5K=D5Lqz?x}-& z{a7&k4IW6q(G4x-`%`?5tW~)o`7T1Xf z)kgc*cZyJ>w3O;q9C|;}%z0*KEZ`mJqX~kw(JkxeCTf(K50H0XF44)CC{b}Zjv_5G z)so2__gg`|1Dz&%0wt}t=@L*fLDJ%TLUU*4%M2p8;Vi>Z@*(5Zr8@hk^o{n>xm9-c zXqMwegFVeHNXzwM5`6Tu%1ePRW~V^-eNji9jU76CfJ~|C>G0kXX)%2K9;tN03FY?D z(MKkx{2G-)S>%D*tW!54KGS_fa zeGgvlm~(Ml6AsUF)x{qGd-QQ02-lj_hSO|nTF$6~g!l6qS#1)uUCUpwfT-=#6-7n) z%bmCHICvFaMSEENs^1lAh#c20NtcQUrI*r^cg6mufm{^?t*LZAfqVw_c$)-W%-Hy4 zR8t(o(7|##-)JtoBrdZ+;u<%Z$|7T`#Oc@0I>xdA(HP%(6Ip?PQY#nrf8*jHYNm$l zOVcC(_T-JX^8G|jrm)7FhunQI3~`DTZa{R-i+1c+%g)5>yh#NgS-QzG5+)5h1>$7y z%Lylmc;VFwUBB~{!E;iwFZ94xMk6(`&*}&D!5<9UId;*+Lhm>(zzU>I^>TdizWtk$ zsg(xiOyV$|PkiO5g8MDtF+f~s45hk}>}o!}69HQ}=?1dpfRQ_y#pd@=5b^nY*BDDkIF6(av zgJg^+CtU)q@er*aQDWX{jYg8_aGQ!XydTarS@8sjcad|KX66Skx@dqN_w3a)Rr>}O zYLs7&@opK6Ii^WdNFPhO!I zV1$aBOaeqN{z|siYrwqqhG@l~Y?{M~*;2oKWXN^|z3=PuNZdy&x0s?FfaxOSD!pI4SH|4lZoiYF^8UBd->9?X|Z@J=*WPc0xjyN3hn z^Xo4LPu3A*D6jV|jO#0cpX(@8Ib6+SL#6I{0;Mwej%Sk$$Er-Xm!FYlt&T@erO FzX0qAjavW! literal 0 HcmV?d00001 diff --git a/crates/exav-unpack/tests/fixtures/zip/legacy_implode.zip b/crates/exav-unpack/tests/fixtures/zip/legacy_implode.zip new file mode 100644 index 0000000000000000000000000000000000000000..9ce50ed2a6d47d0ea4955e225c68e6489b45467a GIT binary patch literal 800 zcmWIWW@h1HU;{#yjcX$Es-`NfVPar#VPRn4WME)O%PcA`(JQGav9XWi{O8BS$7aWA z$I8KGSI@DZgHxW9gP%iRVn4Ge3zH}dGe4UMGYc!fGK&}oYdqV3efIxAaGw4Dem-7q z1|en^W)W8Hs^7n7b8#^Uu`n^SvQ_{2^QSTF`^F-7d;gjVyEWGTeYHQh^2T4i)90g0 z7uTB>HIU@N#~=sI)9G5U(r_{yJq9aX`}LVy0a;QK5wyk6_q)2y|7^w;11HE(}f|HJ1U{3qT{{CDB1)`54C zt74AU=BS>u_OF>LT3xsCS8qnZL8m76d#kih%jmB8CQ+%e?|GtmlEZwT-kVxa47q=F-@T(E;-RbSZ$C^cQ<~VvBPO&b{zh-?w2S*4 zR!xo4{hOU7aP$~!?a^6WG4VBbmR^0qmMpFvxAjGQrOzGvEm0?5P5${Nck7zD$;b0l z^g8)6-~90UC#M_XQ84du@_8%W(A8^Q!ioYEwo4~3J=HzIIpAWnox?>g86A}$X>V6A zUK29!=xVW#jJYEB9IiN*EQ?!f^uBUUmzVe!OaCV~*VvbQwbF0;^J>_WkU zobT@*iKnqDbEp4hfBN`U{~6=#1<_`E*JSm_xw^2)1b8ztaWUXd)WCEN1PWjhDTxPo Tv$BE27=h3nNM8o#0|o{F>=A7& literal 0 HcmV?d00001 diff --git a/crates/exav-unpack/tests/fixtures/zip/legacy_reduce.zip b/crates/exav-unpack/tests/fixtures/zip/legacy_reduce.zip new file mode 100644 index 0000000000000000000000000000000000000000..c413c87aef8083f5e872e6a71b883c01dff026f9 GIT binary patch literal 1058 zcmWIWW@h1HU|?Wn_`YFHL|)ZYrFF~<3@$7T44gpGw9KO761|d&5+EBBI52Qv3KB0O zz{tcbh@_ll3W%f7;K1Z0l@j2#5!t%O|vP&pUnk=COnyNdYVr6lZKW6 z>Fp;kw*`6#%+%CkKP2Tc%Tk#^Y~3 z4S09k`r!X-e1=ngfAf3L61hF`T(qSG_vYIMYa|pTOj{ETWn~VY;Au{qv}>JAF4M{d zW{DCq%3c@t%@r2eI6LgzmNznoIKA0kNiisWu=Lnj%GaL1(D}q^kqe2JA52hpIT0hK z=dmgA_O6YtGE$t9Wxi);nVt@Ek$HB7Pq0!>j=%TlvXh1%JegxGb2M+W6>y~3DNjq? z=)pUeV>9phDS2||7AX2YbGVqnVyQMmXVRp;2NTj{lDVraUz)Hp78EwL9G6r*p%i@S z$k~(=%6tOCd*b#EuK5ihkGW&-`@Ky4hA>3=axG>(=FzmE>4l7N8$0L;N#kN^Mx literal 0 HcmV?d00001 diff --git a/crates/exav-unpack/tests/fixtures/zip/legacy_shrink.zip b/crates/exav-unpack/tests/fixtures/zip/legacy_shrink.zip new file mode 100644 index 0000000000000000000000000000000000000000..efaf97077e89bbbc762199dedbdb1d54d4591200 GIT binary patch literal 825 zcmWIWW@h1HU|?Wm;NGw%BCl$y(orS`1{W3v22P-;n`cmPh+arUNXUgLjEwERGi(&@ z&0uU)@sT!5ebL}y%|6h1>jQt>qM-wtXr z-0aHL+yXuuPDNI^o6PE5JoB0$^KG?*CdqQ{=Q@`yyL?FG=ChlMd;&|=Rjl_Z=E+pL z&MSJdM^nypp_`?`)%z#CZ}`TZU_Q_DROd!w(zDbZ)%laYYI%H8INNNXyl3NavkBL? zosBGAcEK>CWA;4bfR?#C`|E{TB$nyq%+KrJB&oSv&D%qdORXL_71lnh1s}By3q&88M1$B0dZm!u08Ogz3Ub!N(%8C(aygcTO=xpIt8;YpxY zK*ZX@+YW4JEnYiVzEzzNP?EtW+~!$z@}Y$DQWdp5mQz2RvRE?l%0aiHlvK9W zDduO@RO%G2G~Qff>ity8aphUY{C(!B&%z=Vj Vec { // rather than silently truncating and reporting `unsupported`. limits.max_buffer_bytes = limits.max_buffer_bytes.max(4 * 1024 * 1024); let mut b = Budget::new(limits); - extract(Format::Alz, data, &mut b).expect("ALZ extraction must not error") + extract(Format::Alz, &data, &mut b).expect("ALZ extraction must not error") } #[test] @@ -69,7 +69,10 @@ fn every_member_matches_the_oracle_byte_for_byte() { e.unsupported ); assert_eq!(e.data.len(), *size, "{name}: wrong decompressed length"); - let got = format!("{:x}", Sha256::digest(&e.data)); + let got: String = Sha256::digest(&e.data) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); assert!( got.starts_with(digest), "{name}: content differs from `unalz` output (sha256 {got}, expected {digest}…). \ diff --git a/crates/exav-unpack/tests/suites/arc.rs b/crates/exav-unpack/tests/suites/arc.rs index 9b4759ce..9a0cfa7a 100644 --- a/crates/exav-unpack/tests/suites/arc.rs +++ b/crates/exav-unpack/tests/suites/arc.rs @@ -19,7 +19,8 @@ //! arc as stored.arc eicar.com # -s suppresses compression //! ``` -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; const EICAR: &str = "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"; const TEXT: &str = "080d15586f0165e6fe26ccc32060051fb67344c54691e3bca8ea7871b2c5d23f"; diff --git a/crates/exav-unpack/tests/suites/arj_never_silent.rs b/crates/exav-unpack/tests/suites/arj_never_silent.rs index 5abcd13d..e1d76a6a 100644 --- a/crates/exav-unpack/tests/suites/arj_never_silent.rs +++ b/crates/exav-unpack/tests/suites/arj_never_silent.rs @@ -9,7 +9,8 @@ //! so the surrounding structure stays valid and the test exercises the //! path it claims to. -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; fn sample() -> Vec { let p = format!("{}/tests/fixtures/sample.arj", env!("CARGO_MANIFEST_DIR")); diff --git a/crates/exav-unpack/tests/suites/bzip2_blocks.rs b/crates/exav-unpack/tests/suites/bzip2_blocks.rs new file mode 100644 index 00000000..77b9d5c8 --- /dev/null +++ b/crates/exav-unpack/tests/suites/bzip2_blocks.rs @@ -0,0 +1,40 @@ +//! A bzip2 block of incompressible data compresses to more bytes than the block +//! holds. The decoder must still read it whole, not stop at the block size. + +use std::io::Write; + +use exav_unpack::{extract, Budget, Format, Limits}; + +/// Bytes no compressor can shrink, from a fixed seed. +fn noise(len: usize) -> Vec { + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 24) as u8 + }) + .collect() +} + +fn compress(data: &[u8], level: u32) -> Vec { + let mut enc = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::new(level)); + enc.write_all(data).unwrap(); + enc.finish().unwrap() +} + +#[test] +fn incompressible_blocks_are_decoded() { + // Level 1 is 100k blocks, so this is two full blocks and a partial one. + let data = noise(250_000); + for level in [1, 9] { + let bz = compress(&data, level); + assert!(bz.len() > data.len(), "the input must not compress"); + let mut budget = Budget::new(Limits::default()); + let out = extract(Format::Bzip2, &bz, &mut budget) + .unwrap_or_else(|e| panic!("level {level}: {e:?}")); + assert_eq!(out.len(), 1, "level {level}"); + assert!(out[0].data == data, "level {level}: decoded bytes differ"); + } +} diff --git a/crates/exav-unpack/tests/suites/chm.rs b/crates/exav-unpack/tests/suites/chm.rs index a00a11d7..86f0e4b3 100644 --- a/crates/exav-unpack/tests/suites/chm.rs +++ b/crates/exav-unpack/tests/suites/chm.rs @@ -85,7 +85,7 @@ fn truncated_head_does_not_panic() { }; let head = &data[..64.min(data.len())]; let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Chm, head, &mut budget).unwrap(); + let entries = extract(Format::Chm, &head, &mut budget).unwrap(); assert!(entries.is_empty(), "{name}: truncated head yielded members"); } } diff --git a/crates/exav-unpack/tests/suites/cpio_never_silent.rs b/crates/exav-unpack/tests/suites/cpio_never_silent.rs index dc9b94eb..1b2d028d 100644 --- a/crates/exav-unpack/tests/suites/cpio_never_silent.rs +++ b/crates/exav-unpack/tests/suites/cpio_never_silent.rs @@ -6,7 +6,8 @@ //! and a short member gets scanned and found clean on content that is not //! there. Three cpio header flavours share the bug and the fix. -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; /// A `newc`-format cpio header. All fields are 8-char hex, ASCII. fn newc(name: &str, filesize: u32) -> Vec { diff --git a/crates/exav-unpack/tests/suites/decrypt_disabled.rs b/crates/exav-unpack/tests/suites/decrypt_disabled.rs index e9b7f47b..7b83489d 100644 --- a/crates/exav-unpack/tests/suites/decrypt_disabled.rs +++ b/crates/exav-unpack/tests/suites/decrypt_disabled.rs @@ -13,7 +13,7 @@ fn encrypted_zip_is_unsupported_without_decrypt() { &include_bytes!("../fixtures/encrypted/zip_zipcrypto_store.zip")[..], ] { let mut budget = Budget::with_passwords(Limits::default(), vec!["exavtest".into()]); - let entries = extract(Format::Zip, data, &mut budget).unwrap(); + let entries = extract(Format::Zip, &data, &mut budget).unwrap(); assert!( entries.iter().any(|e| e.encrypted && e.unsupported.is_some()), "encrypted member must be reported unsupported (not decrypted) without the decrypt feature", diff --git a/crates/exav-unpack/tests/suites/disabled_formats.rs b/crates/exav-unpack/tests/suites/disabled_formats.rs index be86ccfa..6a02b1ce 100644 --- a/crates/exav-unpack/tests/suites/disabled_formats.rs +++ b/crates/exav-unpack/tests/suites/disabled_formats.rs @@ -98,7 +98,7 @@ fn ordinary_files_are_not_claimed() { &b"{\"json\": true}"[..], ] { assert_eq!( - exav_unpack::detect(blob), + exav_unpack::detect(&blob), None, "claimed an ordinary file: {blob:x?}" ); diff --git a/crates/exav-unpack/tests/suites/diskimage.rs b/crates/exav-unpack/tests/suites/diskimage.rs index beeaa7a1..9e4e118a 100644 --- a/crates/exav-unpack/tests/suites/diskimage.rs +++ b/crates/exav-unpack/tests/suites/diskimage.rs @@ -26,7 +26,8 @@ //! The VHDX is stored gzipped only because its 1 MiB-aligned regions make even a //! 256 KiB disk a 9 MiB file; the bytes inside are qemu-img's, unmodified. -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; /// `sha256sum` of the 256 KiB raw image the fixtures were built from, which is /// also what `qemu-img convert -O raw` returns for each of them. diff --git a/crates/exav-unpack/tests/suites/egg.rs b/crates/exav-unpack/tests/suites/egg.rs index 611ffb6b..d24e2f95 100644 --- a/crates/exav-unpack/tests/suites/egg.rs +++ b/crates/exav-unpack/tests/suites/egg.rs @@ -10,7 +10,8 @@ #![cfg(feature = "egg")] -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; fn members(blob: &[u8]) -> Vec { let mut budget = Budget::new(Limits::default()); @@ -56,7 +57,7 @@ fn corpus() -> Vec<(&'static str, &'static [u8])> { fn every_fixture_is_recognised_as_egg() { for (name, blob) in corpus() { assert_eq!( - exav_unpack::detect(blob), + exav_unpack::detect(&blob), Some(Format::Egg), "{name} was not recognised" ); diff --git a/crates/exav-unpack/tests/suites/entry_accounting.rs b/crates/exav-unpack/tests/suites/entry_accounting.rs index 2389e70a..26320485 100644 --- a/crates/exav-unpack/tests/suites/entry_accounting.rs +++ b/crates/exav-unpack/tests/suites/entry_accounting.rs @@ -18,7 +18,7 @@ fn hits_limit(blob: &[u8], fmt: Format, cap: u64) -> bool { let mut limits = Limits::default(); limits.max_members = cap; let mut budget = Budget::new(limits); - extract(fmt, blob, &mut budget).is_err() + extract(fmt, &blob, &mut budget).is_err() } fn tar_of(members: &[(&str, Vec)]) -> Vec { diff --git a/crates/exav-unpack/tests/suites/ext.rs b/crates/exav-unpack/tests/suites/ext.rs index d4bdd963..a6d05eb3 100644 --- a/crates/exav-unpack/tests/suites/ext.rs +++ b/crates/exav-unpack/tests/suites/ext.rs @@ -35,7 +35,8 @@ //! gzip -9 frag.img //! ``` -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; /// `sha256sum` of the ZIP that was written in, which is what following the /// extent tree has to reproduce. diff --git a/crates/exav-unpack/tests/suites/fat.rs b/crates/exav-unpack/tests/suites/fat.rs index 2e0b46a3..703285ec 100644 --- a/crates/exav-unpack/tests/suites/fat.rs +++ b/crates/exav-unpack/tests/suites/fat.rs @@ -23,7 +23,8 @@ //! gzip -9 fragmented.img //! ``` -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; /// `sha256sum` of the ZIP that was copied in, which is what following the /// cluster chain has to reproduce. diff --git a/crates/exav-unpack/tests/suites/hwp3.rs b/crates/exav-unpack/tests/suites/hwp3.rs index 994f2149..41dd086d 100644 --- a/crates/exav-unpack/tests/suites/hwp3.rs +++ b/crates/exav-unpack/tests/suites/hwp3.rs @@ -13,7 +13,8 @@ #![cfg(feature = "hwp3")] -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; const REAL: &[u8] = include_bytes!("../fixtures/hwp3/tika_testHWP_3.0.hwp"); @@ -34,7 +35,7 @@ fn members(blob: &[u8]) -> Vec { #[test] fn a_real_document_is_recognised() { - assert_eq!(exav_unpack::detect(REAL), Some(Format::Hwp3)); + assert_eq!(exav_unpack::detect(&REAL), Some(Format::Hwp3)); } #[test] diff --git a/crates/exav-unpack/tests/suites/ishield_z.rs b/crates/exav-unpack/tests/suites/ishield_z.rs index 80470639..cdbdaea3 100644 --- a/crates/exav-unpack/tests/suites/ishield_z.rs +++ b/crates/exav-unpack/tests/suites/ishield_z.rs @@ -12,7 +12,8 @@ //! //! Fixtures are the `unshield` project's own examples (MIT); see `NOTICE`. -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; fn fixture(name: &str) -> Vec { let p = format!( diff --git a/crates/exav-unpack/tests/suites/iso_never_silent.rs b/crates/exav-unpack/tests/suites/iso_never_silent.rs index 034eec4a..128e9ec6 100644 --- a/crates/exav-unpack/tests/suites/iso_never_silent.rs +++ b/crates/exav-unpack/tests/suites/iso_never_silent.rs @@ -46,7 +46,7 @@ fn iso(files: &[(&str, u64, u64)], root_len: u64) -> Vec { fn entries(blob: &[u8]) -> Vec { let mut b = Budget::new(Limits::default()); - extract(Format::Iso, blob, &mut b).unwrap_or_default() + extract(Format::Iso, &blob, &mut b).unwrap_or_default() } #[test] @@ -104,18 +104,18 @@ fn a_corrupt_directory_record_is_reported() { #[test] fn a_corrupt_directory_record_is_reported_when_streamed() { - // The scanner walks every ISO through `stream_members`, where the same - // record used to end the directory without a word. + // The scanner walks every ISO through `walk`, where the same record used + // to end the directory without a word. let mut img = iso(&[("A.TXT", 19, 4)], 200); let p = 18 * SECTOR + 33 + "A.TXT".len(); img[p] = 5; let mut unsupported = Vec::new(); let mut b = Budget::new(Limits::default()); - let _ = exav_unpack::stream_members( + let _ = exav_unpack::walk( Format::Iso, - std::io::Cursor::new(img), + &img, &mut b, - &mut |m: &exav_unpack::MemberMeta, _: Option<&mut dyn std::io::Read>, _: &mut Budget| { + &mut |m: &exav_unpack::MemberMeta, _: Option>, _: &mut Budget| { unsupported.extend(m.unsupported); None::<()> }, diff --git a/crates/exav-unpack/tests/suites/lha_header_bounds.rs b/crates/exav-unpack/tests/suites/lha_header_bounds.rs index 8b190ec3..3325490c 100644 --- a/crates/exav-unpack/tests/suites/lha_header_bounds.rs +++ b/crates/exav-unpack/tests/suites/lha_header_bounds.rs @@ -41,7 +41,7 @@ fn the_overflow_reproducers_do_not_reach_the_decoder_unchecked() { let mut budget = Budget::new(Limits::default()); // Either the bounds check refuses it or the decoder reports it; what // must never happen is a panic escaping, or a silent empty success. - let _ = extract(Format::Lha, data, &mut budget); + let _ = extract(Format::Lha, &data, &mut budget); } } diff --git a/crates/exav-unpack/tests/suites/lz4.rs b/crates/exav-unpack/tests/suites/lz4.rs index c76c0a9c..ab953a1d 100644 --- a/crates/exav-unpack/tests/suites/lz4.rs +++ b/crates/exav-unpack/tests/suites/lz4.rs @@ -17,7 +17,8 @@ //! cat one.lz4 eicar.lz4 > two_frames.lz4 //! ``` -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; /// `sha256sum` of what `lz4 -dc` writes for each fixture. const ONE: &str = "30c2abfccdf15a28990bae7bb0efa2444d82af3ce7053ff493956ef9612f775d"; diff --git a/crates/exav-unpack/tests/suites/lzw.rs b/crates/exav-unpack/tests/suites/lzw.rs index 71445e1c..3d2e6c48 100644 --- a/crates/exav-unpack/tests/suites/lzw.rs +++ b/crates/exav-unpack/tests/suites/lzw.rs @@ -15,7 +15,8 @@ //! done //! ``` -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; fn fixture(name: &str) -> Vec { let p = format!("{}/tests/fixtures/lzw/{name}", env!("CARGO_MANIFEST_DIR")); @@ -136,22 +137,13 @@ fn a_truncated_stream_keeps_what_decoded() { #[test] fn an_oversized_stream_is_reported_not_truncated_silently() { - // Decoding past the per-member budget must surface rather than hand back a - // silently shortened member. + // Holding a member larger than the buffer limit must fail as a limit rather + // than hand back a silently shortened member. let z = fixture("bigreset.b16.Z"); let mut limits = Limits::default(); limits.max_buffer_bytes = 4096; let mut b = Budget::new(limits); - let mut out = Vec::new(); - let _ = extract_each(Format::Lzw, &z, &mut b, &mut |e: Entry, _: &mut Budget| { - out.push(e); - None::<()> - }); - assert!( - out.iter().any(|x| x.unsupported.is_some()), - "an over-budget .Z must be reported, got {:?}", - out.iter() - .map(|x| (&x.name, x.unsupported, x.data.len())) - .collect::>() - ); + let err = exav_unpack::extract(Format::Lzw, &z.as_slice(), &mut b) + .expect_err("an over-budget .Z must be reported"); + assert!(!err.is_corrupt(), "a limit, not corruption: {err:?}"); } diff --git a/crates/exav-unpack/tests/suites/mod.rs b/crates/exav-unpack/tests/suites/mod.rs index cf8f7ba6..ac87e7b7 100644 --- a/crates/exav-unpack/tests/suites/mod.rs +++ b/crates/exav-unpack/tests/suites/mod.rs @@ -19,6 +19,8 @@ mod arc; #[cfg(feature = "arj")] mod arj_never_silent; mod broken_media; +#[cfg(feature = "bzip2")] +mod bzip2_blocks; #[cfg(feature = "cab")] mod cab_quantum; #[cfg(feature = "chm")] @@ -111,3 +113,36 @@ mod zip_disguised_members; mod zip_overlap; #[cfg(feature = "zoo")] mod zoo; + +use exav_unpack::{walk, Budget, Entry, Format, LimitHit, Member}; + +/// Every member of `data` as an [`Entry`], a streamed one read into memory, +/// handed to `visit` until it returns `Some`. Unlike `exav_unpack::extract`, +/// the members visited before an error are seen, for the suites that check +/// what a damaged container still yields. +pub(crate) fn extract_each( + fmt: Format, + data: &[u8], + budget: &mut Budget, + visit: &mut dyn FnMut(Entry, &mut Budget) -> Option, +) -> Result, LimitHit> { + walk(fmt, &data, budget, &mut |meta, content, budget| { + let data = match content { + None => Vec::new(), + Some(Member::Bytes(bytes)) => bytes, + Some(Member::Stream(reader)) => { + let mut bytes = Vec::new(); + let _ = reader.read_to_end(&mut bytes); + bytes + } + }; + let entry = Entry { + name: meta.name.clone(), + data, + comp_size: meta.comp_size, + encrypted: meta.encrypted, + unsupported: meta.unsupported, + }; + visit(entry, budget) + }) +} diff --git a/crates/exav-unpack/tests/suites/nsis.rs b/crates/exav-unpack/tests/suites/nsis.rs index 244c15bb..99b2342b 100644 --- a/crates/exav-unpack/tests/suites/nsis.rs +++ b/crates/exav-unpack/tests/suites/nsis.rs @@ -83,7 +83,7 @@ fn truncated_and_garbage_do_not_panic() { // the gitignored sample is present locally. if let Some(data) = real_nsis() { let truncated = &data[..data.len() / 3]; - let _ = extract(Format::Nsis, truncated, &mut budget); + let _ = extract(Format::Nsis, &truncated, &mut budget); } // Signature with no valid firstheader following. diff --git a/crates/exav-unpack/tests/suites/ntfs.rs b/crates/exav-unpack/tests/suites/ntfs.rs index cc28af7e..77d1e631 100644 --- a/crates/exav-unpack/tests/suites/ntfs.rs +++ b/crates/exav-unpack/tests/suites/ntfs.rs @@ -21,7 +21,8 @@ //! # then split payload.zip's single run in two and re-apply the MFT fix-ups //! ``` -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; /// `sha256sum` of what `ntfscat` returns for each file. const PAYLOAD: &str = "49c7e4f4b184954a212c78c87c17705cd4c16f0b8c83a5cd6a750369c47957c5"; diff --git a/crates/exav-unpack/tests/suites/panic_containment.rs b/crates/exav-unpack/tests/suites/panic_containment.rs index 99022c5e..00a749fa 100644 --- a/crates/exav-unpack/tests/suites/panic_containment.rs +++ b/crates/exav-unpack/tests/suites/panic_containment.rs @@ -1,6 +1,6 @@ //! Hostile-input panic containment: a malformed container must never panic the -//! process. Third-party decoders can panic on crafted input, so `extract_each` -//! wraps format dispatch in a catch-unwind boundary that turns any such panic +//! process. Third-party decoders can panic on crafted input, so `walk` wraps +//! format dispatch in a catch-unwind boundary that turns any such panic //! into a clean `LimitHit`. use exav_unpack::{extract, Budget, Format, Limits}; @@ -57,7 +57,7 @@ fn malformed_iso_short_record_does_not_panic() { &include_bytes!("../fixtures/iso_short_record2.iso")[..], ] { let mut budget = Budget::new(Limits::default()); - let _ = extract(Format::Iso, data, &mut budget); + let _ = extract(Format::Iso, &data, &mut budget); } } @@ -78,7 +78,7 @@ fn malformed_upx_nrv_gamma_does_not_panic() { /// Regression for fuzz-found crash (2026-07-01): a truncated CAB with LZX /// compressed data triggers an index-out-of-bounds in `lzxd`'s bitstream /// reader (`bitstream.rs:37` — `self.buffer[1]` when buffer has 1 byte). -/// Must not panic; the `catch_unwind` boundary in `extract_each` turns it +/// Must not panic; the `catch_unwind` boundary in `walk` turns it /// into a clean `LimitHit`. #[test] fn cab_lzxd_bitstream_oob_does_not_panic() { @@ -103,31 +103,7 @@ fn lha_delharc_overflow_does_not_panic() { &include_bytes!("../fixtures/lha_delharc_overflow_min.lha")[..], ] { let mut budget = Budget::new(Limits::default()); - let _ = extract(Format::Lha, data, &mut budget); - } -} - -/// Same crafted LHA headers, but driven through the **streaming** API -/// (`stream_members`) — which has its own `catch_unwind` boundary. LHA is -/// stream-extractable, so the delharc panic must be contained here too. -#[test] -fn lha_delharc_overflow_stream_does_not_panic() { - use exav_unpack::{stream_members, MemberMeta}; - use std::io::{Cursor, Read}; - for data in [ - &include_bytes!("../fixtures/lha_delharc_overflow.lha")[..], - &include_bytes!("../fixtures/lha_delharc_overflow_min.lha")[..], - ] { - let mut budget = Budget::new(Limits::default()); - let mut visit = - |_m: &MemberMeta, r: Option<&mut dyn Read>, _b: &mut Budget| -> Option<()> { - if let Some(r) = r { - let mut sink = Vec::new(); - let _ = r.read_to_end(&mut sink); - } - None - }; - let _ = stream_members(Format::Lha, Cursor::new(data), &mut budget, &mut visit); + let _ = extract(Format::Lha, &data, &mut budget); } } @@ -145,7 +121,7 @@ fn lha_delharc_oom_is_bounded() { /// /// Every other test here uses an input that once panicked a particular decoder. /// Those are real regressions and worth keeping, but they do not test -/// `extract_each`'s `catch_unwind`: fix all of those decoders and they keep +/// `walk`'s `catch_unwind`: fix all of those decoders and they keep /// passing with the boundary deleted. This panics from inside the dispatch for /// input nothing else produces, so the only thing that can turn it into a /// `LimitHit` is the boundary. diff --git a/crates/exav-unpack/tests/suites/partition_never_silent.rs b/crates/exav-unpack/tests/suites/partition_never_silent.rs index 0224b6d9..212d8a9b 100644 --- a/crates/exav-unpack/tests/suites/partition_never_silent.rs +++ b/crates/exav-unpack/tests/suites/partition_never_silent.rs @@ -7,7 +7,8 @@ //! //! Each case here was a silent `break`/`return Ok(None)` before. -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; const SECTOR: usize = 512; const ENTRY_SIZE: u32 = 128; diff --git a/crates/exav-unpack/tests/suites/pdf_parse.rs b/crates/exav-unpack/tests/suites/pdf_parse.rs index 88b26aba..e4807cba 100644 --- a/crates/exav-unpack/tests/suites/pdf_parse.rs +++ b/crates/exav-unpack/tests/suites/pdf_parse.rs @@ -131,7 +131,7 @@ fn ascii_hex_stream_decoded() { trailer\n<< /Size 2 /Root 1 0 R >>\n%%EOF\n" ); let mut budget = Budget::new(Limits::default()); - let entries = extract(Format::Pdf, pdf.as_bytes(), &mut budget).unwrap(); + let entries = extract(Format::Pdf, &pdf.as_bytes(), &mut budget).unwrap(); assert!( entries .iter() diff --git a/crates/exav-unpack/tests/suites/pe_emulation.rs b/crates/exav-unpack/tests/suites/pe_emulation.rs index eb2ab755..30cd629c 100644 --- a/crates/exav-unpack/tests/suites/pe_emulation.rs +++ b/crates/exav-unpack/tests/suites/pe_emulation.rs @@ -103,7 +103,7 @@ fn unpack(file: &[u8]) -> Vec { limits.max_extracted_bytes = 1 << 30; limits.max_buffer_bytes = 1 << 30; let mut b = Budget::new(limits); - extract(Format::PePacked, file, &mut b).expect("extraction stays within budget") + extract(Format::PePacked, &file, &mut b).expect("extraction stays within budget") } #[test] diff --git a/crates/exav-unpack/tests/suites/pe_emulation_corpus.rs b/crates/exav-unpack/tests/suites/pe_emulation_corpus.rs index ee7c0c8b..d0b4c895 100644 --- a/crates/exav-unpack/tests/suites/pe_emulation_corpus.rs +++ b/crates/exav-unpack/tests/suites/pe_emulation_corpus.rs @@ -70,7 +70,7 @@ fn largest_recovered(file: &[u8]) -> usize { limits.max_extracted_bytes = 1 << 30; limits.max_buffer_bytes = 1 << 30; let mut b = Budget::new(limits); - match extract(Format::PePacked, file, &mut b) { + match extract(Format::PePacked, &file, &mut b) { Ok(entries) => entries.iter().map(|e| e.data.len()).max().unwrap_or(0), Err(_) => 0, } diff --git a/crates/exav-unpack/tests/suites/pe_emulation_techniques.rs b/crates/exav-unpack/tests/suites/pe_emulation_techniques.rs index 01a40644..98c8499f 100644 --- a/crates/exav-unpack/tests/suites/pe_emulation_techniques.rs +++ b/crates/exav-unpack/tests/suites/pe_emulation_techniques.rs @@ -171,7 +171,7 @@ fn unpack(file: &[u8]) -> Vec { limits.max_extracted_bytes = 1 << 30; limits.max_buffer_bytes = 1 << 30; let mut b = Budget::new(limits); - extract(Format::PePacked, file, &mut b).expect("extraction stays within budget") + extract(Format::PePacked, &file, &mut b).expect("extraction stays within budget") } fn recovered(entries: &[Entry]) -> bool { diff --git a/crates/exav-unpack/tests/suites/ppt_embedded_storage.rs b/crates/exav-unpack/tests/suites/ppt_embedded_storage.rs index ef7312b7..12d34c52 100644 --- a/crates/exav-unpack/tests/suites/ppt_embedded_storage.rs +++ b/crates/exav-unpack/tests/suites/ppt_embedded_storage.rs @@ -65,7 +65,7 @@ fn ppt_with(records: &[u8]) -> Vec { fn members(blob: &[u8]) -> Vec { let mut budget = Budget::new(Limits::default()); - extract(Format::Ole, blob, &mut budget).expect("ppt extracts") + extract(Format::Ole, &blob, &mut budget).expect("ppt extracts") } fn has_marker(entries: &[exav_unpack::Entry]) -> bool { diff --git a/crates/exav-unpack/tests/suites/protectors.rs b/crates/exav-unpack/tests/suites/protectors.rs index 5a354f75..1be21291 100644 --- a/crates/exav-unpack/tests/suites/protectors.rs +++ b/crates/exav-unpack/tests/suites/protectors.rs @@ -75,7 +75,7 @@ fn pe_with_sections(names: &[&[u8]]) -> Vec { /// through its own content-aware routing. A library caller has to name it. fn reported_reason(pe: &[u8]) -> Option<&'static str> { let mut b = Budget::new(Limits::default()); - let entries = extract(Format::PePacked, pe, &mut b).ok()?; + let entries = extract(Format::PePacked, &pe, &mut b).ok()?; entries.into_iter().find_map(|e| e.unsupported) } diff --git a/crates/exav-unpack/tests/suites/rar_solid.rs b/crates/exav-unpack/tests/suites/rar_solid.rs index 76531bae..ad249883 100644 --- a/crates/exav-unpack/tests/suites/rar_solid.rs +++ b/crates/exav-unpack/tests/suites/rar_solid.rs @@ -29,7 +29,8 @@ //! rar a -ma4 -v4k -m3 -ep vol_rar4.rar one.txt eicar.com //! ``` -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; /// `sha256sum` of the files that went into the archives. const ONE: &str = "30c2abfccdf15a28990bae7bb0efa2444d82af3ce7053ff493956ef9612f775d"; diff --git a/crates/exav-unpack/tests/suites/sevenz_bcj2.rs b/crates/exav-unpack/tests/suites/sevenz_bcj2.rs index 2034517b..50bcda21 100644 --- a/crates/exav-unpack/tests/suites/sevenz_bcj2.rs +++ b/crates/exav-unpack/tests/suites/sevenz_bcj2.rs @@ -14,7 +14,8 @@ //! 7zz a -t7z -mx=9 -mf=BCJ2 bcj2.7z sample.bin eicar.txt //! ``` -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; /// `sha256sum` of the members as extracted by `7zz x`. const SAMPLE_SHA256: &str = "fb051a6c58961fb6aec758acf50d7d0375eecff740d0706388092829e3fb7a41"; diff --git a/crates/exav-unpack/tests/suites/silent_skips.rs b/crates/exav-unpack/tests/suites/silent_skips.rs index 7c0d1152..b427f3cf 100644 --- a/crates/exav-unpack/tests/suites/silent_skips.rs +++ b/crates/exav-unpack/tests/suites/silent_skips.rs @@ -1,7 +1,7 @@ //! Regressions for members that existed but produced nothing. //! //! Each case here was a `continue` that dropped a member the container's own -//! directory names. All are the same class — the bytes are in the file, exav +//! directory names. All are the same class: the bytes are in the file, exav //! failed to read them, so the result must say so rather than come back clean. //! `CONTRIBUTING.md` sets out the classification these are judged against. //! @@ -9,7 +9,8 @@ //! declares bytes the file does not contain, must stay quiet. `ordinary_*` tests //! elsewhere pin that side. -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; /// Every member `extract_each` produces, as `(name, unsupported-reason)`. /// `extract` is not usable here: it discards already-emitted entries when the @@ -139,3 +140,86 @@ fn a_cab_member_beyond_its_folder_data_is_reported() { "and it must carry a reason rather than read as empty: {m:?}" ); } + +/// A one-folder, uncompressed cabinet holding `payload`, whose directory lists +/// `files` as `(uoffFolderStart, size, NUL-terminated name)`. +#[cfg(feature = "cab")] +fn cab_with_files(payload: &[u8], files: &[(u32, u32, Vec)]) -> Vec { + let files_off = 36 + 8; + let data_off = files_off + files.iter().map(|f| 16 + f.2.len()).sum::(); + let mut blob = Vec::new(); + blob.extend_from_slice(b"MSCF"); + blob.extend_from_slice(&0u32.to_le_bytes()); // reserved1 + blob.extend_from_slice(&0u32.to_le_bytes()); // cbCabinet (patched below) + blob.extend_from_slice(&0u32.to_le_bytes()); // reserved2 + blob.extend_from_slice(&(files_off as u32).to_le_bytes()); // coffFiles + blob.extend_from_slice(&0u32.to_le_bytes()); // reserved3 + blob.extend_from_slice(&[3, 1]); // version + blob.extend_from_slice(&1u16.to_le_bytes()); // cFolders + blob.extend_from_slice(&(files.len() as u16).to_le_bytes()); // cFiles + blob.extend_from_slice(&[0; 6]); // flags, setID, iCabinet + blob.extend_from_slice(&(data_off as u32).to_le_bytes()); // coffCabStart + blob.extend_from_slice(&1u16.to_le_bytes()); // cCFData + blob.extend_from_slice(&0u16.to_le_bytes()); // typeCompress = none + for (off, size, name) in files { + blob.extend_from_slice(&size.to_le_bytes()); + blob.extend_from_slice(&off.to_le_bytes()); + blob.extend_from_slice(&[0; 8]); // iFolder, date, time, attribs + blob.extend_from_slice(name); + } + blob.extend_from_slice(&0u32.to_le_bytes()); // csum + blob.extend_from_slice(&(payload.len() as u16).to_le_bytes()); // cbData + blob.extend_from_slice(&(payload.len() as u16).to_le_bytes()); // cbUncomp + blob.extend_from_slice(payload); + let total = blob.len() as u32; + blob[8..12].copy_from_slice(&total.to_le_bytes()); + assert_eq!(exav_unpack::detect(&blob), Some(Format::Cab)); + blob +} + +/// Two CAB members may share their bytes: MSI cabinets list a file twice at one +/// offset when it is installed under two names, as ScreenConnect installers do. +/// A member that starts before the end of the previous one is read again from +/// the folder's start, not reported unreadable. +#[cfg(feature = "cab")] +#[test] +fn cab_members_that_share_bytes_are_all_read() { + let files = [ + (0, 11, b"first\0".to_vec()), + (0, 11, b"same_bytes\0".to_vec()), + (6, 5, b"overlap\0".to_vec()), + ]; + let blob = cab_with_files(b"hello world", &files); + let mut budget = Budget::new(Limits::default()); + let mut got = Vec::new(); + let _ = extract_each::<()>(Format::Cab, &blob, &mut budget, &mut |e: Entry, _| { + got.push((e.name, e.unsupported, e.data)); + None + }); + let want: Vec<(String, Option<&str>, Vec)> = vec![ + ("first".into(), None, b"hello world".to_vec()), + ("same_bytes".into(), None, b"hello world".to_vec()), + ("overlap".into(), None, b"world".to_vec()), + ]; + assert_eq!(got, want); +} + +/// Each restart decodes the folder again up to the member, so a directory that +/// lists the same late member thousands of times is quadratic work unless it is +/// charged. It is, to the scan budget, and the walk stops as a limit. +#[cfg(feature = "cab")] +#[test] +fn cab_restarts_are_charged_to_the_scan_budget() { + let payload = vec![b'x'; 1000]; + let files: Vec<_> = (0..200) + .map(|i| (990, 10, format!("m{i}\0").into_bytes())) + .collect(); + let blob = cab_with_files(&payload, &files); + let mut limits = Limits::default(); + limits.max_scanned_bytes = 50_000; + let mut budget = Budget::new(limits); + let r = extract_each::<()>(Format::Cab, &blob, &mut budget, &mut |_, _| None); + // Uncharged, the walk would read 200 × 10 bytes and succeed. + let err = r.expect_err("200 restarts at offset 990 exceed a 50,000-byte budget"); + assert!(err.reason.contains("scan budget"), "{err:?}"); +} diff --git a/crates/exav-unpack/tests/suites/stream_read_errors.rs b/crates/exav-unpack/tests/suites/stream_read_errors.rs index d2264172..e1e7a09b 100644 --- a/crates/exav-unpack/tests/suites/stream_read_errors.rs +++ b/crates/exav-unpack/tests/suites/stream_read_errors.rs @@ -1,10 +1,11 @@ //! A source that fails part way is reported, never read as a short file. //! -//! The streaming walk reads its container through `Read + Seek`, and for a -//! network range reader a failed read is an ordinary event. Taking it for end -//! of file drops every member past that point without a word. +//! The walk reads its container through a `ByteSource`, and for a network range +//! reader a failed read is an ordinary event. Taking it for end of file drops +//! every member past that point without a word. -use exav_unpack::{is_streamable, stream_members, Budget, Format, Limits, MemberMeta}; +use exav_unpack::source::BlockCache; +use exav_unpack::{walk, Budget, Format, Limits, Member}; use std::io::{self, Read, Seek, SeekFrom}; /// Seeks anywhere in `len` bytes; every read fails. @@ -31,7 +32,7 @@ impl Seek for Unreadable { } #[test] -fn a_failing_source_is_reported_by_every_streamed_format() { +fn a_failing_source_is_reported_by_every_format() { let formats = [ Format::Gzip, Format::Tar, @@ -54,19 +55,26 @@ fn a_failing_source_is_reported_by_every_streamed_format() { Format::OneNote, Format::Swf, Format::Szdd, + Format::Lzw, + Format::Lz4, + Format::Dmg, + Format::Rar, + Format::Ole, ]; let mut silent = Vec::new(); - for fmt in formats.into_iter().filter(|f| is_streamable(*f)) { + for fmt in formats { let mut budget = Budget::new(Limits::default()); // A member's bytes are read by the visitor, which sees their errors. - let mut visit = |_: &MemberMeta, r: Option<&mut dyn Read>, _: &mut Budget| { - io::copy(r?, &mut io::sink()).err().map(|e| e.to_string()) + let mut visit = |_: &_, content: Option>, _: &mut Budget| match content? { + Member::Stream(r) => io::copy(r, &mut io::sink()).err().map(|e| e.to_string()), + Member::Bytes(_) => None, }; - let src = Unreadable { + let src = BlockCache::new(Unreadable { pos: 0, len: 1 << 20, - }; - match stream_members(fmt, src, &mut budget, &mut visit) { + }) + .unwrap(); + match walk(fmt, &src, &mut budget, &mut visit) { Err(hit) if hit.reason.contains("link down") => {} Ok(Some(seen)) if seen.contains("link down") => {} other => silent.push(format!("{fmt:?}: {other:?}")), diff --git a/crates/exav-unpack/tests/suites/udf.rs b/crates/exav-unpack/tests/suites/udf.rs index ba9a2cb1..8449a520 100644 --- a/crates/exav-unpack/tests/suites/udf.rs +++ b/crates/exav-unpack/tests/suites/udf.rs @@ -21,7 +21,8 @@ //! past sector 256, making even a three-file image ~900 KiB of mostly zeroes. //! The bytes inside are genisoimage's. -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; /// `sha256sum` of what `7zz x` writes for each member. const README_SHA256: &str = "8d2f3443b4461106fa29ee57260e553ee6acb25cc0a00a28a777013d2420985e"; @@ -62,19 +63,27 @@ fn members(blob: &[u8]) -> Vec { out } -/// What [`exav_unpack::stream_members`] yields, as `(name, sha256, unsupported)`. +/// What a [`exav_unpack::walk`] visitor reading each member as it comes sees, +/// as `(name, sha256, unsupported)`. fn streamed(blob: &[u8]) -> Vec<(String, String, Option<&'static str>)> { let mut out = Vec::new(); let mut b = Budget::new(Limits::default()); - let _ = exav_unpack::stream_members( + let _ = exav_unpack::walk( Format::Iso, - std::io::Cursor::new(blob.to_vec()), + &blob, &mut b, - &mut |m: &exav_unpack::MemberMeta, r: Option<&mut dyn std::io::Read>, _: &mut Budget| { - let mut data = Vec::new(); - if let Some(r) = r { - std::io::Read::read_to_end(r, &mut data).expect("read member"); - } + &mut |m: &exav_unpack::MemberMeta, + content: Option>, + _: &mut Budget| { + let data = match content { + None => Vec::new(), + Some(exav_unpack::Member::Bytes(d)) => d, + Some(exav_unpack::Member::Stream(r)) => { + let mut d = Vec::new(); + std::io::Read::read_to_end(r, &mut d).expect("read member"); + d + } + }; out.push((m.name.clone(), sha256_hex(&data), m.unsupported)); None::<()> }, diff --git a/crates/exav-unpack/tests/suites/vhd.rs b/crates/exav-unpack/tests/suites/vhd.rs index 11c8b6f8..a8151bf5 100644 --- a/crates/exav-unpack/tests/suites/vhd.rs +++ b/crates/exav-unpack/tests/suites/vhd.rs @@ -5,7 +5,8 @@ //! user with no third-party tool. Anything exav cannot reconstruct must be //! reported, not passed over. -use exav_unpack::{extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{Budget, Entry, Format, Limits}; const SECTOR: usize = 512; const FOOTER: usize = 512; diff --git a/crates/exav-unpack/tests/suites/wim.rs b/crates/exav-unpack/tests/suites/wim.rs index 9b28f9c0..e4bc7448 100644 --- a/crates/exav-unpack/tests/suites/wim.rs +++ b/crates/exav-unpack/tests/suites/wim.rs @@ -22,7 +22,8 @@ //! wimcapture hardsrc/ hard_LZX.wim --compress=LZX //! ``` -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; /// `sha256sum` of the files that went into the images. const EICAR: &str = "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"; diff --git a/crates/exav-unpack/tests/suites/zip_codecs.rs b/crates/exav-unpack/tests/suites/zip_codecs.rs index ed96782b..6ee5cfe5 100644 --- a/crates/exav-unpack/tests/suites/zip_codecs.rs +++ b/crates/exav-unpack/tests/suites/zip_codecs.rs @@ -35,26 +35,24 @@ fn zip_lzma_member_is_decoded() { #[test] #[cfg(feature = "lzip")] fn zip_lzma_member_is_decoded_once_when_streamed() { - use exav_unpack::{stream_members, MemberMeta}; - use std::io::Read; + use exav_unpack::{walk, Member, MemberMeta}; let blob = fixture("eicar_lzma.zip"); let mut budget = Budget::new(Limits::default()); let mut seen = Vec::new(); - let mut visit = |m: &MemberMeta, r: Option<&mut dyn Read>, _: &mut Budget| { - let mut data = Vec::new(); - if let Some(r) = r { - r.read_to_end(&mut data).unwrap(); - } + let mut visit = |m: &MemberMeta, content: Option>, _: &mut Budget| { + let data = match content { + None => Vec::new(), + Some(Member::Bytes(d)) => d, + Some(Member::Stream(r)) => { + let mut d = Vec::new(); + r.read_to_end(&mut d).unwrap(); + d + } + }; seen.push((m.name.clone(), m.unsupported, data)); None::<()> }; - stream_members( - Format::Zip, - std::io::Cursor::new(blob), - &mut budget, - &mut visit, - ) - .unwrap(); + walk(Format::Zip, &blob, &mut budget, &mut visit).unwrap(); assert_eq!( seen.len(), 1, @@ -69,14 +67,8 @@ fn zip_lzma_member_is_decoded_once_when_streamed() { limits.max_buffer_bytes = 16; let mut budget = Budget::new(limits); let blob = fixture("eicar_lzma.zip"); - let mut visit = |_: &MemberMeta, _: Option<&mut dyn Read>, _: &mut Budget| None::<()>; - let hit = stream_members( - Format::Zip, - std::io::Cursor::new(blob), - &mut budget, - &mut visit, - ) - .unwrap_err(); + let mut visit = |_: &MemberMeta, _: Option>, _: &mut Budget| None::<()>; + let hit = walk(Format::Zip, &blob, &mut budget, &mut visit).unwrap_err(); assert!( !hit.is_corrupt() && hit.reason.contains("--max-object-bytes"), "{hit:?}" @@ -174,6 +166,101 @@ fn zip_bzip2_member_is_decoded() { ); } +/// PKZIP 1.x methods: Shrink (1), Reduce (2-5), Implode (6). The fixtures are +/// from the `zip` crate's own test data (zip-rs/zip2, MIT), each holding the +/// same 1092-byte text. +const LEGACY: [&str; 3] = [ + "legacy_shrink.zip", + "legacy_reduce.zip", + "legacy_implode.zip", +]; + +#[test] +fn legacy_methods_are_decoded() { + use exav_unpack::{walk, Member, MemberMeta}; + for name in LEGACY { + let blob = fixture(name); + let entries = extract(Format::Zip, &blob, &mut Budget::new(Limits::default())).unwrap(); + assert_eq!(entries.len(), 1, "{name}"); + assert_eq!(entries[0].unsupported, None, "{name}"); + assert_eq!(entries[0].data.len(), 1092, "{name}"); + assert!(entries[0].data.starts_with(b"The play of Hamlet"), "{name}"); + + let mut got = Vec::new(); + let mut visit = |m: &MemberMeta, content: Option>, b: &mut Budget| { + got.push(( + m.unsupported, + content.map(|c| c.into_bytes(m, b).unwrap().0), + )); + None::<()> + }; + walk( + Format::Zip, + &blob, + &mut Budget::new(Limits::default()), + &mut visit, + ) + .unwrap(); + assert_eq!(got.len(), 1, "{name}"); + assert_eq!(got[0].0, None, "{name}"); + assert_eq!(got[0].1.as_ref().map(Vec::len), Some(1092), "{name}"); + } +} + +/// The `zip` crate decodes these methods whole, into a buffer it reserves from +/// the member's declared size. Over the buffer limit, the member is not +/// decoded and the limit is reported. +#[test] +fn legacy_methods_over_the_buffer_limit_are_a_limit() { + use exav_unpack::{walk, Member, MemberMeta}; + for name in LEGACY { + let blob = fixture(name); + let mut limits = Limits::default(); + limits.max_buffer_bytes = 1000; + let mut visited = 0; + let mut visit = |_: &MemberMeta, _: Option>, _: &mut Budget| { + visited += 1; + None::<()> + }; + let hit = walk(Format::Zip, &blob, &mut Budget::new(limits), &mut visit).unwrap_err(); + assert_eq!(visited, 0, "{name}"); + assert!( + !hit.is_corrupt() && hit.reason.contains("--max-object-bytes"), + "{name}: {hit:?}" + ); + } +} + +/// A member in a codec the `zip` crate lacks, larger than the buffer limit, +/// is decoded as it is read, and the member after it is still reached. +#[test] +#[cfg(feature = "bzip2")] +fn a_large_raw_decoded_member_streams_and_the_walk_goes_on() { + use exav_unpack::{walk, Member, MemberMeta}; + // big.bin: 20000 bytes, bzip2 (method 12); after.txt: deflated. + let blob = fixture("bzip2_large_then_small.zip"); + let mut limits = Limits::default(); + limits.max_buffer_bytes = 8192; + let mut budget = Budget::new(limits); + let mut seen = Vec::new(); + let mut visit = |m: &MemberMeta, content: Option>, _: &mut Budget| { + let mut d = Vec::new(); + if let Some(Member::Stream(r)) = content { + r.read_to_end(&mut d).unwrap(); + } + seen.push((m.name.clone(), m.unsupported, d.len())); + None::<()> + }; + walk(Format::Zip, &blob, &mut budget, &mut visit).unwrap(); + assert_eq!( + seen, + [ + ("big.bin".to_string(), None, 20000), + ("after.txt".to_string(), None, 72) + ] + ); +} + // --- Orphan local headers must never be silently dropped -------------------- // // A credible local file header the central directory doesn't cover *is* a member @@ -202,7 +289,7 @@ fn lfh(name: &str, method: u16, flags: u16, payload: &[u8], declared_comp: Optio fn orphan_entries(blob: &[u8]) -> Vec { let mut budget = Budget::new(Limits::default()); - extract(Format::Zip, blob, &mut budget).unwrap() + extract(Format::Zip, &blob, &mut budget).unwrap() } #[test] @@ -418,6 +505,7 @@ fn orphan_stored_member_with_deferred_size_is_reported_not_guessed() { /// a 2-byte APPNOTE 5.9 header at the front of the member instead of in coder /// properties — so this is the part 7z's tests do NOT cover. #[test] +#[cfg(feature = "sevenz")] fn zip_ppmd_member_is_decoded() { use std::io::Write; diff --git a/crates/exav-unpack/tests/suites/zip_deflate64.rs b/crates/exav-unpack/tests/suites/zip_deflate64.rs index ff772922..8cc8a305 100644 --- a/crates/exav-unpack/tests/suites/zip_deflate64.rs +++ b/crates/exav-unpack/tests/suites/zip_deflate64.rs @@ -59,7 +59,7 @@ fn crc32(data: &[u8]) -> u32 { fn only_member(blob: &[u8]) -> exav_unpack::Entry { let mut budget = Budget::new(Limits::default()); - let mut entries = extract(Format::Zip, blob, &mut budget).expect("extract"); + let mut entries = extract(Format::Zip, &blob, &mut budget).expect("extract"); assert_eq!(entries.len(), 1, "expected exactly one member"); let e = entries.remove(0); assert!( diff --git a/crates/exav-unpack/tests/suites/zip_disguised_members.rs b/crates/exav-unpack/tests/suites/zip_disguised_members.rs index 286ae911..158a9b35 100644 --- a/crates/exav-unpack/tests/suites/zip_disguised_members.rs +++ b/crates/exav-unpack/tests/suites/zip_disguised_members.rs @@ -3,16 +3,13 @@ //! came off live samples in a clamd differential run, where each cost a real //! detection. //! -//! Every case is asserted against BOTH ZIP walkers — the buffered -//! [`extract`] path and the [`stream_members`] path the top-level scan takes. -//! They are separate implementations with separate copies of the skip logic, and -//! the first fix for the trailing-slash trick landed in only one of them: the -//! sample still scanned clean afterwards, because the scanner never went through -//! the code that was fixed. A test that exercises one walker proves nothing -//! about the other. +//! Every case is asserted through [`extract`] and through a [`walk`] visitor +//! reading each member as it comes, the way the scanner does. The first fix for +//! the trailing-slash trick once landed in a second ZIP walker the scanner did +//! not use, and the sample still scanned clean; asserting on the walk the +//! scanner takes is what catches that. -use exav_unpack::{extract, stream_members, Budget, Entry, Format, Limits, MemberMeta}; -use std::io::Read; +use exav_unpack::{extract, walk, Budget, Entry, Format, Limits, Member, MemberMeta}; /// Bitwise CRC-32 (IEEE), so the fixtures carry real checksums without pulling a /// dependency into the test. @@ -88,27 +85,31 @@ fn zip_stored(members: &[(&str, &[u8], u16, u32)]) -> Vec { out } -/// Members the buffered walker yields. +/// Members [`extract`] yields. fn buffered(blob: &[u8]) -> Vec { let mut budget = Budget::new(Limits::default()); - extract(Format::Zip, blob, &mut budget).expect("extract") + extract(Format::Zip, &blob, &mut budget).expect("extract") } -/// Members the streaming walker yields, as `(name, contents)`. A member with no -/// reader (nothing decodable) comes back with empty contents. +/// Members a [`walk`] visitor sees, as `(name, contents)`. A member with no +/// content (nothing decodable) comes back empty. fn streamed(blob: &[u8]) -> Vec<(String, Vec)> { let mut budget = Budget::new(Limits::default()); let mut seen: Vec<(String, Vec)> = Vec::new(); - let cur = std::io::Cursor::new(blob.to_vec()); - let _ = stream_members::<_, ()>( + let _ = walk::<()>( Format::Zip, - cur, + &blob, &mut budget, - &mut |meta: &MemberMeta, rdr: Option<&mut dyn Read>, _b: &mut Budget| { - let mut buf = Vec::new(); - if let Some(r) = rdr { - let _ = r.read_to_end(&mut buf); - } + &mut |meta: &MemberMeta, content: Option>, _b: &mut Budget| { + let buf = match content { + None => Vec::new(), + Some(Member::Bytes(d)) => d, + Some(Member::Stream(r)) => { + let mut d = Vec::new(); + let _ = r.read_to_end(&mut d); + d + } + }; seen.push((meta.name.clone(), buf)); None }, diff --git a/crates/exav-unpack/tests/suites/zoo.rs b/crates/exav-unpack/tests/suites/zoo.rs index ee276ee9..60913bad 100644 --- a/crates/exav-unpack/tests/suites/zoo.rs +++ b/crates/exav-unpack/tests/suites/zoo.rs @@ -11,7 +11,8 @@ //! Fixtures are the `unarc-rs` project's own test archives (MIT OR Apache-2.0), //! written by real `zoo`; see `NOTICE`. -use exav_unpack::{detect, extract_each, Budget, Entry, Format, Limits}; +use super::extract_each; +use exav_unpack::{detect, Budget, Entry, Format, Limits}; fn fixture(name: &str) -> Vec { let p = format!("{}/tests/fixtures/zoo/{name}", env!("CARGO_MANIFEST_DIR")); diff --git a/crates/exav-update/Cargo.toml b/crates/exav-update/Cargo.toml index 6078d32b..a7ffa767 100644 --- a/crates/exav-update/Cargo.toml +++ b/crates/exav-update/Cargo.toml @@ -22,7 +22,7 @@ ureq = "2" # Verify a downloaded `.exavdb`'s embedded SHA-256 before installing it, so an # interrupted/corrupt download is rejected rather than clobbering a good cache. # Same hash exav-core stamps into the cache; RustCrypto, no C/asm. -sha2 = "0.10" +sha2 = "0.11" # docs.rs builds with every feature on, so the parts gated behind a feature # appear on the rendered page diff --git a/crates/exav-update/src/lib.rs b/crates/exav-update/src/lib.rs index 88cf2952..657b994f 100644 --- a/crates/exav-update/src/lib.rs +++ b/crates/exav-update/src/lib.rs @@ -243,7 +243,7 @@ fn fetch_into( if auth.is_some() && scheme == "http" { return Err(io::Error::new( io::ErrorKind::InvalidInput, - "refusing to send credentials over http:// — use https://, or drop \ + "refusing to send credentials over http://: use https://, or drop \ the `user:pass@` and authenticate another way", )); } @@ -381,13 +381,13 @@ fn verify_exav_db(body: &[u8]) -> Result<(), String> { return Err("truncated download (shorter than an empty database header)".into()); } if &body[..EXAV_DB_MAGIC.len()] != EXAV_DB_MAGIC { - return Err("not an exav database (.exavdb) — wrong magic".into()); + return Err("not an exav database (.exavdb): wrong magic".into()); } let (head_payload, trailer) = body.split_at(body.len() - EXAV_DB_DIGEST_LEN); let payload = &head_payload[EXAV_DB_HEADER_LEN..]; if Sha256::digest(payload).as_slice() != trailer { return Err( - "integrity check failed (SHA-256 mismatch) — download corrupt or interrupted".into(), + "integrity check failed (SHA-256 mismatch): download corrupt or interrupted".into(), ); } Ok(()) @@ -436,7 +436,8 @@ fn sanitize_component(raw: &str) -> Option { } /// Map a source `url` to the on-disk path exav installs it at: -/// `/env///`. Namespacing by origin means two +/// `/env///-.`, the digest being of +/// the whole URL (see below). Namespacing by origin means two /// feeds that happen to share a filename (e.g. two CVD mirrors both ending in /// `daily.cvd`) never clobber each other. Every component is sanitized, and `.` /// /`..` are dropped, so a crafted URL can never escape `sigdir`. Errors only if diff --git a/crates/exav-x86/README.md b/crates/exav-x86/README.md index 6988855f..7246dca5 100644 --- a/crates/exav-x86/README.md +++ b/crates/exav-x86/README.md @@ -40,11 +40,11 @@ the system-instruction space, SSE/MMX with the `0F 38`/`0F 3A` escapes, 3DNow!, VEX (`C4`/`C5`), EVEX (`62`) and AMD's XOP (`8F`). No REX, no RIP-relative addressing, no 64-bit forms. -Operands are modelled for the general-purpose and x87 encodings. For the SIMD -maps the register *file* an operand names — MMX, XMM/YMM, mask — is not -modelled: a register number is reported as the encoding's own, and a vector -index register is not reported at all. Nothing in exav interprets those -operands; their identity and their length are what matter. +Operands are modelled for the general-purpose and x87 encodings, and for the +`0F` SIMD encodings whose shape is in the generated table, which report their +register file (`Op::Mmx`, `Op::Xmm`) and memory width. Elsewhere in the SIMD +maps (VEX, EVEX, XOP) a register operand is not reported, and a vector index +register never is; their identity and their length are what matter. Coverage is measured, not asserted. The corpus sweep decodes every byte offset of a set of real packed samples and compares each site where `iced-x86` decodes diff --git a/crates/exav-x86/src/lib.rs b/crates/exav-x86/src/lib.rs index 9a108d95..bd7dd6bd 100644 --- a/crates/exav-x86/src/lib.rs +++ b/crates/exav-x86/src/lib.rs @@ -21,12 +21,13 @@ //! escape map, the system-instruction space, SSE/MMX with the `0F 38` and //! `0F 3A` escapes, 3DNow!, VEX, EVEX and AMD's XOP. //! -//! Operands are modelled for the general-purpose and x87 encodings. For the -//! SIMD maps the register *file* an operand names — MMX, XMM/YMM, mask — is -//! not: a register number is reported as the encoding's own, and a vector index -//! register is not reported at all. Nothing in exav interprets those operands, -//! and reporting one as a general-purpose register would be a plausible-looking -//! lie. What those encodings are asked for is their identity and their length. +//! Operands are modelled for the general-purpose and x87 encodings, and for the +//! `0F` SIMD encodings whose shape is in the generated table, which report their +//! register file (`Op::Mmx`, `Op::Xmm`) and memory width. Elsewhere in the SIMD +//! maps (VEX, EVEX, XOP, and an unmodelled `0F` shape) a register operand is not +//! reported at all, and a vector index register never is: reporting one as a +//! general-purpose register would be a plausible-looking lie. What those +//! encodings are asked for is their identity and their length. //! //! The scope was measured, not guessed: instrumenting the emulator across the //! packed-sample corpus records which encodings real stubs actually execute, @@ -639,8 +640,8 @@ pub enum Op { Xmm(u8), /// An MMX register, `mm0`–`mm7`. Mmx(u8), - /// A memory operand whose width is not one of [`Size`]'s three — the x87 - /// operand sizes (8, 10, 28 and 108 bytes) and nothing else. + /// A memory operand whose width is not one of [`Size`]'s three: the x87 + /// operand sizes (8, 10, 28 and 108 bytes) and the SIMD widths. MemWide { base: Option, index: Option, diff --git a/crates/exav/Cargo.toml b/crates/exav/Cargo.toml index fdb6e71a..95ce2dcf 100644 --- a/crates/exav/Cargo.toml +++ b/crates/exav/Cargo.toml @@ -23,44 +23,45 @@ name = "exav" path = "src/main.rs" [features] -default = ["yara", "all-formats", "decrypt", "dlp", "icap"] +default = ["yara", "all-formats", "decrypt", "dlp", "phishing", "icap"] # ICAP (RFC 3507) server, so exav can stand in for a `c-icap` container at a -# proxy's adaptation hook. ON by default — pure-Rust, std-only, no extra crates -# — because the published image has to be able to serve it. It still binds -# nothing unless `--icap` asks it to. +# proxy's adaptation hook. On by default (pure Rust, std-only, no extra crates) +# because the published image has to be able to serve it. It still binds +# nothing unless `--listen icap://...` asks it to. icap = [] -# Full YARA rule matching via the native engine in exav-core. ON by default — -# pure-Rust, no runtime codegen. Disable with `--no-default-features` for a -# smaller build (~13 fewer crates: drops the YARA parser + regex-automata). +# YARA rule matching via the native engine in exav-core. On by default; pure +# Rust, no runtime codegen. Without it, rule files load but never match. yara = ["exav-core/yara"] # Lets a scanned file ask a decoder to panic, abort or exhaust the stack, so the # tests can check what the CLI reports when one does. A crash that a caller # cannot tell from a clean scan is the worst outcome exav has, and that is only # observable from outside the process. Never enabled by a shipped build. testing-faults = ["exav-core/testing-faults"] -# HTTP(S) support. OFF by default so the standard build is 100% pure-Rust and -# links no TLS stack — `ureq → rustls → ring` is the only C/asm in the tree. -# Split into two independently-selectable halves so a deployment can take the -# signature updater WITHOUT exposing the network-facing scan verb (or vice versa): +# HTTP(S) support. Off by default so the standard build is pure Rust and links +# no TLS stack (`ureq -> rustls -> ring` is the only C/asm in the tree). +# Split into two halves so a deployment can take the signature updater without +# the network-facing scan verb, or the reverse: # -# * `http-scan` — scan an `http(s)://` argument + the daemon `SCANURL` command. -# This lets a daemon *client* make the daemon fetch a URL, so -# leave it off on an updater-only or untrusted-client daemon. -# * `http-update` — what `--auto-update` fetches with: `--sig-sources` (exact -# URLs, CVD mirror bases, or a file of either) and `--db-url` -# (pull a prebuilt `.exavdb`). Pulls the `exav-update` crate. +# * `http-scan`: scan an `http(s)://` argument, and the daemon `SCANURL` +# command. This lets a daemon client make the daemon fetch a +# URL, so leave it off on an updater-only or untrusted-client +# daemon. +# * `http-update`: what `--auto-update` fetches with: `--sig-sources` (exact +# URLs, CVD mirror bases, or a file of either) and `--db-url` +# (a prebuilt `.exavdb`). Pulls the `exav-update` crate. # -# `http` enables both (unchanged meaning; the published image builds with it). +# `http` enables both; the published image builds with it. http = ["http-scan", "http-update"] http-scan = ["exav-core/http"] http-update = ["dep:exav-update"] -checksums = ["exav-core/checksums"] -# DLP structured-data heuristic (credit-card / SSN counting). Enables the -# `--structured-cc-count` / `--structured-ssn-count` flags. Tiny, pure-Rust. +# DLP structured-data heuristic (credit-card / SSN counting): what +# `--dlp-credit-cards` / `--dlp-ssns` run. Tiny, pure-Rust. dlp = ["exav-core/dlp"] +# The phishing heuristic `--detect phishing` runs. Tiny, pure-Rust. +phishing = ["exav-core/phishing"] decrypt = ["exav-core/decrypt"] -# Archive/container formats, forwarded to exav-core → exav-unpack. `all-formats` +# Archive/container formats, forwarded to exav-core, then exav-unpack. `all-formats` # (in `default`) is every extractor; pick individual formats for a smaller # binary (e.g. `--no-default-features --features yara,zip`). all-formats = ["exav-core/all-formats"] @@ -128,10 +129,10 @@ screnc = ["exav-core/screnc"] pe-emu = ["exav-core/pe-emu"] [dependencies] -exav-core = { path = "../exav-core", version = "0.0.1", default-features = false } +exav-core = { path = "../exav-core", version = "0.0.2", default-features = false } # Standalone signature updater (its own TLS stack), enabled by the `http` # feature so the default pure-Rust build never links it. -exav-update = { path = "../exav-update", version = "0.0.1", optional = true } +exav-update = { path = "../exav-update", version = "0.0.2", optional = true } clap.workspace = true anyhow.workspace = true walkdir.workspace = true @@ -168,7 +169,7 @@ assets = [ ["target/release/exav", "usr/bin/", "755"], ["README.md", "usr/share/doc/exav/README.md", "644"], ["../../NOTICE", "usr/share/doc/exav/NOTICE", "644"], - ["../../packaging/exav-clamd.service", "lib/systemd/system/exav-clamd.service", "644"], + ["../../packaging/exav.service", "lib/systemd/system/exav.service", "644"], ] diff --git a/crates/exav/src/daemon.rs b/crates/exav/src/daemon.rs index dc04ffd4..09fcf07e 100644 --- a/crates/exav/src/daemon.rs +++ b/crates/exav/src/daemon.rs @@ -10,10 +10,11 @@ //! `PING` -> `PONG` //! `VERSION` -> `exav ` //! `STATS` -> a short status block ending with `END` -//! `RELOAD` -> `RELOADING`; in the prefork pool this signals the -//! supervisor to re-read the data dir and re-fork the -//! workers with the new signatures (the mechanism a -//! stock `freshclam`'s `NotifyClamd` drives) +//! `RELOAD` -> `RELOADING`; the supervisor re-reads the +//! signatures (in the prefork pool, re-forking the +//! workers with them), as `SIGUSR2` does. The +//! mechanism a stock `freshclam`'s `NotifyClamd` +//! drives //! `SCAN ` -> `: OK` / `: FOUND` / `… ERROR` //! `CONTSCAN ` -> recurse a directory, one reply line per file //! `MULTISCAN ` -> alias of CONTSCAN @@ -21,7 +22,7 @@ //! The payload is materialized to a seekable source //! (kept in RAM when small, spilled to an auto-deleted //! temp file when large) and given the FULL -//! container-aware scan — so malware inside an archive +//! container-aware scan, so malware inside an archive //! is detected, matching clamd. Bounded by //! `--max-input-bytes` (disk is the ceiling). Reply //! `stream: …` @@ -35,7 +36,7 @@ //! [,"location":"outer.zip/…/inside.txt"]} //! (location present only for a NESTED hit; it is //! the `/`-joined container member-name path from -//! the stream to the matched leaf — control bytes +//! the stream to the matched leaf; control bytes //! sanitised, capped ~512 chars) //! {"v":1,"status":"PARTIAL","category":C[,"reason":R]} //! (C ∈ LIMITS-EXCEEDED / UNSCANNABLE / @@ -56,7 +57,7 @@ //! `EXINSTREAM MULTI` -> exav extension: several files in ONE request, so a //! multi-volume archive split across them (`x.7z.001`, //! `.002`, …) is rejoined and scanned as the one -//! archive it is — sent one at a time, no part decodes +//! archive it is. Sent one at a time, no part decodes //! and every one of them replies `clean`. Framing, //! repeated per file: //! `` then the INSTREAM chunk @@ -85,7 +86,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use exav_core::{ - analyze_all_with_outcome, scan_path, scan_seekable_located, AllMatchOutcome, ScanOptions, + analyze_all_seekable, scan_path, scan_seekable_located, AllMatchOutcome, ScanOptions, ScanReport, Scanner, Verdict, VerdictCategory, }; use walkdir::WalkDir; @@ -285,7 +286,7 @@ impl FdSource for AncillaryReader<'_> { } } -/// Send `cmd` on `stream` with `fd` attached as `SCM_RIGHTS` ancillary data — +/// Send `cmd` on `stream` with `fd` attached as `SCM_RIGHTS` ancillary data: /// the client half of `FILDES`, and the one thing a plain `write` cannot do. /// /// The descriptor the daemon receives is its own, pointing at the same open @@ -385,7 +386,7 @@ pub const DEFAULT_MAX_CONNECTIONS: usize = 128; /// Per-read socket timeout. Bounds how long a connection may block the daemon /// waiting for the client to send data (a command, or the next INSTREAM chunk), -/// so a slow/idle client — the classic slow-loris — can't pin a connection slot +/// so a slow/idle client (the classic slow-loris) can't pin a connection slot /// (or, in the thread model where there is no per-job kill, a worker) forever. /// Applied to every accepted stream; a stall past this closes the connection. const SOCKET_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); @@ -412,17 +413,89 @@ impl Drop for ConnGuard { } } +/// The database the thread-model listeners share, swapped whole on a reload. +/// A connection or request already in progress keeps the one it started with. +pub struct SharedDb(std::sync::RwLock>); + +impl SharedDb { + pub fn new(db: Arc) -> Self { + Self(std::sync::RwLock::new(db)) + } + + /// The database to serve the next connection or request with. + pub fn current(&self) -> Arc { + // A poisoned lock means a panic happened while swapping, not that the + // database is unusable. + Arc::clone(&self.0.read().unwrap_or_else(|e| e.into_inner())) + } + + pub(crate) fn replace(&self, db: Scanner) { + *self.0.write().unwrap_or_else(|e| e.into_inner()) = Arc::new(db); + } +} + +/// The thread model's supervisor: reload `db` on `RELOAD`, `SIGUSR2`, or a +/// change to the watched signature source, and reopen the `--log` file on +/// `SIGHUP`, as clamd does. A failed reload keeps the database in use. Never +/// returns. +pub fn supervise( + db: &SharedDb, + watch: Option, + reload_db: &dyn Fn() -> Result, +) -> ! { + #[cfg(unix)] + { + if let Err(e) = arm_wakeup() { + eprintln!( + "exav: reload signals will wait for the next {}s poll: {e}", + SUPERVISOR_TICK.as_secs() + ); + } + // Restarting: the listeners' threads must not see their reads and + // accepts interrupted by a signal meant for this one. + install_restarting_handler(libc::SIGHUP, on_sighup); + install_restarting_handler(libc::SIGUSR2, on_sigusr2); + } + let mut last = watch.as_deref().and_then(datadir_mtime); + loop { + #[cfg(unix)] + nap(SUPERVISOR_TICK); + #[cfg(not(unix))] + std::thread::sleep(SUPERVISOR_TICK); + let disk_changed = match watch.as_deref().and_then(datadir_mtime) { + Some(t) if source_changed(last, t) => { + last = Some(t); + true + } + _ => false, + }; + if RELOAD_REQUESTED.swap(false, Ordering::Relaxed) || disk_changed { + match reload_db() { + Ok(new_db) => { + eprintln!( + "exav: reloading signatures ({} known)", + new_db.signature_count() + ); + exav_core::warm_up(&new_db); + db.replace(new_db); + } + Err(e) => eprintln!("exav: signature reload failed, keeping current DB: {e}"), + } + } + } +} + /// Run the daemon until the listener errors (e.g. the process is killed). /// `allow_shutdown` controls whether the `SHUTDOWN` command stops the daemon /// (here it exits the process; the thread model has no supervisor to unwind). -/// `allow_http_scan` controls whether the exav-only `SCANURL` command fetches — +/// `allow_http_scan` controls whether the exav-only `SCANURL` command fetches, /// off by default, since it lets any client that can reach the socket make the /// daemon fetch a URL. /// -/// The database arrives already shared so one load can answer on more than one -/// listener in the same process. +/// Each connection is served with the database current when it opened; a +/// reload ([`supervise`]) swaps it for the connections after. pub fn run( - db: Arc, + db: Arc, addr: ListenAddr, opts: Arc, allow_shutdown: bool, @@ -432,8 +505,8 @@ pub fn run( use std::sync::atomic::{AtomicUsize, Ordering}; let conns = Arc::new(AtomicUsize::new(0)); // A client that disconnects right after reading a reply delivers SIGPIPE on - // the next write. `main` set the default disposition — which kills the - // process — so that a one-shot scan piped into `head` ends quietly; a + // the next write. `main` set the default disposition, which kills the + // process, so that a one-shot scan piped into `head` ends quietly; a // listener wants the opposite. Ignoring it makes the write fail with EPIPE // and cost one connection instead of the daemon. #[cfg(unix)] @@ -453,9 +526,9 @@ pub fn run( let _ = stream.set_read_timeout(Some(SOCKET_READ_TIMEOUT)); if conns.fetch_add(1, Ordering::Relaxed) >= max_connections { conns.fetch_sub(1, Ordering::Relaxed); - continue; // at capacity — drop the connection + continue; // at capacity: drop the connection } - let (db, opts, c) = (Arc::clone(&db), Arc::clone(&opts), Arc::clone(&conns)); + let (db, opts, c) = (db.current(), Arc::clone(&opts), Arc::clone(&conns)); std::thread::spawn(move || { let _guard = ConnGuard(c); // The fd-capturing reader handles FILDES; the writer is a @@ -477,7 +550,7 @@ pub fn run( &opts, &|| {}, &|| {}, - &|| {}, + &request_reload, &shutdown, allow_http_scan, ) { @@ -498,9 +571,9 @@ pub fn run( let stream = TcpListenerStream(tcp); if conns.fetch_add(1, Ordering::Relaxed) >= max_connections { conns.fetch_sub(1, Ordering::Relaxed); - continue; // at capacity — drop the connection + continue; // at capacity: drop the connection } - let (db, opts, c) = (Arc::clone(&db), Arc::clone(&opts), Arc::clone(&conns)); + let (db, opts, c) = (db.current(), Arc::clone(&opts), Arc::clone(&conns)); std::thread::spawn(move || { let _guard = ConnGuard(c); let shutdown = || { @@ -517,7 +590,7 @@ pub fn run( &opts, &|| {}, &|| {}, - &|| {}, + &request_reload, &shutdown, allow_http_scan, ) { @@ -537,26 +610,27 @@ pub fn run( // `--workers N` switches the daemon from the in-process thread model above to a // pool of N persistent worker *processes*. Each worker handles one scan at a // time (sequentially), so a single job can be bounded and, if it goes rogue, -// killed without touching any other in-flight work — the one thing the thread +// killed without touching any other in-flight work, the one thing the thread // model cannot do safely at all (no safe thread-kill in Rust/C). // // Why processes / why Unix-only here: // * Workers `fork()` from the parent *after* the DB is loaded and warmed, so -// the (large, read-only) signature DB is shared via copy-on-write — no +// the (large, read-only) signature DB is shared via copy-on-write: no // re-load, low memory. // * They inherit the listening socket and `accept()` on it directly, so the // full clamd protocol (incl. `FILDES` fd-passing via `SCM_RIGHTS`) is // handled in the worker with zero parent relay. // * Limits are enforced by the kernel, the only layer that can stop a stuck -// call inside a dependency: `RLIMIT_AS` (memory) / `RLIMIT_CPU` (CPU time) -// trigger a kernel kill, and a per-job `setitimer(SIGALRM)` whose handler -// `_exit()`s gives a hard wall-clock bound even on a non-yielding CPU loop. +// call inside a dependency: `RLIMIT_AS` fails the allocation past the +// memory cap, a per-job soft `RLIMIT_CPU` raises `SIGXCPU`, and a per-job +// `setitimer(SIGALRM)` gives a wall-clock bound even on a non-yielding CPU +// loop. Both signal handlers answer the client and `_exit()`. // * Workers recycle after `max_jobs` to bound slow leaks/fragmentation; the // supervisor respawns them by forking from the clean parent (pristine COW). // // This is the deterministic-caps backstop (Layer 3): the in-core // `max_scanned_bytes`/ratio/recursion caps still fire first and identically in -// both models — the pool only adds the hard kill for the residual tail. +// both models; the pool only adds the hard kill for the residual tail. /// Exit code a worker uses when its per-job wall-clock alarm fires. #[cfg(unix)] @@ -571,16 +645,16 @@ pub struct PoolConfig { pub max_scan_time: std::time::Duration, /// Per-worker address-space cap in bytes (`RLIMIT_AS`). pub max_memory_bytes: u64, - /// Per-worker CPU-seconds cap (`RLIMIT_CPU`; kernel `SIGXCPU`/`SIGKILL`). + /// Per-job CPU-seconds cap (soft `RLIMIT_CPU`, re-armed per job; `SIGXCPU`). pub max_cpu_secs: u64, /// Recycle a worker after this many jobs (bounds slow leaks). pub max_jobs: u64, /// Whether the `SHUTDOWN` command is honoured (clamd default: yes). When - /// false, `SHUTDOWN` replies with an error instead of stopping the daemon — + /// false, `SHUTDOWN` replies with an error instead of stopping the daemon, /// useful when the socket/port is reachable by untrusted clients. pub allow_shutdown: bool, /// Whether the exav-only `SCANURL` command fetches (default: no). When - /// false, `SCANURL` replies with an error instead of fetching — useful for + /// false, `SCANURL` replies with an error instead of fetching, useful for /// the same reason: it lets any client that can reach the socket make the /// daemon fetch a URL. pub allow_http_scan: bool, @@ -625,12 +699,12 @@ fn bind_listener(addr: &ListenAddr) -> io::Result { /// listening before any `chmod` runs and, under a permissive umask, listening /// at 0777 while it does. Binding under a umask of 0777 instead creates it with /// no permissions at all: the only mode a client can ever find on the path is -/// the one asked for. A failed `chmod` is fatal for the same reason — the +/// the one asked for. A failed `chmod` is fatal for the same reason: the /// socket would otherwise stay at mode 000 and refuse everyone silently. /// /// The path itself is still a name another process can replace between the two /// calls if it can write the directory, so a socket belongs somewhere only its -/// owner can write — `$XDG_RUNTIME_DIR` or `/var/run`, not world-writable +/// owner can write (`$XDG_RUNTIME_DIR` or `/var/run`), not world-writable /// `/tmp`, where another local user can pre-create the path. #[cfg(unix)] fn bind_unix_socket( @@ -664,25 +738,59 @@ fn bind_unix_socket( #[cfg(unix)] static SHUTDOWN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); -/// Set to true when a DB reload is requested: by SIGHUP (which a worker raises on -/// a `RELOAD` command, and which the CLI's background updater raises after a -/// successful download) or by the supervisor noticing the data dir changed on -/// disk. The supervisor drains it each tick and re-forks the pool. -#[cfg(unix)] +/// Set to true when a DB reload is requested: by SIGUSR2 (which a worker raises +/// on a `RELOAD` command, and which the CLI's background updater raises after a +/// successful download), as clamd does. The supervisor drains it each tick. static RELOAD_REQUESTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); +/// Set by SIGHUP in the supervisor, for it to pass the signal on to its +/// children: each reopens its own `--log` handle. +#[cfg(unix)] +static HUP_RECEIVED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + #[cfg(unix)] extern "C" fn on_shutdown(_sig: libc::c_int) { SHUTDOWN.store(true, std::sync::atomic::Ordering::Relaxed); wake(); } +/// SIGUSR2: reload the signatures. #[cfg(unix)] -extern "C" fn on_sighup(_sig: libc::c_int) { +extern "C" fn on_sigusr2(_sig: libc::c_int) { RELOAD_REQUESTED.store(true, std::sync::atomic::Ordering::Relaxed); wake(); } +/// SIGHUP: reopen the `--log` file. +#[cfg(unix)] +extern "C" fn on_sighup(_sig: libc::c_int) { + crate::LOG_REOPEN.store(true, std::sync::atomic::Ordering::Relaxed); + HUP_RECEIVED.store(true, std::sync::atomic::Ordering::Relaxed); + wake(); +} + +/// SIGHUP in a worker or the side listener's child: reopen its own handle. +#[cfg(unix)] +extern "C" fn on_sighup_child(_sig: libc::c_int) { + crate::LOG_REOPEN.store(true, std::sync::atomic::Ordering::Relaxed); +} + +/// The dispositions every child of the pool starts from: the supervisor's +/// flag-setting handlers must not fire in it, its SIGTERM is the default until +/// the child installs its own, and a SIGUSR2 sent to the whole process group is +/// the supervisor's to act on. +#[cfg(unix)] +fn child_signals() { + // SAFETY: sets this process's own dispositions, right after `fork`. + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_DFL); + libc::signal(libc::SIGINT, libc::SIG_DFL); + libc::signal(libc::SIGCHLD, libc::SIG_DFL); + libc::signal(libc::SIGUSR2, libc::SIG_IGN); + } + install_restarting_handler(libc::SIGHUP, on_sighup_child); +} + /// Wakes the supervisor so a worker exit is reaped at once. #[cfg(unix)] extern "C" fn on_sigchld(_sig: libc::c_int) { @@ -743,32 +851,26 @@ fn wake() { } } -/// Public so the CLI's updater thread can raise a reload after writing new -/// signatures to the data dir (equivalent to sending `RELOAD` over the socket). -/// Only called from the `http-update`-gated updater, so it's dead otherwise. -#[cfg(unix)] -#[cfg_attr(not(feature = "http-update"), allow(dead_code))] +/// Ask the supervisor for a reload: what `RELOAD` does in the thread model, and +/// what the CLI's updater thread does after writing new signatures. pub fn request_reload() { RELOAD_REQUESTED.store(true, std::sync::atomic::Ordering::Relaxed); - // Also poke the process so a sleeping supervisor wakes immediately. - unsafe { - libc::kill(libc::getpid(), libc::SIGHUP); - } + #[cfg(unix)] + wake(); } -/// The supervisor's **idle** poll cadence — the upper bound on how long a +/// The supervisor's **idle** poll cadence: the upper bound on how long a /// *signal-less* database change (a sidecar that writes the volume without /// `RELOAD`) can go unnoticed. Everything urgent is signal-driven and interrupts /// the sleep immediately, so this timer's only job is the mtime poll: a worker -/// exit (SIGCHLD) triggers instant reap+respawn, `RELOAD`/the updater/`SIGHUP` +/// exit (SIGCHLD) triggers instant reap+respawn, `RELOAD`/the updater/`SIGUSR2` /// trigger an instant reload, and SIGTERM/SIGINT an instant shutdown. It is -/// therefore a few seconds, not sub-second — polling the filesystem twice a +/// therefore a few seconds, not sub-second: polling the filesystem twice a /// second for a database that changes a few times a day would wake the process /// (and, on an NFS-mounted DB directory, generate GETATTR/READDIR traffic) for /// nothing. Still ~60× more responsive than clamd's 600 s `SelfCheck`; use -/// `RELOAD`/`NotifyClamd`/`SIGHUP` when you want a change picked up instantly. -#[cfg(unix)] -const SUPERVISOR_TICK: std::time::Duration = std::time::Duration::from_secs(10); +/// `RELOAD`/`NotifyClamd`/`SIGUSR2` when you want a change picked up instantly. +pub(crate) const SUPERVISOR_TICK: std::time::Duration = std::time::Duration::from_secs(10); /// A child of a retired generation, finishing the job it had when a reload /// replaced it. @@ -805,19 +907,15 @@ fn retire_grace(max_scan_time: std::time::Duration) -> std::time::Duration { } } -/// Newest mtime of the watched source — the reload trigger for a sidecar that +/// Newest mtime of the watched source, the reload trigger for a sidecar that /// writes the volume without sending `RELOAD` (clamd's `SelfCheck`). For a /// **directory** this is the newest mtime **across the whole tree**: the loader /// reads the directory recursively (including the updater's `env//…` -/// subtree), so the watch must too — an in-place overwrite deep in the tree bumps +/// subtree), so the watch must too: an in-place overwrite deep in the tree bumps /// only its own directory's mtime, which a top-level-only scan would miss. For a -/// single **file** (a prebuilt database) it is just that file's mtime — an atomic +/// single **file** (a prebuilt database) it is just that file's mtime; an atomic /// swap replaces its inode, and so its mtime, so the poll fires. `None` if the /// path can't be stat'd. Symlinks are not followed (no cycles). -/// -/// Also the ICAP server's reload trigger, which is why it is not Unix-gated: -/// that server runs on every platform. -#[cfg(any(unix, feature = "icap"))] pub(crate) fn datadir_mtime(dir: &std::path::Path) -> Option { let mut newest = std::fs::metadata(dir).and_then(|m| m.modified()).ok()?; // WalkDir over a file yields just that file, so this also covers the @@ -837,7 +935,6 @@ pub(crate) fn datadir_mtime(dir: &std::path::Path) -> Option, now: std::time::SystemTime, @@ -875,7 +972,7 @@ fn nap(d: std::time::Duration) { /// The worker's per-job wall-clock alarm. Terminating immediately is the whole /// point: the scan blew its time budget (possibly stuck inside a dependency /// that never returns to a cooperative checkpoint), and there is no safe way to -/// unwind in-process — so we `_exit` (async-signal-safe) and let the supervisor +/// unwind in-process, so we `_exit` (async-signal-safe) and let the supervisor /// respawn a replacement. The client gets a verdict first, as in /// [`on_sigabrt`]: a connection closed with no reply reads as clean to some /// clients. @@ -894,6 +991,27 @@ extern "C" fn on_sigalrm(_sig: libc::c_int) { unsafe { libc::_exit(EXIT_TIMEOUT) } } +/// The job used its CPU budget (see [`arm_cpu_limit`]): answered as +/// [`on_sigalrm`] answers, since the budget is `--max-scan-secs` too. +#[cfg(unix)] +extern "C" fn on_sigxcpu(_sig: libc::c_int) { + let fd = CURRENT_CONN_FD.load(std::sync::atomic::Ordering::Relaxed); + if fd >= 0 { + unsafe { + libc::write( + fd, + TIMEOUT_REPLY.as_ptr() as *const libc::c_void, + TIMEOUT_REPLY.len(), + ); + } + } + unsafe { libc::_exit(EXIT_CPU) } +} + +/// Exit code a worker uses when a job runs out of CPU time. +#[cfg(unix)] +const EXIT_CPU: i32 = 18; + /// Reply [`on_sigalrm`] emits, framed like [`ABORT_REPLY`]. #[cfg(unix)] const TIMEOUT_REPLY: &[u8] = b": scan exceeded --max-scan-secs LIMITS-EXCEEDED ERROR\n\0"; @@ -913,14 +1031,14 @@ const ABORT_REPLY: &[u8] = b": scan aborted (out of memory) ERROR\n\0"; /// Say something before dying. /// /// A worker killed mid-scan closes its connection with no reply at all, and a -/// clean close is indistinguishable from "scanned, found nothing" — so a crash +/// clean close is indistinguishable from "scanned, found nothing", so a crash /// reads as a clean verdict, which is the one answer a scanner must never give /// by accident. Measured: 55 worker deaths in an 8,978-file run, every one /// recorded as clean by the client. /// /// `SIGABRT` (Rust's allocation-failure path) is catchable, so those can be /// reported. A `SIGKILL` from the OOM killer cannot be, which is why the client -/// must ALSO treat a silent close as an error — the two fixes are complementary, +/// must ALSO treat a silent close as an error. The two fixes are complementary, /// not alternatives. /// /// Async-signal-safe: only `write` and `_exit`, no allocation, no locks. @@ -987,6 +1105,20 @@ fn install_handler(sig: libc::c_int, handler: extern "C" fn(libc::c_int)) { } } +/// [`install_handler`] with `SA_RESTART`, for a signal that only sets a flag in +/// a process whose blocking calls (a client read, `accept`) must carry on. +#[cfg(unix)] +fn install_restarting_handler(sig: libc::c_int, handler: extern "C" fn(libc::c_int)) { + // SAFETY: as in `install_handler`. + unsafe { + let mut sa: libc::sigaction = std::mem::zeroed(); + sa.sa_sigaction = handler as usize; + libc::sigemptyset(&mut sa.sa_mask); + sa.sa_flags = libc::SA_RESTART; + libc::sigaction(sig, &sa, std::ptr::null_mut()); + } +} + #[cfg(unix)] fn set_rlimit(resource: libc::c_int, limit: u64) { if limit == 0 { @@ -1001,8 +1133,33 @@ fn set_rlimit(resource: libc::c_int, limit: u64) { } } +/// Give the next job `secs` of CPU on top of what this worker has already used. +/// +/// `RLIMIT_CPU` counts the whole life of the process, so it is re-armed per job. +/// Only the soft limit moves: an unprivileged process cannot raise a hard limit +/// it has lowered. Its `SIGXCPU` is caught by [`on_sigxcpu`]. +#[cfg(unix)] +fn arm_cpu_limit(secs: u64) { + if secs == 0 { + return; + } + unsafe { + let mut ru: libc::rusage = std::mem::zeroed(); + let mut rl: libc::rlimit = std::mem::zeroed(); + if libc::getrusage(libc::RUSAGE_SELF, &mut ru) != 0 + || libc::getrlimit(libc::RLIMIT_CPU, &mut rl) != 0 + { + return; + } + let us = |t: libc::timeval| t.tv_sec as u64 * 1_000_000 + t.tv_usec as u64; + let used = (us(ru.ru_utime) + us(ru.ru_stime)).div_ceil(1_000_000); + rl.rlim_cur = (used.saturating_add(secs) as libc::rlim_t).min(rl.rlim_max); + libc::setrlimit(libc::RLIMIT_CPU, &rl); + } +} + /// Bytes of address space this process already occupies, from `/proc/self/statm` -/// (page count in field 0). `None` when it cannot be read — the caller then +/// (page count in field 0). `None` when it cannot be read; the caller then /// applies the configured cap alone. /// /// Needed because `RLIMIT_AS` is a whole-process ceiling while the per-job @@ -1022,7 +1179,7 @@ fn current_address_space() -> Option { /// The pool's design puts three layers in a fixed order: the in-core /// size/ratio/recursion caps decide first and produce a *verdict* /// (`LIMITS-EXCEEDED`), and `RLIMIT_AS` is only a backstop for the residual -/// tail. That order is not automatic — it holds only while the in-core budget +/// tail. That order is not automatic: it holds only while the in-core budget /// is smaller than the address space the worker gets, and nothing enforced it. /// /// It did not hold, and the inversion is easy to reach: on a modest host the @@ -1030,7 +1187,7 @@ fn current_address_space() -> Option { /// Extraction buffers are charged cumulatively and never released, so a scan /// can hold most of that budget live at once and meet the kernel limit first. /// A campaign run turned up dozens of aborts from exactly that. Every one was -/// reported as an error rather than a clean, so nothing was missed — but a scan +/// reported as an error rather than a clean, so nothing was missed, but a scan /// that should have said "I hit a limit" instead said "I died", which is a /// worse answer and a noisier one. /// @@ -1039,26 +1196,36 @@ fn current_address_space() -> Option { #[cfg(unix)] pub(crate) fn fit_limits_to_job_memory(opts: &mut ScanOptions, job_memory: u64) { /// Of the memory a job is granted, the share extraction buffers may claim. - /// The rest covers the matcher's own working set — chiefly the lowercase - /// copy it makes of each buffer it scans — plus allocator slack. + /// The rest covers the matcher's own working set (chiefly the lowercase + /// copy it makes of each buffer it scans) plus allocator slack. const EXTRACTION_SHARE_NUM: u64 = 1; const EXTRACTION_SHARE_DEN: u64 = 2; let budget = (job_memory / EXTRACTION_SHARE_DEN) * EXTRACTION_SHARE_NUM; - if budget == 0 || opts.limits.max_extracted_bytes <= budget { + if budget == 0 { return; } - let was = opts.limits.max_extracted_bytes; - opts.limits.max_extracted_bytes = budget; - // A single member may not exceed the whole extraction budget either. - opts.limits.max_buffer_bytes = opts.limits.max_buffer_bytes.min(budget); - eprintln!( - "exav: extraction budget {} MiB exceeds the {} MiB of address space available; using {} \ - MiB so a size limit is reported rather than the scan being killed for hitting one", - was >> 20, - job_memory >> 20, - budget >> 20, - ); + // The total a scan holds, and what one object may take of it, whether in + // an extractor's buffer or read whole by the engine. + let lowered = [ + &mut opts.limits.max_extracted_bytes, + &mut opts.limits.max_buffer_bytes, + &mut opts.deep_analysis_max, + ] + .into_iter() + .fold(false, |lowered, v| { + let over = *v > budget; + *v = (*v).min(budget); + lowered | over + }); + if lowered { + eprintln!( + "exav: a scan may hold at most {} MiB (half the {} MiB of memory it gets), so \ + a size limit is reported rather than the scan being killed for hitting one", + budget >> 20, + job_memory >> 20, + ); + } } /// This process's cgroup memory ceiling, if it has one. @@ -1143,7 +1310,7 @@ fn affordable_job_memory(workers: usize, shared_db_bytes: u64) -> Option { /// /// A third, not a token slice. A single scan's peak is NOT one buffer: the /// matcher allocates a full-size lowercase copy of every buffer it scans for - /// case-insensitive partitions, and nesting stacks those — a container, its + /// case-insensitive partitions, and nesting stacks those: a container, its /// member and that member's own member each hold a buffer AND a copy while /// the walk is inside them. So the per-job figure below is a floor on what a /// scan can want, not a ceiling, and the pool has to be sized with room for @@ -1153,7 +1320,7 @@ fn affordable_job_memory(workers: usize, shared_db_bytes: u64) -> Option { /// the per-worker grants plus the shared database left too little for the /// kernel and page cache, and dozens of workers were killed over a single /// run. An OOM kill cannot be caught, so it costs a scan its answer - /// entirely — strictly worse than the scan reporting that it hit a limit, + /// entirely, strictly worse than the scan reporting that it hit a limit, /// which is why the headroom is this generous. const HEADROOM_NUM: u64 = 1; const HEADROOM_DEN: u64 = 3; @@ -1171,7 +1338,7 @@ fn affordable_job_memory(workers: usize, shared_db_bytes: u64) -> Option { .ok()?; // `/proc/meminfo` is the HOST's memory even inside a container, so on its // own it hands a memory-capped container per-job grants its cgroup will - // never honour — and the cgroup's OOM killer then arrives before this + // never honour, and the cgroup's OOM killer then arrives before this // process's own `RLIMIT_AS` ever would, costing the scan its answer for // exactly the reason the headroom above exists to avoid. Containers are how // this daemon is mostly run, and every Kubernetes pod carries a limit. @@ -1188,7 +1355,7 @@ fn affordable_job_memory(workers: usize, shared_db_bytes: u64) -> Option { /// /// Separate from [`on_sigalrm`], which serves a pool worker that a parent will /// reap and report on. Here nobody is watching, so the handler has to say what -/// happened itself — through `write(2)`, the only output an async-signal-safe +/// happened itself, through `write(2)`, the only output an async-signal-safe /// handler may use. #[cfg(unix)] extern "C" fn on_oneshot_alarm(_sig: libc::c_int) { @@ -1198,8 +1365,8 @@ extern "C" fn on_oneshot_alarm(_sig: libc::c_int) { // Exit 3, the `PARTIAL` code, rather than the pool's dedicated timeout // code: a scan that ran out of time is one more object exav declined to // call clean, and it should sort with the others. The handler cannot - // consult `--partial-as` — reading it here would not be - // async-signal-safe — so a run that folds partials elsewhere still gets + // consult `--partial-as` (reading it here would not be + // async-signal-safe), so a run that folds partials elsewhere still gets // 3 from this path alone. libc::_exit(3) } @@ -1214,7 +1381,7 @@ extern "C" fn on_oneshot_alarm(_sig: libc::c_int) { /// never returns. /// /// Both are kernel-enforced backstops, not the primary bound. The in-core -/// budgets decide first and produce a verdict; these catch what escapes them — +/// budgets decide first and produce a verdict; these catch what escapes them: /// an allocation the budget cannot see, or a decoder that loops without /// producing output. #[cfg(unix)] @@ -1363,8 +1530,8 @@ pub(crate) fn await_connection(listener: std::os::fd::RawFd, retire: std::os::fd /// /// `datadir` (when `Some`) is polled for on-disk changes so a sidecar that writes /// the volume triggers a reload without sending `RELOAD`. `reload_db` re-reads the -/// signatures; on a `RELOAD`/SIGHUP/data-dir change the supervisor calls it, -/// warms the result, and re-forks the pool with the new DB — a failed reload is +/// signatures; on a `RELOAD`/SIGUSR2/data-dir change the supervisor calls it, +/// warms the result, and re-forks the pool with the new DB. A failed reload is /// logged and the running DB is kept. `side` is an additional listener served by /// one more child, forked, reaped and reloaded exactly as the workers are. #[cfg(unix)] @@ -1402,8 +1569,9 @@ pub fn run_prefork( if cfg.max_memory_bytes > afford { eprintln!( "exav: per-job memory {} MiB x {} workers exceeds what this host can back; \ - using {} MiB per job (RAM minus the {} MiB shared database). Raise RAM, \ - lower --workers, or set a smaller --max-process-bytes to silence this.", + using {} MiB per job (two thirds of RAM, less the {} MiB shared database, \ + over the workers). Raise RAM, lower --workers, or set a smaller \ + --max-process-bytes to silence this.", cfg.max_memory_bytes >> 20, cfg.workers, afford >> 20, @@ -1423,7 +1591,7 @@ pub fn run_prefork( // Bound the POOL, not just each worker. // // `RLIMIT_AS` is per process, so N workers at a B-byte budget permit N*B in - // total — and nothing checked that against the machine. On a 5.9 GB host, 4 + // total, and nothing checked that against the machine. On a 5.9 GB host, 4 // workers at 2 GiB each permits 8 GiB, so the per-process backstop never // fires and the SYSTEM OOM killer picks workers off instead. That is strictly // worse: a `SIGKILL` cannot be caught, so the worker dies without answering @@ -1453,6 +1621,7 @@ pub fn run_prefork( install_handler(libc::SIGTERM, on_shutdown); install_handler(libc::SIGINT, on_shutdown); install_handler(libc::SIGHUP, on_sighup); + install_handler(libc::SIGUSR2, on_sigusr2); install_handler(libc::SIGCHLD, on_sigchld); let mut gen = Generation::new()?; @@ -1520,7 +1689,22 @@ pub fn run_prefork( } } - // Reload trigger: an explicit RELOAD/SIGHUP, or the data dir changed on + // Log rotation: every child holds its own handle. + if HUP_RECEIVED.swap(false, Ordering::Relaxed) { + for &pid in children + .iter() + .chain(side_pid.iter()) + .chain(retiring.keys()) + { + // SAFETY: signals a child of this process that has not been + // reaped, so the pid cannot have been reused. + unsafe { + libc::kill(pid, libc::SIGHUP); + } + } + } + + // Reload trigger: an explicit RELOAD/SIGUSR2, or the data dir changed on // disk (a sidecar wrote it). Coalesce both into one reload per tick. let disk_changed = match datadir.as_deref().and_then(datadir_mtime) { Some(t) => { @@ -1623,7 +1807,7 @@ pub fn run_prefork( /// Ask the kernel to kill this process when its parent goes. /// /// A worker spends its life blocked in `accept()`, so it cannot notice that the -/// supervisor has died — and when the supervisor is killed outright rather than +/// supervisor has died, and when the supervisor is killed outright rather than /// asked to stop (the OOM killer, `kill -9`, a container stop, a test harness /// dropping the child) it never gets to take its pool down with it. The workers /// are reparented to init and block forever on a socket nobody will connect to @@ -1631,7 +1815,7 @@ pub fn run_prefork( /// per run of this crate's daemon tests, accumulating across runs until the /// process table or the descriptor ceiling gives out. /// -/// `PR_SET_PDEATHSIG` is Linux-only and there is no portable equivalent — the +/// `PR_SET_PDEATHSIG` is Linux-only and there is no portable equivalent: the /// alternatives all need the worker to be watching a descriptor it is not /// watching. On other Unixes a supervisor killed outright still leaves its /// workers behind; one asked to stop takes them with it as it always did. @@ -1652,7 +1836,7 @@ fn die_with_parent(supervisor: libc::pid_t) { } // The parent can die between `fork` and the `prctl`, in which case the // signal it would have raised was already missed and this worker would - // block forever — the same leak through a narrower window. Comparing + // block forever: the same leak through a narrower window. Comparing // against the pid the supervisor recorded of itself is what makes that // check sound: a parent that is no longer *that* process is gone, // whatever pid took its place. @@ -1684,14 +1868,9 @@ fn spawn_side( -1 => Err(io::Error::last_os_error()), 0 => { let retire = gen.enter(); - unsafe { - libc::signal(libc::SIGTERM, libc::SIG_DFL); - libc::signal(libc::SIGINT, libc::SIG_DFL); - libc::signal(libc::SIGHUP, libc::SIG_DFL); - libc::signal(libc::SIGCHLD, libc::SIG_DFL); - } + child_signals(); // After the resets above, so the SIGTERM it arranges is the default - // disposition — a handler inherited from the supervisor would + // disposition; a handler inherited from the supervisor would // otherwise decide what parent death means here. die_with_parent(supervisor); (side.serve)(Arc::clone(db), Arc::clone(opts), retire); @@ -1744,7 +1923,7 @@ fn worker_main( // configured value would give a scan only what is left over rather than the // budget the setting names, and jobs would be killed for exceeding a ceiling // nobody asked for. Adding what the process already uses makes the number - // mean what it says — memory available TO A SCAN. + // mean what it says: memory available TO A SCAN. // // Zero means "no limit", so it must not be added to: `already` is non-zero, // and the sum would pin the address space at the worker's current footprint, @@ -1756,17 +1935,13 @@ fn worker_main( cfg.max_memory_bytes.saturating_add(already), ); } - set_rlimit(libc::RLIMIT_CPU as libc::c_int, cfg.max_cpu_secs); install_handler(libc::SIGALRM, on_sigalrm); + install_handler(libc::SIGXCPU, on_sigxcpu); // Report an allocation failure to the client instead of dying silently. install_handler(libc::SIGABRT, on_sigabrt); - // Replace the handlers inherited from the supervisor, so its kill - // terminates us and its flag-setting handlers don't fire here. - unsafe { - libc::signal(libc::SIGINT, libc::SIG_DFL); - libc::signal(libc::SIGHUP, libc::SIG_DFL); - libc::signal(libc::SIGCHLD, libc::SIG_DFL); - } + // Replace the handlers inherited from the supervisor, so its flag-setting + // handlers don't fire here. + child_signals(); install_handler(libc::SIGTERM, on_sigterm); // Only now, with SIGTERM handled here: arranged before, parent death would // run the handler this worker inherited from the supervisor, which only @@ -1775,6 +1950,7 @@ fn worker_main( die_with_parent(supervisor); let arm = || { IN_JOB.store(true, std::sync::atomic::Ordering::Relaxed); + arm_cpu_limit(cfg.max_cpu_secs); set_timer(cfg.max_scan_time) }; let disarm = || { @@ -1784,7 +1960,7 @@ fn worker_main( // A `RELOAD` command reaches a worker, not the supervisor; forward it by // signalling the parent, which owns the pool and does the re-fork. let reload = || unsafe { - libc::kill(libc::getppid(), libc::SIGHUP); + libc::kill(libc::getppid(), libc::SIGUSR2); }; // `SHUTDOWN` likewise forwards to the supervisor (SIGTERM → graceful // teardown of the whole pool). Disabled → report it wasn't honoured. @@ -1898,6 +2074,7 @@ fn log_child_exit(pid: libc::pid_t, status: libc::c_int, what: &str, retired: bo let cause = if libc::WIFEXITED(status) { match libc::WEXITSTATUS(status) { EXIT_TIMEOUT => "scan wall-clock timeout".to_string(), + EXIT_CPU => "scan CPU-time limit".to_string(), EXIT_ABORTED => "aborted mid-scan (reported to the client)".to_string(), EXIT_STOPPED => "stopped".to_string(), 0 if retired => "retired after a reload".to_string(), @@ -1906,7 +2083,9 @@ fn log_child_exit(pid: libc::pid_t, status: libc::c_int, what: &str, retired: bo } } else if libc::WIFSIGNALED(status) { match libc::WTERMSIG(status) { - libc::SIGKILL => "killed (OOM / RLIMIT_AS)".to_string(), + // RLIMIT_AS fails an allocation (SIGABRT below); SIGKILL comes from + // the system OOM killer, a hard RLIMIT_CPU or an operator. + libc::SIGKILL => "killed by SIGKILL (system OOM killer or operator)".to_string(), libc::SIGXCPU => "CPU-time limit (RLIMIT_CPU)".to_string(), libc::SIGABRT => "aborted (allocation failure under RLIMIT_AS)".to_string(), sig => format!("signal {sig}"), @@ -1956,7 +2135,7 @@ fn read_command(r: &mut BufReader) -> io::Result( // commands the daemon speaks. Format matches clamd: `| COMMANDS: // `. // The list is what a client uses to decide what it may send, so a verb - // that works and is missing here is a capability nobody discovers — and + // that works and is missing here is a capability nobody discovers, and // a verb that is listed but refused is a capability nobody can use. // `SCANURL` is therefore listed only when this daemon would honour it: // built with `http-scan` and started with `--allow-http-scan`. @@ -2252,7 +2431,7 @@ fn dispatch( // // The clamd fields keep their positions and their names, because // `clamdtop` finds its columns by both. What exav knows and clamd does - // not — where this process spent its scan time — follows them as + // not (where this process spent its scan time) follows them as // `SCANSTATS`/`MATCHERSTATS` lines, which a clamd client ignores and a // human or a scraper can read. `QUEUE` reports the scans actually in // flight rather than the constant `0 items` a stub would give, since a @@ -2278,7 +2457,7 @@ fn dispatch( "SCAN" => vec![scan_one_path(db, opts, arg)], "CONTSCAN" | "MULTISCAN" => scan_tree(db, opts, arg), // All-match: report every matching signature per file (not just the - // first), one reply line each — clamd's ALLMATCHSCAN semantics. + // first), one reply line each: clamd's ALLMATCHSCAN semantics. "ALLMATCHSCAN" => scan_tree_allmatch(db, opts, arg), "INSTREAM" => vec![instream(db, opts, reader)?], // Extended INSTREAM: same chunk framing, structured JSON reply with the @@ -2315,7 +2494,7 @@ fn write_reply(w: &mut W, id: Option, reply: &str, delim: Delim) // breaks clamd-session clients like clamdtop, which then see // `2: STATE:`/`2: THREADS:` instead of the bare field lines and can't parse // the body. Each CONTSCAN file result is a separate reply message (its own - // call here), so it still gets its own id — only intra-message lines change. + // call here), so it still gets its own id; only intra-message lines change. if let Some(n) = id { write!(w, "{n}: ")?; } @@ -2328,7 +2507,7 @@ fn write_reply(w: &mut W, id: Option, reply: &str, delim: Delim) /// [`exav_core::Verdict`] classification. A partial verdict is surfaced as /// ` () ERROR` (never `OK`) so the never-silent-skip invariant /// holds on the wire, and the tag/detail come from the same source the one-shot -/// CLI uses — the two surfaces cannot drift apart. +/// CLI uses, so the two surfaces cannot drift apart. fn verdict_line(target: &str, report: &ScanReport) -> String { let mut owned; let report = if report.verdict.category() == VerdictCategory::Partial { @@ -2342,12 +2521,12 @@ fn verdict_line(target: &str, report: &ScanReport) -> String { match v.category() { VerdictCategory::Infected => format!("{target}: {} FOUND", v.detail().unwrap_or_default()), VerdictCategory::Clean => format!("{target}: OK"), - // The same grammar the one-shot CLI prints — `reason CATEGORY STATUS` — + // The same grammar the one-shot CLI prints (`reason CATEGORY STATUS`), // so the two surfaces do not describe one verdict two ways. Only the // status word differs, and it has to: clamd's vocabulary is `OK`, // `FOUND` and `ERROR`, and a real client reads a word outside it as // `OK`. Measured against clamdscan 1.4.3, which rewrites an unknown - // status to `OK` and exits 0 — so `PARTIAL` on this wire would turn + // status to `OK` and exits 0, so `PARTIAL` on this wire would turn // exav's fail-closed answer into a fail-open one at every existing // client. `ERROR` is the only word here that fails closed. // @@ -2398,7 +2577,7 @@ fn scan_one_path(db: &Scanner, opts: &ScanOptions, path: &str) -> String { /// Scanning a directory one file at a time misses one whole class of archive: a /// set split across `big.7z.001`, `.002`, `.003` has no piece that decodes on /// its own, so every part reports `OK` and the archive is never opened. Sets are -/// therefore rejoined per directory and scanned as the archives they are — see +/// therefore rejoined per directory and scanned as the archives they are. See /// [`volume_set_lines`] for how the result is attributed back to files. fn scan_tree(db: &Scanner, opts: &ScanOptions, path: &str) -> Vec { if path.is_empty() { @@ -2438,7 +2617,7 @@ fn scan_tree(db: &Scanner, opts: &ScanOptions, path: &str) -> Vec { /// Rejoin the multi-volume sets among `files` and scan each as one archive. /// /// Returns the status text to report for each part of a set that resolved to -/// something other than clean — keyed by the part's own path, so the caller +/// something other than clean, keyed by the part's own path, so the caller /// still emits exactly one reply line per file. A piece of an infected archive /// is not a clean file, and it is the piece the operator has to act on. /// @@ -2478,7 +2657,7 @@ fn volume_set_lines( // operator what was actually found, which `big.7z.002` does not. // // Built by `verdict_line` so a part's reply and a whole file's reply - // share one grammar — including `--partial-as`, which `verdict_line` + // share one grammar, including `--partial-as`, which `verdict_line` // applies. The empty target leaves a bare `": "` in front for the // caller to fill in with the part's path. let line = verdict_line("", &v.report); @@ -2520,32 +2699,31 @@ fn scan_tree_allmatch(db: &Scanner, opts: &ScanOptions, path: &str) -> Vec Vec { if path.is_empty() { return vec!["ALLMATCHSCAN: missing path ERROR".to_string()]; } - let cap = opts.deep_analysis_max; - let mut data = Vec::new(); - let read = File::open(Path::new(path)).and_then(|f| { - f.take(cap.saturating_add(1)) - .read_to_end(&mut data) - .map(|_| ()) - }); - if let Err(e) = read { - return vec![format!("{path}: {e} ERROR")]; - } - if data.len() as u64 > cap { - // Too big to buffer for all-match; single-match scan (never skip). + let opened = File::open(Path::new(path)).and_then(|f| f.metadata().map(|m| (f, m.len()))); + let (file, size) = match opened { + Ok(o) => o, + Err(e) => return vec![format!("{path}: {e} ERROR")], + }; + if opts.max_scan_size.is_some_and(|max| size > max) { return vec![scan_one_path(db, opts, path)]; } // Isolate a panic on a crafted file into an ERROR for this target. let found = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - analyze_all_with_outcome(db, &data, opts) + analyze_all_seekable(db, file, size, opts) })); + let found = match found { + Ok(Ok(found)) => Ok(found), + Ok(Err(e)) => return vec![format!("{path}: {e} ERROR")], + Err(panic) => Err(panic), + }; match found { Ok((dets, _)) if !dets.is_empty() => dets .into_iter() @@ -2553,16 +2731,18 @@ fn scan_one_allmatch(db: &Scanner, opts: &ScanOptions, path: &str) -> Vec vec![match outcome { - AllMatchOutcome::Complete => format!("{path}: OK"), - AllMatchOutcome::LimitsExceeded(r) => format!("{path}: {r} LIMITS-EXCEEDED"), - AllMatchOutcome::Unscannable(r) => format!("{path}: {r} UNSCANNABLE"), - AllMatchOutcome::PasswordProtected(r) => { - format!("{path}: {r} PASSWORD-PROTECTED") - } - }], + // The same line a single-match scan gives, `--partial-as` included. + Ok((_, outcome)) => { + let verdict = match outcome { + AllMatchOutcome::Complete => Verdict::Clean, + AllMatchOutcome::LimitsExceeded(reason) => Verdict::LimitsExceeded { reason }, + AllMatchOutcome::Unscannable(reason) => Verdict::Unscannable { reason }, + AllMatchOutcome::PasswordProtected(reason) => Verdict::PasswordProtected { reason }, + }; + vec![verdict_line(path, &ScanReport::new(verdict, Vec::new()))] + } Err(_) => vec![format!("{path}: scan failed (internal error) ERROR")], } } @@ -2597,7 +2777,7 @@ fn scan_url(db: &Scanner, opts: &ScanOptions, url: &str) -> String { /// timeout and so never trips it. pub(crate) const MAX_DRAIN_BYTES: u64 = 64 * 1024 * 1024; -/// A stream payload materialized into a **seekable** source — in RAM when small, +/// A stream payload materialized into a **seekable** source: in RAM when small, /// else spilled to an auto-deleting temp file. A seekable view is what lets a /// streamed input (INSTREAM/EXINSTREAM, and the CLI's stdin) get full /// container-aware scanning (a ZIP's central directory is at the end) at ANY size @@ -2631,6 +2811,23 @@ impl StreamPayload { } } } + + /// Every detection in the payload, at any size (see + /// [`exav_core::analyze_all_seekable`]). + pub(crate) fn all_matches( + &self, + db: &Scanner, + opts: &ScanOptions, + ) -> io::Result<(Vec<(String, exav_core::Method)>, AllMatchOutcome)> { + match self { + StreamPayload::Mem(v) => { + exav_core::analyze_all_seekable(db, std::io::Cursor::new(&v[..]), self.len(), opts) + } + StreamPayload::Disk(tmp, n) => { + exav_core::analyze_all_seekable(db, tmp.reopen()?, *n, opts) + } + } + } } /// A stream read into a seekable payload, as far as it could be held. @@ -2758,7 +2955,7 @@ pub(crate) fn scan_payload( /// [`scan_held`], timed into the process counters. /// /// The ICAP listener times its own scans (it names the object from the request -/// target, which this cannot see), so this is the daemon's stream verbs only — +/// target, which this cannot see), so this is the daemon's stream verbs only: /// double-counting a scan would make the throughput figure a fiction. fn scan_payload_timed( db: &Scanner, @@ -2780,7 +2977,7 @@ fn scan_payload_timed( } /// Scan an INSTREAM chunk stream. The payload is materialized to a seekable -/// source (RAM or a temp file) and given the full container-aware scan — so +/// source (RAM or a temp file) and given the full container-aware scan, so /// malware inside an archive sent over INSTREAM is detected, matching clamd (the /// old flat-only path missed it). Any unread chunks are drained so the connection /// stays in sync for the next command. @@ -2805,20 +3002,18 @@ fn instream( } /// `EXINSTREAM`: scan a file sent over the exact INSTREAM chunk framing and reply -/// with one line of compact JSON — `{"v":1,"verdict":...}`. Unlike INSTREAM's -/// flat scan, the payload is buffered (bounded by `--max-input-bytes`) and run -/// through the full container-aware analysis, so a detection carries its nested -/// `location` (the `/`-joined member path). The verdict *classification* matches -/// INSTREAM: a detection beats a limit; a not-fully-scanned stream is -/// `unscannable`, never `clean`. An over-limit stream is `unscannable` (never -/// clean). One detection per scan (first / most relevant). +/// with one line of compact JSON, `{"v":1,"status":...}` (module docs). The +/// payload is buffered and scanned as INSTREAM's is, and a detection also +/// carries its nested `location` (the `/`-joined member path). A detection beats +/// a limit, a stream not fully examined is `PARTIAL` with its category, never +/// `OK`, and one cut before its terminator is `ERROR`. One detection per scan. fn exinstream( db: &Scanner, opts: &ScanOptions, reader: &mut BufReader, ) -> io::Result { - // Materialize to a seekable source (RAM small / temp file large) — same path - // as INSTREAM — bounded by `--max-input-bytes` (disk is the ceiling). That is + // Materialize to a seekable source (RAM small / temp file large), the same + // path as INSTREAM, bounded by `--max-input-bytes` (disk is the ceiling). That is // the flag this cap actually reads: `opts.max_scan_size` is exav-core's // per-top-level-file bound, which `--max-input-bytes` sets. Naming the other // flag here would send an operator to raise a setting that changes nothing. @@ -2902,7 +3097,7 @@ fn exinstream_multi( let mut payloads: Vec> = Vec::new(); let mut entries: Vec = Vec::new(); // Set when the request stops before its terminator. Every file already read - // still gets its verdict — those streams were complete — but the request as + // still gets its verdict (those streams were complete), but the request as // a whole is answered as incomplete, because the files that never arrived // are indistinguishable from files the client chose not to send. let mut request_truncated = false; @@ -3008,7 +3203,7 @@ fn exinstream_multi( } let mut out = json!({"v": 1, "files": entries}); if request_truncated { - // The per-file verdicts above stand — those streams arrived whole. This + // The per-file verdicts above stand: those streams arrived whole. This // says the LIST is short: files the client meant to send never got here, // and a caller treating "every file came back clean" as "the batch is // clean" would be wrong about a batch it never fully sent. @@ -3045,7 +3240,7 @@ fn verdict_fields(report: &ScanReport, location: Option) -> serde_json:: report }; let v = &report.verdict; - // `status` / `category` / `reason` — the same three names the one-shot + // `status` / `category` / `reason`: the same three names the one-shot // `--json` uses, so one schema describes both. The old `verdict`/`tag`/ // `message` trio named the same things differently here, and its // `"unscannable"` collided with the *category* of that name: a @@ -3084,8 +3279,8 @@ fn verdict_fields(report: &ScanReport, location: Option) -> serde_json:: /// Render a scan verdict as one line of compact JSON for `EXINSTREAM`. fn verdict_json(report: &ScanReport, location: Option) -> String { // `"v"` is the schema version every reply carries. Key *order* is not part - // of the contract — the object is serialised with sorted keys, and a JSON - // consumer reads by name — so nothing here depends on where it lands. + // of the contract: the object is serialised with sorted keys, and a JSON + // consumer reads by name, so nothing here depends on where it lands. let mut o = serde_json::json!({"v": 1}); if let Some(fields) = verdict_fields(report, location).as_object() { for (k, val) in fields { @@ -3113,7 +3308,7 @@ struct Instream<'a, R: Read> { /// The stream ended WITHOUT its zero-length terminator: the client closed or /// was cut off part-way through. The bytes received are a prefix of the file /// the client meant to send, so a verdict on them is a verdict on something - /// nobody chose to scan — and `OK` on a prefix is the shape of a bypass: + /// nobody chose to scan, and `OK` on a prefix is the shape of a bypass: /// send the benign head of a file, hang up, collect a clean answer. The /// caller must reply with an error instead. truncated: bool, @@ -3144,7 +3339,7 @@ impl<'a, R: Read> Instream<'a, R> { fn next_chunk(&mut self) -> io::Result<()> { let mut len = [0u8; 4]; if read_full(self.inner, &mut len)? < 4 { - // The client stopped before the terminator — either at a chunk + // The client stopped before the terminator, either at a chunk // boundary or part-way through a length. A well-formed INSTREAM ends // with a zero-length chunk, so either way this is not the file the // client set out to send. @@ -3162,16 +3357,16 @@ impl<'a, R: Read> Instream<'a, R> { /// Consume any remaining chunks up to the terminator (used when the scan /// stopped early on a detection), up to [`MAX_DRAIN_BYTES`]. /// - /// Draining is a courtesy — it lets the connection carry another command - /// instead of being reset — and a courtesy with no bound is a way to hold a + /// Draining is a courtesy (it lets the connection carry another command + /// instead of being reset), and a courtesy with no bound is a way to hold a /// worker. Every read completes inside the socket timeout, so the timeout /// never fires while a client keeps sending; without a cap one connection /// can occupy a slot for as long as it likes, and in the thread model there /// is no per-command timer behind it. /// /// Past the cap the reply still goes out and the rest of the client's bytes - /// are left in the socket. The session is then desynchronised — the leftover - /// payload parses as garbage commands — which is the client's own doing and + /// are left in the socket. The session is then desynchronised (the leftover + /// payload parses as garbage commands), which is the client's own doing and /// costs it a byte for every byte the daemon reads. What it no longer buys /// is a worker held for free. /// @@ -3282,6 +3477,27 @@ mod tests { ); } + /// Every cap on what a scan holds fits the grant, not only the total: the + /// engine reads an object whole up to `deep_analysis_max`, and a total + /// already under the grant says nothing about one object's cap. + #[test] + fn every_holding_cap_fits_the_grant() { + let mut opts = ScanOptions::default(); + opts.limits.max_extracted_bytes = 1024 << 20; + opts.deep_analysis_max = 256 << 20; + fit_limits_to_job_memory(&mut opts, 300 << 20); + assert_eq!(opts.deep_analysis_max, 150 << 20); + + let mut opts = ScanOptions::default(); + opts.limits.max_extracted_bytes = 100 << 20; + opts.limits.max_buffer_bytes = u64::MAX; + opts.deep_analysis_max = u64::MAX; + fit_limits_to_job_memory(&mut opts, 1024 << 20); + assert_eq!(opts.limits.max_extracted_bytes, 100 << 20); + assert_eq!(opts.limits.max_buffer_bytes, 512 << 20); + assert_eq!(opts.deep_analysis_max, 512 << 20); + } + /// A signal that lands while the supervisor is awake (reaping, reloading) /// still cuts its next nap short, so a crashed worker is replaced and a /// `RELOAD` served at once rather than a tick later. SIGUSR2 stands in for @@ -3516,7 +3732,7 @@ mod tests { // Real DEFLATE ZIPs (generated by Python's zipfile). Compressed so the EICAR // is NOT visible to a flat scan of the container, forcing real extraction of - // each level — which is what exercises the nested-location chain. + // each level, which is what exercises the nested-location chain. // `ZIP_EICAR_INSIDE`: a `.zip` with member `inside.txt` = EICAR. const ZIP_EICAR_INSIDE: &[u8] = &[ 80, 75, 3, 4, 20, 0, 0, 0, 8, 0, 230, 190, 244, 92, 60, 207, 81, 104, 70, 0, 0, 0, 68, 0, @@ -3548,9 +3764,9 @@ mod tests { #[test] fn instream_extracts_archives() { - // INSTREAM must detect malware INSIDE an archive, as clamd does — a + // INSTREAM must detect malware INSIDE an archive, as clamd does; a // flat-only scan of the stream is not enough. EICAR is DEFLATE-compressed inside - // the zip, so a flat scan of the raw stream can't see it — extraction is + // the zip, so a flat scan of the raw stream can't see it: extraction is // required. let r = one(&instream_msg(ZIP_EICAR_INSIDE), 0); assert!( @@ -3633,15 +3849,15 @@ mod tests { #[test] fn exinstream_multi_rejoins_a_split_archive() { - // The reason the verb exists. Each part on its own decodes to nothing — - // asserted below — so three separate EXINSTREAM calls would all reply - // `clean` and the archive would never be opened. + // The reason the verb exists. Each part on its own decodes to nothing + // (asserted below), so three separate EXINSTREAM calls would all reply + // `OK` and the archive would never be opened. let parts = split_set("payload", ZIP_EICAR_INSIDE, 3); for (name, part) in &parts { let solo = one(&exinstream_msg(part), 0); assert!( solo.contains(r#""status":"OK""#), - "{name} is detectable alone — the fixture proves nothing: {solo}" + "{name} is detectable alone, so the fixture proves nothing: {solo}" ); } let msg = exinstream_multi_msg( @@ -3869,7 +4085,7 @@ mod tests { #[test] fn contscan_rejoins_a_split_archive_in_a_directory() { // A directory holding `payload.zip.001..003` is one archive, not three - // files. Scanned one at a time — which is all CONTSCAN did — no part + // files. Scanned one at a time (which is all CONTSCAN did), no part // decodes and every line reads `OK`. let dir = crate::tmpfile::TempDir::new().unwrap(); for (name, part) in split_set("payload", ZIP_EICAR_INSIDE, 3) { @@ -4146,7 +4362,7 @@ mod tests { #[test] fn scanurl_allowed_attempts_fetch() { // Allowed: the daemon tries to fetch rather than refusing. A closed - // loopback port fails fast with a fetch ERROR — the point is the + // loopback port fails fast with a fetch ERROR; the point is the // request got past the gate, not that it downloaded anything. let r = one_allow_http_scan(b"zSCANURL http://127.0.0.1:9/x\0", 0, true); assert!(r.contains("ERROR") && !r.contains("disabled"), "got {r}"); @@ -4188,10 +4404,31 @@ mod tests { assert!(rg.trim_end_matches('\0').ends_with("OK"), "got {rg}"); } + /// A file ALLMATCHSCAN could not fully examine is answered in the same + /// words as SCAN, ending in `ERROR`: a bare category is a word clamd + /// clients read as `OK`. + #[test] + fn allmatchscan_ends_a_partial_in_error() { + let dir = crate::tmpfile::TempDir::new().unwrap(); + let damaged = dir.path().join("damaged.gz"); + let mut gz = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]; + gz.extend_from_slice(&[0xff; 64]); + std::fs::write(&damaged, gz).unwrap(); + let path = damaged.to_string_lossy(); + let db = Scanner::builtin(); + let lines = scan_one_allmatch(&db, &ScanOptions::default(), &path); + assert_eq!( + lines, + [scan_one_path(&db, &ScanOptions::default(), &path)], + "ALLMATCHSCAN and SCAN disagree" + ); + assert!(lines[0].ends_with(" UNSCANNABLE ERROR"), "{lines:?}"); + } + #[test] fn shutdown_disabled_reports_error_and_does_not_exit() { // With the hook returning false (disabled), SHUTDOWN must reply an ERROR - // rather than silently succeeding — and obviously not kill the test. + // rather than silently succeeding, and obviously not kill the test. let (mut client, server) = UnixStream::pair().unwrap(); let db = Scanner::builtin(); let opts = ScanOptions::default(); diff --git a/crates/exav/src/endpoint.rs b/crates/exav/src/endpoint.rs index bf9818f8..00bc815b 100644 --- a/crates/exav/src/endpoint.rs +++ b/crates/exav/src/endpoint.rs @@ -163,7 +163,7 @@ impl Endpoint { if opts.mode.is_some() { return Err(format!( "`{s}`: `mode` sets the permissions of a socket file, and a host:port \ - listener has none — restrict it with the bind address and a firewall" + listener has none: restrict it with the bind address and a firewall" )); } let (hostport, path) = match rest.split_once('/') { @@ -206,8 +206,8 @@ fn check_host_port(hostport: &str, whole: &str) -> Result { } Err(format!( "`{whole}`: `{hostport}` has no port, so nothing can be bound to it. If you \ - meant several ICAP services, a comma separates addresses rather than names \ - — repeat the key instead: `?service=…&service=…`" + meant several ICAP services, a comma separates addresses rather than names: \ + repeat the key instead, `?service=…&service=…`" )) } @@ -237,7 +237,7 @@ fn resolve_services( }; return Err(format!( "`{whole}`: the clamd protocol has no services, so {given} means nothing here \ - — an `icap://` address is the one that takes `/`" + (an `icap://` address is the one that takes `/`)" )); } match (path, query) { @@ -333,7 +333,7 @@ fn parse_service(v: &str, whole: &str) -> Result { if name.contains(',') { return Err(format!( "`{whole}`: a comma separates addresses, not service names; repeat the key \ - instead — `?service=…&service=…`" + instead: `?service=…&service=…`" )); } if name.contains('/') { diff --git a/crates/exav/src/icap/config.rs b/crates/exav/src/icap/config.rs index 7b7791eb..663846ac 100644 --- a/crates/exav/src/icap/config.rs +++ b/crates/exav/src/icap/config.rs @@ -56,17 +56,19 @@ pub(crate) struct IcapConfig { /// pointing at those aliases are not going to be reconfigured just because /// the server behind them changed. pub services: Vec, - /// Bytes advertised in the `Preview` header of an `OPTIONS` response — how - /// much of a body a client should send before pausing for a verdict. - pub preview_size: usize, + /// Bytes advertised in the `Preview` header of an `OPTIONS` response: how + /// much of a body a client should send before pausing for a verdict. `None` + /// leaves the header out. + pub preview_size: Option, /// Value of the `Transfer-Preview` header in an `OPTIONS` response. `*` /// asks the client to preview every object. pub transfer_preview: String, /// Concurrent connections accepted, and the value advertised as /// `Max-Connections`. c-icap `MaxServers` x `ThreadsPerChild`. pub max_connections: usize, - /// Seconds a client may cache the `OPTIONS` answer (`Options-TTL`). - pub options_ttl: u32, + /// Seconds a client may cache the `OPTIONS` answer (`Options-TTL`). `None` + /// leaves the header out, which RFC 3507 reads as never expiring. + pub options_ttl: Option, /// Requests served on one connection before it is closed. c-icap /// `MaxKeepAliveRequests`. pub keepalive_requests: u64, @@ -116,10 +118,10 @@ impl Default for IcapConfig { "srv_clamav".to_string(), "virus_scan".to_string(), ], - preview_size: 4096, + preview_size: Some(4096), transfer_preview: "*".to_string(), max_connections: 100, - options_ttl: 3600, + options_ttl: Some(3600), keepalive_requests: 100, idle_timeout: Duration::from_secs(600), max_drain_bytes: crate::daemon::MAX_DRAIN_BYTES, diff --git a/crates/exav/src/icap/mod.rs b/crates/exav/src/icap/mod.rs index 422eb5a3..3a4ea6db 100644 --- a/crates/exav/src/icap/mod.rs +++ b/crates/exav/src/icap/mod.rs @@ -90,12 +90,10 @@ // annotation. The lint level applies to every submodule below. #![forbid(unsafe_code)] -use std::path::PathBuf; -use std::process::ExitCode; use std::sync::Arc; use std::time::Duration; -use exav_core::{ScanOptions, Scanner}; +use exav_core::ScanOptions; use crate::Cli; @@ -108,16 +106,12 @@ mod wire; pub(crate) use self::config::{IcapConfig, InfectionHeader}; pub(crate) use self::server::Server; -use self::server::{FixedDb, ReloadableDb, Signatures}; +use self::server::{FixedDb, Signatures}; +use crate::daemon::SharedDb; /// Version string reported in the `Service` and `Server` headers. const VERSION: &str = env!("CARGO_PKG_VERSION"); -/// How often the signature source is polled for a change. Matches the clamd -/// daemon's supervisor tick, so both surfaces pick up a sidecar's write at the -/// same rate. -const RELOAD_TICK: Duration = Duration::from_secs(10); - /// Build the server configuration from the parsed arguments. pub fn config_from_cli(cli: &Cli) -> Result { let d = IcapConfig::default(); @@ -141,7 +135,7 @@ pub fn config_from_cli(cli: &Cli) -> Result { _ => d.listen, }, services, - preview_size: cli.icap_preview_size.unwrap_or(d.preview_size), + preview_size: cli.icap_preview_size.map_or(d.preview_size, |p| p.value()), // `off` omits the header. An empty string used to be the way to say // that, which is a sentinel nobody guesses and a shell quoting accident // away from being set by mistake. @@ -158,21 +152,17 @@ pub fn config_from_cli(cli: &Cli) -> Result { .as_ref() .and_then(|e| e.max_connections) .unwrap_or(d.max_connections), - options_ttl: cli.icap_options_ttl.unwrap_or(d.options_ttl), - keepalive_requests: cli - .icap_keepalive_requests - .unwrap_or(d.keepalive_requests) - .max(1), - idle_timeout: Duration::from_secs( - cli.icap_idle_timeout - .unwrap_or(d.idle_timeout.as_secs()) - .max(1), - ), + options_ttl: cli.icap_options_ttl.map_or(d.options_ttl, |t| t.value()), + keepalive_requests: match cli.icap_keepalive_requests { + Some(0) => u64::MAX, + Some(n) => n, + None => d.keepalive_requests, + }, + idle_timeout: cli + .icap_idle_timeout + .map_or(d.idle_timeout, Duration::from_secs), max_drain_bytes: d.max_drain_bytes, - max_header_bytes: cli - .icap_max_header_size - .unwrap_or(d.max_header_bytes) - .max(1024), + max_header_bytes: cli.icap_max_header_size.unwrap_or(d.max_header_bytes), service_label: d.service_label, infection_header: cli.icap_infection_header.unwrap_or(d.infection_header), partial_as: cli.partial_as.unwrap_or(d.partial_as), @@ -192,13 +182,14 @@ pub fn bind(cfg: IcapConfig) -> std::io::Result { fn announce(server: &Server) { let cfg = server.config(); eprintln!( - "exav: serving ICAP on tcp:{} (services: {}; preview {} B)", + "exav: serving ICAP on tcp:{} (services: {}; preview {})", server .local_addr() .map(|a| a.to_string()) .unwrap_or_else(|_| cfg.listen.clone()), cfg.services.join(", "), - cfg.preview_size, + cfg.preview_size + .map_or_else(|| "off".to_string(), |n| format!("{n} B")), ); // Said at startup as well as per object, because a deployment that passes // what it could not examine should be legible from its logs alone — nobody @@ -213,25 +204,12 @@ fn announce(server: &Server) { } } -/// Serve ICAP on its own listener thread, and make this thread the supervisor -/// that watches the signature source and swaps the database in — which is also -/// what moves the `ISTag` and so invalidates every downstream ICAP cache. -/// -/// Never returns: the process serves until it is stopped. -pub fn serve_alone( - server: Server, - db: Arc, - opts: Arc, - watch: Option, - reload: &dyn Fn() -> Result, - metrics_interval: Duration, -) -> ExitCode { +/// Serve ICAP on a listener thread of its own, over the database the thread +/// model's supervisor ([`crate::daemon::supervise`]) swaps on a reload. A swap +/// moves the `ISTag` and so invalidates every downstream ICAP cache. +pub fn spawn(server: Server, db: Arc, opts: Arc) { announce(&server); - // Serving ICAP alone binds no clamd port, so `STATS` cannot be asked here at - // all and the log is the only channel the totals have. - crate::metrics::spawn_reporter(metrics_interval, "icap"); - let shared = Arc::new(ReloadableDb::from_arc(db)); - let handle: Arc = Arc::clone(&shared) as Arc; + let handle: Arc = db; std::thread::spawn(move || { if let Err(e) = server.run(handle, opts, &|_| true) { eprintln!("exav: icap: listener stopped: {e}"); @@ -241,37 +219,6 @@ pub fn serve_alone( // orchestrator can only restart what it can see has stopped. std::process::exit(2); }); - - let Some(dir) = watch else { - // Nothing to watch (the built-in baseline). Park instead of returning, - // because returning would drop into the caller's exit path. - loop { - std::thread::sleep(RELOAD_TICK); - } - }; - - let mut last = crate::daemon::datadir_mtime(&dir); - loop { - std::thread::sleep(RELOAD_TICK); - let Some(now) = crate::daemon::datadir_mtime(&dir) else { - continue; - }; - if crate::daemon::source_changed(last, now) { - last = Some(now); - match reload() { - Ok(new_db) => { - eprintln!( - "exav: icap: reloading signatures ({} known)", - new_db.signature_count() - ); - shared.replace(new_db); - } - // Keeping the database already loaded is the safe failure: a - // half-written volume must not downgrade a running scanner. - Err(e) => eprintln!("exav: icap: signature reload failed, keeping current DB: {e}"), - } - } - } } /// Hand the bound listener to the prefork supervisor as a child process of its diff --git a/crates/exav/src/icap/response.rs b/crates/exav/src/icap/response.rs index 17fb2a3f..70672e9b 100644 --- a/crates/exav/src/icap/response.rs +++ b/crates/exav/src/icap/response.rs @@ -20,6 +20,9 @@ pub(super) enum Decision { /// tag (`LIMITS-EXCEEDED`, `UNSCANNABLE`, `PASSWORD-PROTECTED`) and the /// detail behind it. Partial(&'static str, String), + /// The scan itself failed. Blocked whatever `--partial-as` says: the + /// object was not examined at all. + Error(String), } impl Decision { @@ -52,7 +55,7 @@ impl Decision { match self { Self::Clean => crate::metrics::Category::Clean, Self::Infected(_) => crate::metrics::Category::Infected, - Self::Partial(_, _) => crate::metrics::Category::Partial, + Self::Partial(_, _) | Self::Error(_) => crate::metrics::Category::Partial, } } @@ -78,6 +81,15 @@ impl Decision { headers.extend(partial_headers(tag, reason)); headers } + Self::Error(reason) => { + let mut headers = Vec::with_capacity(3); + if policy == InfectionHeader::Blocks { + headers.push(infection_found("Heuristics.Exav.ScanError")); + } + headers.push(("X-Exav-Status", "ERROR".to_string())); + headers.push(("X-Exav-Reason", header_safe(reason))); + headers + } } } @@ -91,7 +103,7 @@ impl Decision { /// block, which is the opposite of what the operator asked for. fn pass_headers(&self) -> Vec<(&'static str, String)> { match self { - Self::Clean | Self::Infected(_) => Vec::new(), + Self::Clean | Self::Infected(_) | Self::Error(_) => Vec::new(), Self::Partial(tag, reason) => partial_headers(tag, reason), } } @@ -102,6 +114,7 @@ impl Decision { Self::Clean => "clean".to_string(), Self::Infected(name) => format!("infected: {}", header_safe(name)), Self::Partial(tag, reason) => format!("{tag}: {}", header_safe(reason)), + Self::Error(reason) => format!("ERROR: {}", header_safe(reason)), } } } @@ -343,9 +356,13 @@ pub(super) fn options(cfg: &IcapConfig) -> Response { .header("Methods", "REQMOD, RESPMOD") .header("Service", cfg.service_label.clone()) .header("Allow", "204") - .header("Preview", cfg.preview_size.to_string()) - .header("Max-Connections", cfg.max_connections.to_string()) - .header("Options-TTL", cfg.options_ttl.to_string()); + .header("Max-Connections", cfg.max_connections.to_string()); + if let Some(n) = cfg.preview_size { + r = r.header("Preview", n.to_string()); + } + if let Some(ttl) = cfg.options_ttl { + r = r.header("Options-TTL", ttl.to_string()); + } if !cfg.transfer_preview.is_empty() { r = r.header("Transfer-Preview", cfg.transfer_preview.clone()); } @@ -422,6 +439,13 @@ fn block_page(decision: &Decision) -> String { html_escape(reason) ), ), + Decision::Error(reason) => ( + "Scan failed", + format!( + "exav could not scan this object: {}. It is blocked.", + html_escape(reason) + ), + ), }; format!( "\n\ diff --git a/crates/exav/src/icap/server.rs b/crates/exav/src/icap/server.rs index 9a7488fa..2da8637b 100644 --- a/crates/exav/src/icap/server.rs +++ b/crates/exav/src/icap/server.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use std::io::{BufRead, BufReader, Write}; use std::net::{SocketAddr, TcpListener, TcpStream}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use exav_core::{scan_seekable_located, ScanOptions, Scanner}; @@ -48,30 +48,10 @@ impl Signatures for FixedDb { } } -/// A database that can be swapped underneath a running server. -pub(super) struct ReloadableDb(RwLock>); - -impl ReloadableDb { - /// Wrap a database already shared with another server, so one load answers - /// on every listener a process binds until the first reload replaces it. - pub(super) fn from_arc(db: Arc) -> Self { - Self(RwLock::new(db)) - } - - /// Install a freshly loaded database. Requests already in flight finish - /// against the database they started with. - pub(super) fn replace(&self, db: Scanner) { - // A poisoned lock here means a panic happened while swapping, not that - // the database is unusable, so the value is taken either way rather - // than turning a past panic into a permanently broken server. - let mut guard = self.0.write().unwrap_or_else(|e| e.into_inner()); - *guard = Arc::new(db); - } -} - -impl Signatures for ReloadableDb { +/// The thread model's database, which the supervisor swaps on a reload. +impl Signatures for crate::daemon::SharedDb { fn scanner(&self) -> Arc { - Arc::clone(&self.0.read().unwrap_or_else(|e| e.into_inner())) + self.current() } } @@ -502,7 +482,7 @@ fn modify( // now looking at a block they did not expect. eprintln!( "exav: icap: cannot pass an object past --max-input-bytes to a client that sent no \ - `Allow: 204` — the discarded tail is not available to hand back; blocking instead" + `Allow: 204`: the discarded tail is not available to hand back; blocking instead" ); } log_block(req, &decision); @@ -598,8 +578,10 @@ fn decide( ) -> Decision { let decision = match scanned { Ok(Ok((report, _loc))) => Decision::from_report(&report), - Ok(Err(e)) => Decision::Partial("UNSCANNABLE", format!("scan failed: {e}")), - Err(_) => Decision::Partial("UNSCANNABLE", "scan failed (internal error)".to_string()), + // Not a partial: nothing was examined, and `--partial-as ok` must not + // deliver it. + Ok(Err(e)) => Decision::Error(format!("scan failed: {e}")), + Err(_) => Decision::Error("scan failed (internal error)".to_string()), }; apply_found_policy(decision) } @@ -743,6 +725,17 @@ fn method_name(req: &RequestHead) -> &'static str { mod tests { use super::*; + /// A scan that failed is an error, not a partial: `--partial-as ok` passes + /// a partial, and must never pass an object nothing examined. + #[test] + fn a_failed_scan_is_an_error_that_blocks() { + let failed = decide(Ok(Err(std::io::Error::other("read failed")))); + assert!(matches!(failed, Decision::Error(_)), "{}", failed.summary()); + assert!(failed.blocks()); + let panicked = decide(Err(Box::new("boom"))); + assert!(matches!(panicked, Decision::Error(_))); + } + #[test] fn extracts_the_request_target() { assert_eq!( @@ -767,7 +760,7 @@ mod tests { #[test] fn a_reloadable_database_changes_its_istag() { - let db = ReloadableDb::from_arc(Arc::new(Scanner::builtin())); + let db = crate::daemon::SharedDb::new(Arc::new(Scanner::builtin())); let before = response::istag(&db.scanner()); let mut b = exav_core::loader::Builder::new(); diff --git a/crates/exav/src/main.rs b/crates/exav/src/main.rs index d11990e2..b66ff9ac 100644 --- a/crates/exav/src/main.rs +++ b/crates/exav/src/main.rs @@ -12,16 +12,17 @@ //! //! # `unsafe` //! -//! This is the one crate in the workspace without `#![forbid(unsafe_code)]`, -//! and [`daemon`] is the only module that accounts for it: the prefork pool is -//! `fork`, `waitpid`, `setrlimit`, `sigaction`, the `umask` that fixes the -//! socket's permissions at creation, and `SCM_RIGHTS` descriptor passing in -//! both directions (the daemon's `FILDES`, the client's `--send-as fd`), none of +//! This is the one crate in the workspace without `#![forbid(unsafe_code)]`. +//! [`daemon`] accounts for nearly all of it: the prefork pool is `fork`, +//! `waitpid`, `setrlimit`, `sigaction`, the `umask` that fixes the socket's +//! permissions at creation, and `SCM_RIGHTS` descriptor passing in both +//! directions (the daemon's `FILDES`, the client's `--send-as fd`), none of //! which has a safe binding that does not itself pull in a raw-syscall crate -//! larger than the code it replaces. Nothing on the scanning path is unsafe — -//! no scanned byte reaches any of it, and the engine, extractor, emulator and -//! decoder all forbid it outright. The `icap` module, whose parsers read -//! straight off the network, carries `#![deny(unsafe_code)]` of its own. +//! larger than the code it replaces. The rest is two `SIGPIPE` dispositions set +//! here. Nothing on the scanning path is unsafe: no scanned byte reaches any of +//! it, and the engine, extractor, emulator and decoder all forbid it outright. +//! The `icap` module, whose parsers read straight off the network, carries +//! `#![forbid(unsafe_code)]` of its own. mod daemon; mod endpoint; @@ -111,16 +112,25 @@ fn read_password_list(list: &std::path::Path) -> std::io::Result> { Ok(s.lines().map(str::to_string).collect()) } -/// The `--log` sink. A process scans once, so one lazily-opened handle is the -/// whole mechanism; `None` means no `--log` was given. -static LOG_FILE: std::sync::Mutex> = std::sync::Mutex::new(None); +/// The `--log` sink, and the path it was opened from; `None` means no `--log` +/// was given. +static LOG_FILE: std::sync::Mutex> = std::sync::Mutex::new(None); -fn log_open(path: &std::path::Path) -> std::io::Result<()> { - let f = std::fs::OpenOptions::new() +/// Set by `SIGHUP` in a listener: reopen the `--log` file before the next line, +/// so a rotated log is followed, as clamd does. +pub(crate) static LOG_REOPEN: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +fn log_file(path: &std::path::Path) -> std::io::Result { + std::fs::OpenOptions::new() .create(true) .append(true) - .open(path)?; - *LOG_FILE.lock().unwrap_or_else(|e| e.into_inner()) = Some(f); + .open(path) +} + +fn log_open(path: &std::path::Path) -> std::io::Result<()> { + let f = log_file(path)?; + *LOG_FILE.lock().unwrap_or_else(|e| e.into_inner()) = Some((f, path.to_path_buf())); Ok(()) } @@ -135,7 +145,15 @@ fn log_open(path: &std::path::Path) -> std::io::Result<()> { pub(crate) fn log_line(line: &str) { use std::io::Write; let mut g = LOG_FILE.lock().unwrap_or_else(|e| e.into_inner()); - if let Some(f) = g.as_mut() { + if let Some((f, path)) = g.as_mut() { + if LOG_REOPEN.swap(false, std::sync::atomic::Ordering::Relaxed) { + // A failed reopen keeps the old handle: a line in the rotated file + // is better than a line lost. + match log_file(path) { + Ok(new) => *f = new, + Err(e) => eprintln!("exav: --log {}: reopening: {e}", path.display()), + } + } let mut buf = String::with_capacity(line.len() + 1); buf.push_str(line); buf.push('\n'); @@ -263,22 +281,20 @@ struct Cli { #[arg(long = "files-from", value_name = "FILE", env = "EXAV_FILES_FROM")] file_list: Option, - /// Append scan results to FILE as well as writing them to stdout. The file - /// is opened once and appended to, so it survives log rotation the same way - /// clamd's `LogFile` does. + /// Append scan results to FILE as well as writing them to stdout. A + /// listener reopens it on `SIGHUP`, as clamd does with its `LogFile`, so + /// logrotate can move it. #[arg(long = "log", value_name = "FILE", env = "EXAV_LOG")] log: Option, - /// Load this exact FILE or DIR instead of --sig-dir. Recognizes `.ndb`, - /// `.hdb`/`.hsb`, `.fdb` (fuzzy), exav `.db`, and `.cvd`/`.cld` containers; - /// a DIR is scanned recursively (hidden `.`-prefixed sub-dirs skipped). + /// Load this exact FILE or DIR instead of --sig-dir: a prebuilt `.exavdb` + /// file, or a directory of signature files (`.cvd`/`.cld` containers, + /// `.ndb`, `.hdb`/`.hsb`, `.ldb`, `.yar`, ...), read recursively with hidden + /// `.`-prefixed sub-dirs skipped. /// - /// Distinct from --sig-dir, which names the *directory signatures live in* — - /// the one --auto-update writes into and the one a sidecar populates. This - /// names what to load, and is how a deployment loads a prebuilt `.exavdb` - /// that lives somewhere other than that directory. With neither, and no real - /// database, exav refuses to run unless --allow-no-db (built-in EICAR-only - /// baseline, testing only). + /// When given, it is also what --auto-update writes to. With neither this + /// nor signatures in --sig-dir, exav refuses to run unless --allow-no-db + /// (built-in EICAR-only baseline, testing only). #[arg( short = 'd', long = "database", @@ -287,11 +303,10 @@ struct Cli { )] database: Option, - /// Directory signatures live in (.ndb/.hdb/.hsb/.cvd/...), loaded - /// recursively. Populate it with `cvd`/`freshclam` (+ + /// Directory signatures live in (.cvd/.ndb/.hdb/.hsb/...), loaded + /// recursively when -d is not given. Populate it with `cvd`/`freshclam` (+ /// `clamav-unofficial-sigs` for third-party feeds), or let --auto-update - /// keep it current — it is the directory written to, so it stays a - /// directory even when -d points the load somewhere else. + /// keep it current (it writes here unless -d names somewhere else). #[arg( long = "sig-dir", value_name = "DIR", @@ -360,14 +375,14 @@ struct Cli { )] startup_timeout: Option, - /// Seconds between signature update checks (--auto-update). Floored at 60 - /// however low this goes: a zero-second sleep is not a fast poll, it is a - /// loop with no delay aimed at someone else's servers. [default: 86400] + /// Seconds between signature update checks (--auto-update), or `off` to + /// fetch at startup only. Floored at 60: a shorter sleep is a loop aimed at + /// someone else's servers. [default: 86400, or 300 with --db-url] #[arg( long = "update-interval-secs", value_name = "SECS|off", env = "EXAV_UPDATE_INTERVAL_SECS", - value_parser = parse_limit_secs + value_parser = parse_secs_or_off )] update_interval_secs: Option, @@ -454,15 +469,16 @@ struct Cli { ping: bool, /// Bytes of a body an ICAP client should send before pausing for a verdict - /// (the `Preview` header). [default: 4096] + /// (the `Preview` header). `0` previews the headers only; `off` leaves the + /// header out, so clients send whole bodies. [default: 4096] #[cfg(feature = "icap")] #[arg( long = "icap-preview-bytes", value_name = "SIZE|off", env = "EXAV_ICAP_PREVIEW_BYTES", - value_parser = parse_size_usize + value_parser = parse_size_or_omit )] - icap_preview_size: Option, + icap_preview_size: Option>, /// Which objects a client should preview (the `Transfer-Preview` header); /// `*` means all, `off` omits the header. [default: *] @@ -475,45 +491,50 @@ struct Cli { icap_transfer_preview: Option, /// Seconds a client may cache the ICAP `OPTIONS` answer (`Options-TTL`). - /// [default: 3600] + /// `0` asks it to re-ask every time; `off` leaves the header out, which + /// RFC 3507 reads as never expiring. [default: 3600] #[cfg(feature = "icap")] #[arg( long = "icap-options-ttl-secs", value_name = "SECS|off", env = "EXAV_ICAP_OPTIONS_TTL_SECS", - value_parser = parse_limit_secs_u32 + value_parser = parse_secs_or_omit )] - icap_options_ttl: Option, + icap_options_ttl: Option>, - /// Requests served on one ICAP connection before it is closed. - /// [default: 100] [c-icap.conf: MaxKeepAliveRequests] + /// Requests served on one ICAP connection before it is closed; `off` or + /// `0` means no limit. [default: 100] [c-icap.conf: MaxKeepAliveRequests] #[cfg(feature = "icap")] #[arg( long = "icap-max-requests", - value_name = "N", - env = "EXAV_ICAP_MAX_REQUESTS" + value_name = "N|off", + env = "EXAV_ICAP_MAX_REQUESTS", + value_parser = parse_count )] icap_keepalive_requests: Option, - /// Seconds an idle ICAP connection is held open. [default: 600] + /// Seconds an ICAP connection may go without sending or taking a byte, + /// between requests or in the middle of one. No `off`: without it a + /// client that stops sending holds its connection forever. [default: 600] /// [c-icap.conf: KeepAliveTimeout] #[cfg(feature = "icap")] #[arg( long = "icap-idle-secs", - value_name = "SECS|off", + value_name = "SECS", env = "EXAV_ICAP_IDLE_SECS", - value_parser = parse_limit_secs + value_parser = parse_timeout_secs )] icap_idle_timeout: Option, /// Largest ICAP head plus encapsulated HTTP headers accepted in one - /// request. [default: 65536] + /// request, at least 1K. No `off`: this is what stops a client that sends + /// header bytes forever. [default: 64K] #[cfg(feature = "icap")] #[arg( long = "icap-max-header-bytes", - value_name = "SIZE|off", + value_name = "SIZE", env = "EXAV_ICAP_MAX_HEADER_BYTES", - value_parser = parse_size_usize + value_parser = parse_header_bytes )] icap_max_header_size: Option, @@ -579,10 +600,9 @@ struct Cli { /// Daemon worker model (Unix): a worker count, or `threads`. /// [default: CPU cores] /// - /// A count runs a prefork pool — one worker process per core, each scanning - /// one job at a time under kernel-enforced per-job limits (see - /// --max-scan-secs and --max-process-bytes) and recycled per - /// --max-jobs-per-worker, so a + /// A count runs a prefork pool: worker processes, each scanning one job at + /// a time under kernel-enforced per-job limits (see --max-scan-secs and + /// --max-process-bytes) and recycled per --max-jobs-per-worker, so a /// runaway scan is isolated and hard-killed. `threads` runs the listeners /// in one process instead, which cannot kill a single job but is the only /// model where both listeners share one set of counters. @@ -594,12 +614,13 @@ struct Cli { )] workers: Option, - /// Hard wall-clock budget, in seconds (0 = none). Unix only. In the prefork - /// pool it is per scan job, and on expiry the worker is killed and the - /// connection dropped; it also caps CPU time (RLIMIT_CPU). In a one-shot - /// run it bounds the whole run, which then exits 3 saying so — running out - /// of time is a scan that stopped short, not a scanner that failed. The - /// deterministic in-core caps still apply first in every mode. + /// Hard wall-clock budget, in seconds (`off` or `0` = none). Unix only. In + /// the prefork pool it is per scan job: on expiry the job is answered + /// LIMITS-EXCEEDED and its worker replaced; it also caps CPU time + /// (RLIMIT_CPU). In a one-shot run it bounds the whole run, which then + /// exits 3 saying so: running out of time is a scan that stopped short, not + /// a scanner that failed. The deterministic in-core caps still apply first + /// in every mode. /// [default in the pool: 120; unset otherwise] #[arg( long = "max-scan-secs", @@ -609,11 +630,14 @@ struct Cli { )] max_scan_time: Option, - /// Address-space cap (RLIMIT_AS), bounding memory bombs. Unix only. Per - /// worker in the prefork pool, whole-process in a one-shot run. Setting it - /// also lowers the in-core extraction budget to fit inside it, so a scan - /// reports a limit instead of being killed for hitting one. K/M/G/T - /// suffixes; 0 = none. [default in the pool: 2G; unset otherwise] + /// The memory one scan may use: an address-space cap (RLIMIT_AS) on each + /// worker in the prefork pool, on the whole process otherwise. Unix only. + /// What a scan holds is kept to half of it (the rest is the matcher's + /// working set), lowering the default 1G extraction budget and + /// --max-object-bytes when they are larger, so a scan reports + /// LIMITS-EXCEEDED instead of being killed. K/M/G/T suffixes; `off` or `0` + /// means no cap. [default in the pool: 2G, lowered to what the host's RAM + /// can back; unset otherwise] #[arg( long = "max-process-bytes", value_name = "SIZE|off", @@ -622,28 +646,17 @@ struct Cli { )] max_scan_memory: Option, - /// Cap the per-shard automaton-BUILD transient (`--build-db` only): shard - /// each large partition so no single Aho-Corasick construction exceeds ~this - /// many bytes. NOTE this bounds the per-shard *transient*, not the total peak - /// — the resident parsed-signature set (~2 GB for main+daily) sits under it, - /// so peak ≈ this + that floor. K/M/G/T suffixes. - /// - /// This trades BUILD memory for SCAN structure: every shard is another walk - /// of every buffer for as long as that database is in use. On a daily-only - /// set, `256M` gave the PE partition 11 shards (15 walks per PE) against 3 - /// shards at `1G` (7 walks) — but the measured scan-time difference was - /// **within run-to-run noise**, so do not expect a speed-up from raising it. - /// - /// The reason is worth knowing: partitions and shards hold *disjoint* - /// pattern sets, so N walks are not N times the work — they match N - /// different pattern sets over the same bytes. More walks means more state - /// transitions but smaller, more cache-friendly automata, and the two effects - /// largely cancel. Merging partitions to remove a walk was measured to be - /// 10% SLOWER (see the target-0 note in `engine`). + /// Peak memory for building one matcher shard (`--build-db` only): each + /// large partition is split so no single Aho-Corasick construction exceeds + /// about this many bytes. It bounds that transient, not the total: the + /// parsed signatures stay resident underneath it, so the peak is this plus + /// that floor. K/M/G/T suffixes; `off` or `0` means one shard per + /// partition. /// - /// Use the largest value the build host can afford, on the general principle - /// that fewer, larger automata are the simpler shape — but treat it as a - /// build-memory knob, not a performance one. + /// Each shard is one more walk of every scanned buffer, but shards hold + /// disjoint pattern sets, so measured scan time does not change noticeably + /// with it. Treat it as a build-memory knob: lower it on a small build + /// host, otherwise use the largest value the host can afford. #[arg( long = "build-shard-bytes", value_name = "SIZE|off", @@ -653,19 +666,20 @@ struct Cli { max_build_memory: Option, /// Prefork only (requires --workers N): recycle a worker process after this - /// many jobs to bound slow leaks/fragmentation (0 = never). Mirrors Apache - /// MaxRequestsPerChild. [default: 1000] + /// many jobs to bound slow leaks/fragmentation; `off` or `0` never + /// recycles. Mirrors Apache MaxRequestsPerChild. [default: 1000] #[arg( long = "max-jobs-per-worker", - value_name = "N", - env = "EXAV_MAX_JOBS_PER_WORKER" + value_name = "N|off", + env = "EXAV_MAX_JOBS_PER_WORKER", + value_parser = parse_count )] max_jobs_per_worker: Option, /// Largest top-level input exav will scan. A larger one has its first bytes /// scanned and is reported LIMITS-EXCEEDED unless they hold a detection - /// (never a silent OK, unlike ClamAV). K/M/G/T suffixes; - /// `0` means no limit. exav default: no limit. `--clamav-compat` sets 100M. + /// (never a silent OK, unlike ClamAV). K/M/G/T suffixes; `off` or `0` + /// means no limit. [default: no limit; 100M with --clamav-compat] #[arg( long = "max-input-bytes", env = "EXAV_MAX_INPUT_BYTES", @@ -682,9 +696,10 @@ struct Cli { /// up: the alternative is that a large upload competes for space with /// everything else on the host. /// - /// `off` makes --spill-threshold-bytes a hard per-object memory ceiling — - /// anything larger is reported UNSCANNABLE, because there is nowhere left to - /// put it. For a read-only root filesystem, a container with no writable + /// `off` makes --spill-threshold-bytes a hard per-object memory ceiling: + /// anything larger has that much scanned and is reported LIMITS-EXCEEDED, + /// and an archive member past --max-object-bytes is not scanned. For a + /// read-only root filesystem, a container with no writable /// temp directory, or a deployment that would rather refuse a large object /// than let a scanned payload touch a disk. Worst-case memory is then /// --spill-threshold-bytes times the number of concurrent scans. @@ -692,26 +707,27 @@ struct Cli { spill_dir: Option, /// How much of a streamed object is held in RAM before it spills to a temp - /// file. K/M/G/T suffixes. [default: 16M] + /// file. K/M/G/T suffixes; `0` spills every object at once. [default: 16M] /// /// This is what bounds a listener's memory: a connection costs this much /// whatever the object on it weighs. Raising it trades RAM for fewer temp - /// files; lowering it does the reverse. + /// files; lowering it does the reverse. It is a size rather than a limit, + /// so it has no `off`: to keep every object in RAM, `--spill-dir off`. #[arg( long = "spill-threshold-bytes", - value_name = "SIZE|off", + value_name = "SIZE", env = "EXAV_SPILL_THRESHOLD_BYTES", - value_parser = parse_size + value_parser = parse_size_not_off )] spill_threshold: Option, - /// The most temp space one object may occupy. K/M/G/T suffixes; `0` means - /// no limit. [default: 2G] [clamd.conf: StreamMaxLength] + /// The most temp space one object may occupy. K/M/G/T suffixes; `off` or + /// `0` means no limit. [default: 2G] [clamd.conf: StreamMaxLength] /// - /// An object past it is reported UNSCANNABLE — never clean, and never a - /// dropped connection. To stop spilling altogether use `--spill-dir off`; - /// `0` here is the opposite, and reads as "no ceiling" like every other - /// --max- flag — which leaves --max-total-spill-bytes as the only bound on + /// An object past it is scanned as far as it was held and reported + /// LIMITS-EXCEEDED: never clean, and never a dropped connection. To stop + /// spilling altogether use `--spill-dir off`; `off` here is the opposite, + /// no ceiling, which leaves --max-total-spill-bytes as the only bound on /// one object. #[arg( long = "max-spill-bytes", @@ -722,7 +738,7 @@ struct Cli { max_spill_bytes: Option, /// The most temp space every in-flight object may occupy **together**, - /// within one process. K/M/G/T suffixes; `0` means no limit. + /// within one process. K/M/G/T suffixes; `off` or `0` means no limit. /// [default: 8G] /// /// The one a per-object cap cannot stand in for: a hundred connections at @@ -739,30 +755,17 @@ struct Cli { )] max_total_spill_bytes: Option, - /// Cap on what decompression may *produce* across one top-level file: - /// caps deep/structural analysis size and the summed extracted bytes with - /// one value. K/M/G/T suffixes; `0` means no limit. exav defaults when - /// unset: 256M deep-analysis, 1G extracted total. `--clamav-compat` sets - /// 400M for both. - #[arg( - long = "max-extracted-bytes", - env = "EXAV_MAX_EXTRACTED_BYTES", - value_name = "SIZE|off", - value_parser = parse_size - )] - max_extracted_bytes: Option, - - /// The most memory any **single** materialized object (a decompressed - /// member/sub-container, an LZ window, a decrypted blob) may use. Every - /// forced-materialization site obeys it. Not a cap on total - /// memory: several buffers are live at once across nesting levels, and - /// `--max-extracted-bytes` is what bounds their sum. K/M/G/T suffixes; - /// `off` or `0` means no limit. exav default: 256M. + /// The most memory one object may take when it is held whole: a file, a + /// decompressed member, a decoder's window, a decrypted blob. Several can be + /// live at once across nesting levels; --max-process-bytes is what bounds + /// the whole scan. K/M/G/T suffixes; `off` or `0` means no limit. + /// [default: 256M] /// - /// It is also the largest file the full signature engine can scan, and it - /// wins over `--max-extracted-bytes` when both are given. A larger file is - /// matched against literal signatures and whole-file hashes only, and - /// reported LIMITS-EXCEEDED unless one of those matches. + /// A larger object gets the same scan read through a block cache, or a + /// spill file for a member, except for the checks that parse an object + /// whole (a PE's structure, YARA's pe/elf/dotnet modules, containers read + /// whole); an object one of those applied to is reported LIMITS-EXCEEDED + /// unless something is found. #[arg( long = "max-object-bytes", env = "EXAV_MAX_OBJECT_BYTES", @@ -771,13 +774,12 @@ struct Cli { )] max_buffer_bytes: Option, - /// Cumulative scan-reach limit: the most bytes fed to the matcher across one - /// top-level file (streamed members + re-scanned/carved regions). This is a - /// **CPU/time** bound, NOT a memory bound — streamed members are scanned - /// without being held in RAM (that is capped by --max-object-bytes), so this - /// can be set far higher to fully scan multi-gigabyte members, paying only in - /// scan time. Guards re-scanning/decompression-time bombs. K/M/G/T suffixes; - /// `off` or `0` means no limit. exav default: 10G. + /// The most bytes fed to the matcher for one top-level file: its members, + /// decoded payloads and carved regions, summed. A **CPU/time** bound, not a + /// memory one: a streamed member is scanned without being held, so this can + /// be set far higher to scan multi-gigabyte members in full, paying only in + /// scan time. Guards decompression bombs. K/M/G/T suffixes; `off` or `0` + /// means no limit. [default: 10G] [clamscan: --max-scansize] #[arg( long = "max-matcher-bytes", env = "EXAV_MAX_MATCHER_BYTES", @@ -788,8 +790,8 @@ struct Cli { /// Most x86 instructions the PE unpacking emulator may run across one /// top-level file, summed over every packed executable it contains. A scan - /// that runs out is LIMITS-EXCEEDED. `off` or `0` means no limit. exav - /// default: 1000000000 (seconds to tens of seconds of CPU). + /// that runs out is LIMITS-EXCEEDED. `off` or `0` means no limit. + /// [default: 1000000000, seconds to tens of seconds of CPU] #[arg( long = "max-pe-emulation-steps", env = "EXAV_MAX_PE_EMULATION_STEPS", @@ -798,27 +800,34 @@ struct Cli { )] max_pe_emulation_steps: Option, - /// Maximum nesting depth for recursive unpacking — a zip inside a tar - /// inside a disk image. exav default: 16. `--clamav-compat` sets 17. + /// Maximum nesting depth for recursive unpacking: a zip inside a tar + /// inside a disk image. At least 1, and there is no `off`: every level + /// costs stack, and an archive nested a million deep would otherwise + /// crash the scan instead of being reported. [default: 16; 17 with + /// --clamav-compat] /// /// Not directory depth: `find`, `du` and `tree` all spell that /// `--max-depth`, so the bare word would be read as a bound on the /// directory walk, which this is not and which only `--no-recursive` - /// touches. A flag that silently bounds something other than what the - /// reader assumes is worse than a longer name. + /// touches. #[arg( long = "max-unpack-depth", env = "EXAV_MAX_UNPACK_DEPTH", - value_name = "N" + value_name = "N", + value_parser = parse_unpack_depth )] max_recursion: Option, - /// Maximum number of members visited across the whole recursive walk. - /// exav default: 100000 — higher than ClamAV's 10000 because exav descends - /// into nested archives ClamAV does not, so the same file yields more - /// countable members (see `Limits::max_members`). `--clamav-compat` sets - /// 10000. - #[arg(long = "max-members", env = "EXAV_MAX_MEMBERS", value_name = "N")] + /// Maximum number of members visited across the whole recursive walk; + /// `off` or `0` means no limit. Higher than ClamAV's 10000 by default, + /// because exav descends into nested archives ClamAV does not, so the same + /// file yields more members. [default: 100000; 10000 with --clamav-compat] + #[arg( + long = "max-members", + env = "EXAV_MAX_MEMBERS", + value_name = "N|off", + value_parser = parse_count + )] max_members: Option, /// Encodings to recover a payload from before scanning it: `all` (the @@ -863,11 +872,13 @@ struct Cli { /// /// A leak detector rather than a malware one: what it finds is the /// organisation's own data on its way somewhere, so it says `--dlp-` and - /// not `--detect`. Needs the `dlp` feature. [clamscan: --structured-cc-count] + /// not `--detect`. N is at least 1. Needs the `dlp` feature. + /// [clamscan: --structured-cc-count] #[arg( long = "dlp-credit-cards", value_name = "N", - env = "EXAV_DLP_CREDIT_CARDS" + env = "EXAV_DLP_CREDIT_CARDS", + value_parser = parse_dlp_count )] structured_cc_count: Option, @@ -875,7 +886,12 @@ struct Cli { /// valid US Social Security numbers. Off unless set. See /// --dlp-credit-cards. Needs the `dlp` feature. /// [clamscan: --structured-ssn-count] - #[arg(long = "dlp-ssns", value_name = "N", env = "EXAV_DLP_SSNS")] + #[arg( + long = "dlp-ssns", + value_name = "N", + env = "EXAV_DLP_SSNS", + value_parser = parse_dlp_count + )] structured_ssn_count: Option, /// Heuristic detectors to switch on, over and above the signature database: @@ -963,8 +979,8 @@ struct Cli { )] partial_as: Option, - /// Password to try when decrypting encrypted archive members (ZIP - /// ZipCrypto/AES). Repeatable (comma-separated in the environment): + /// Password to try on encrypted content (ZIP, 7z, DMG, PDF, Office, ARJ). + /// Repeatable (comma-separated in the environment): /// `--passwords a --passwords b` builds a pool, tried in order. Unioned with /// any passwords loaded from `.pwdb` databases. When a scan reports /// `password-protected`, re-run with the right password. @@ -990,11 +1006,11 @@ struct Cli { /// Shortcut that sets exav to a stock ClamAV build's documented defaults for /// apples-to-apples differential testing. Equivalent to `--max-input-bytes - /// 100M --max-extracted-bytes 400M --max-unpack-depth 17 --max-members 10000 - /// --decode none --partial-as ok`, plus narrowing the unpacking reach to the - /// formats stock ClamAV handles and reporting under ClamAV's vocabulary - /// where the two engines name the same fact differently. It leaves - /// `--max-object-bytes`, `--max-matcher-bytes`, spill settings, `--detect`, + /// 100M --max-unpack-depth 17 --max-members 10000 --decode none --partial-as + /// ok`, with 400M extracted per file (ClamAV's scan size), plus narrowing + /// the unpacking reach to the formats stock ClamAV handles and reporting + /// under ClamAV's vocabulary where the two engines name the same fact + /// differently. It leaves `--max-matcher-bytes`, spill settings, `--detect`, /// and update/network/worker settings on exav defaults. This DELIBERATELY /// REDUCES exav's detection capability so results reproduce clamscan's — it /// is a diff-testing mode, NOT recommended for production. Off by default @@ -1064,7 +1080,8 @@ struct Cli { #[arg(short = 'v', long = "verbose", env = "EXAV_VERBOSE", value_parser = env_switch())] verbose: bool, - /// Print only errors and detections: no per-file `OK` lines, no summary. + /// Print only detections, PARTIAL results and errors: no per-file `OK` + /// lines, no summary. /// /// The single output dial, with -v at the other end. One flag rather than /// one per suppressed line, because how much output a run makes is one @@ -1355,7 +1372,7 @@ fn restore_default_sigpipe() {} /// The listener's disposition: a peer that hangs up costs one connection, not /// the process. The inverse of [`restore_default_sigpipe`], which `main` runs /// first for the sake of the one-shot scan. -#[cfg(all(unix, feature = "icap"))] +#[cfg(unix)] fn ignore_sigpipe() { // SAFETY: as above — this process's own disposition for one signal, before // any connection is accepted. @@ -1366,9 +1383,8 @@ fn ignore_sigpipe() { /// Nothing to do off Unix: there is no `SIGPIPE`, and a peer that hangs up /// mid-response surfaces as an ordinary write error on the socket. Provided so -/// the ICAP listener's call sites stay unconditional — gating each of them -/// instead is how this came to not compile for Windows at all. -#[cfg(all(not(unix), feature = "icap"))] +/// the listeners' call sites stay unconditional. +#[cfg(not(unix))] fn ignore_sigpipe() {} /// Name the exav spelling for the `clamscan` flags a migrating user types from @@ -1406,7 +1422,7 @@ fn clamscan_flag_hint(args: &[String]) -> Option { "exav reports and does not move or delete; act on the exit code" } "-z" | "--allmatch" => "use --all-matches, which keeps scanning past the first match", - "-f" | "--file-list" => "use --files-from, the spelling xargs and tar use", + "-f" | "--file-list" => "use --files-from, the spelling tar and rsync use", "--datadir" => "use --sig-dir (or -d) to name the directory signatures live in", "--tempdir" => "use --spill-dir, where a streamed object waits while it is scanned", "--statistics" => "use --profile, a per-matcher timing breakdown", @@ -1432,13 +1448,23 @@ fn clamscan_flag_hint(args: &[String]) -> Option { // trade is "a clear error naming the exav flag", which is this. "--max-filesize" => "use --max-input-bytes, the largest top-level input scanned", "--max-scansize" => { - "use --max-extracted-bytes, what decompression may produce for one top-level file" + "use --max-matcher-bytes, the most bytes scanned for one top-level file" + } + // Conditions ClamAV reports as detections, which exav reports as + // PARTIAL unless asked otherwise. + "--alert-encrypted" | "--alert-encrypted-archive" | "--alert-encrypted-doc" => { + "use --partial-as password-protected=found, which reports them as \ + Heuristics.Encrypted.*" + } + "--alert-exceeds-max" => { + "use --partial-as limits-exceeded=found, which reports them as \ + Heuristics.Limits.Exceeded.*" } // `--max-depth` never shipped under that name, but it is what // `find`/`du`/`tree` call directory depth, so someone will reach for // it meaning the walk. Answer it too, and say which one it is not. "--max-recursion" | "--max-depth" => { - "use --max-unpack-depth, the nesting of containers inside containers — \ + "use --max-unpack-depth, the nesting of containers inside containers; \ the directory walk is all-or-nothing via --no-recursive" } "--max-files" => "use --max-members, members visited across the whole recursive walk", @@ -1452,11 +1478,34 @@ fn clamscan_flag_hint(args: &[String]) -> Option { }) } +/// A flag, or its environment variable, that an earlier exav took and this one +/// does not, with what replaced it. The variable is checked too: clap ignores +/// one it does not know, which would drop the setting without a word. +fn removed_flag(args: &[String]) -> Option { + const EXTRACTED: &str = "--max-extracted-bytes was removed in 0.0.2: \ + --max-process-bytes bounds the memory a scan may use, and \ + --max-object-bytes one object"; + let given = args + .iter() + .any(|a| a.split('=').next() == Some("--max-extracted-bytes")); + (given || std::env::var_os("EXAV_MAX_EXTRACTED_BYTES").is_some()).then(|| { + if given { + EXTRACTED.to_string() + } else { + format!("EXAV_MAX_EXTRACTED_BYTES is set, and {EXTRACTED}") + } + }) +} + fn main() -> ExitCode { restore_default_sigpipe(); // Before `Cli::parse`, which exits the process on an unknown argument and // would never reach this. let argv: Vec = std::env::args().skip(1).collect(); + if let Some(msg) = removed_flag(&argv) { + eprintln!("exav: {msg}"); + return ExitCode::from(2); + } if let Some(hint) = clamscan_flag_hint(&argv) { eprintln!("exav: {hint}"); eprintln!("exav: the full mapping is at https://exav.org/reference/clamav-flag-matrix/"); @@ -1639,9 +1688,7 @@ fn main() -> ExitCode { // periodic half starts once that load has succeeded. #[cfg(unix)] let auto_update = { - // Only the prefork supervisor acts on a reload request; anywhere else - // the mtime watch on the same path is what notices. - let reload = if pool_workers > 0 { + let reload = if serves { signatures::Reload::Signal } else { signatures::Reload::Watch @@ -1683,7 +1730,7 @@ fn main() -> ExitCode { // only way here is a build with no updater to run. None => { eprintln!( - "exav: fetching signatures needs the updater — build with \ + "exav: fetching signatures needs the updater: build with \ `--features http-update`" ); return ExitCode::from(2); @@ -1741,9 +1788,9 @@ fn main() -> ExitCode { if cap != 0 && cap < MIN_PROCESS_BYTES { eprintln!( "exav: --max-process-bytes {} MiB is below the {} MiB this process needs to \ - start at all — the cap covers exav itself and the signature database, not \ - just a scan's buffers. Raise it (a real database wants ~2G), or pass 0 for \ - no cap and bound the scan with --max-input-bytes/--max-extracted-bytes.", + start at all: the cap covers exav itself and the signature database, not \ + just a scan's buffers. Raise it (a real database wants ~2G), or pass `off` \ + for no cap and bound the scan with --max-input-bytes/--max-object-bytes.", cap >> 20, MIN_PROCESS_BYTES >> 20, ); @@ -1781,9 +1828,9 @@ fn main() -> ExitCode { // baseline explicitly with --allow-no-db (testing/CI only). if is_effectively_empty(&db) && !cli.allow_no_db { eprintln!( - "exav: no signature database loaded — refusing to run (it would report real \ - malware as clean). Load signatures with -d/--sig-dir, or pass --allow-no-db \ - to use the built-in EICAR-only baseline (testing only)." + "exav: no signature database loaded, refusing to run (it would report real \ + malware as clean). Load signatures with -d/--database or --sig-dir, or pass \ + --allow-no-db to use the built-in EICAR-only baseline (testing only)." ); return ExitCode::from(2); } @@ -1990,13 +2037,10 @@ fn run_listeners(cli: &Cli, db: Scanner, pool_workers: usize) -> ExitCode { let db = std::sync::Arc::new(db); let opts = std::sync::Arc::new(opts); - // ICAP alone: its own listener thread, and this thread becomes the - // supervisor that watches the signature source and swaps the database in. - #[cfg(feature = "icap")] - let clamd = clamd_endpoint(cli); - #[cfg(not(feature = "icap"))] let clamd = clamd_endpoint(cli); + // ICAP alone: its own listener thread, and this thread becomes the + // supervisor that reloads the database. #[cfg(feature = "icap")] if clamd.is_none() { let server = icap_server.expect("an icap:// address is the only listener asked for"); @@ -2005,7 +2049,12 @@ fn run_listeners(cli: &Cli, db: Scanner, pool_workers: usize) -> ExitCode { // than inside the ICAP module, which is `forbid(unsafe_code)` and stays // that way. ignore_sigpipe(); - return icap::serve_alone(server, db, opts, watch, &reload, metrics_interval(cli)); + // Serving ICAP alone binds no clamd port, so `STATS` cannot be asked + // here at all and the log is the only channel the totals have. + metrics::spawn_reporter(metrics_interval(cli), "icap"); + let shared = std::sync::Arc::new(daemon::SharedDb::new(db)); + icap::spawn(server, std::sync::Arc::clone(&shared), opts); + daemon::supervise(&shared, watch, &reload); } // `?max-connections=` on the clamd address, the same option the ICAP one @@ -2072,54 +2121,41 @@ fn run_listeners(cli: &Cli, db: Scanner, pool_workers: usize) -> ExitCode { // Thread model: one process, so one reporter covers every listener it runs. metrics::spawn_reporter(metrics_interval(cli), "daemon"); - // With ICAP alongside, the clamd listener moves to a thread of its own so - // this one can stay the supervisor that watches the signature source — the - // arrangement ICAP already has when it serves alone. + // Each listener on a thread of its own, and this one the supervisor that + // reloads the database they share. + let shared = std::sync::Arc::new(daemon::SharedDb::new(db)); + // Before any listener starts. The disposition is process-wide, so leaving + // it to `daemon::run` on its thread would leave ICAP serving under + // `SIG_DFL` until that thread got there: a window in which a proxy hanging + // up mid-response kills the whole process. + ignore_sigpipe(); #[cfg(feature = "icap")] if let Some(server) = icap_server { - let (clamd_db, clamd_opts) = (std::sync::Arc::clone(&db), std::sync::Arc::clone(&opts)); - let allow_shutdown = shutdown_allowed(cli.allow_shutdown); - let allow_http_scan = cli.allow_http_scan; - // Before the thread starts, for the same reason the ICAP-only path sets - // it before serving. The disposition is process-wide, so leaving it to - // `daemon::run` on the new thread would leave this one serving ICAP - // under `SIG_DFL` until that thread got there — a window in which a - // proxy hanging up mid-response kills the whole process. - ignore_sigpipe(); - std::thread::spawn(move || { - if let Err(e) = daemon::run( - clamd_db, - addr, - clamd_opts, - allow_shutdown, - allow_http_scan, - clamd_max_connections, - ) { - eprintln!("exav: daemon error: {e}"); - } - // The listener is the whole job, and an orchestrator can only - // restart what it can see has stopped. - std::process::exit(2); - }); - // The reporter is already running for this process (started above), so - // one line covers both listeners rather than each announcing its own - // share of a total they contribute to together. - return icap::serve_alone(server, db, opts, watch, &reload, std::time::Duration::ZERO); - } - match daemon::run( - db, - addr, - opts, - shutdown_allowed(cli.allow_shutdown), - cli.allow_http_scan, - clamd_max_connections, - ) { - Ok(()) => ExitCode::SUCCESS, - Err(e) => { + icap::spawn( + server, + std::sync::Arc::clone(&shared), + std::sync::Arc::clone(&opts), + ); + } + let clamd_db = std::sync::Arc::clone(&shared); + let allow_shutdown = shutdown_allowed(cli.allow_shutdown); + let allow_http_scan = cli.allow_http_scan; + std::thread::spawn(move || { + if let Err(e) = daemon::run( + clamd_db, + addr, + opts, + allow_shutdown, + allow_http_scan, + clamd_max_connections, + ) { eprintln!("exav: daemon error: {e}"); - ExitCode::from(2) } - } + // The listener is the whole job, and an orchestrator can only restart + // what it can see has stopped. + std::process::exit(2); + }); + daemon::supervise(&shared, watch, &reload) } /// Build the scan options from the CLI. `--clamav-compat` is a *preset*: it @@ -2153,6 +2189,9 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { // differently. Affects names only — nothing is detected in one mode and not // the other. opts.clamav_compat = compat; + // What a scan makes of an object too large to hold goes to the spill files + // `--spill-dir` configures. + opts.spill = Some(std::sync::Arc::new(spill::ScanSpill)); // --max-input-bytes: per top-level file cap. exav default unlimited; // compat 100M. @@ -2169,25 +2208,12 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { None => compat.then_some(100 * MIB), }; - // --max-extracted-bytes: what decompression may produce (deep-analysis size - // + summed extracted bytes, set together). Unset, the two keep their own - // defaults: 256M deep-analysis (`ScanOptions::default`), 1G extracted total - // (`Limits::default`); compat 400M for both. - match cli.max_extracted_bytes { - Some(0) => { - opts.deep_analysis_max = u64::MAX; - opts.limits.max_extracted_bytes = u64::MAX; - } - Some(s) => { - opts.deep_analysis_max = s; - opts.limits.max_extracted_bytes = s; - } - None => { - if let Some(s) = compat.then_some(400 * MIB) { - opts.deep_analysis_max = s; - opts.limits.max_extracted_bytes = s; - } - } + // What a scan may hold, unless --max-process-bytes lowers it + // (`fit_limits_to_job_memory`): 1G extracted in total (`Limits::default`), + // and ClamAV's 400M scan size under compat. + if compat { + opts.deep_analysis_max = 400 * MIB; + opts.limits.max_extracted_bytes = 400 * MIB; } // --max-object-bytes: the global peak-buffer limit. Sets the unpack // per-object cap and the core-side structural buffer (deep_analysis_max) @@ -2215,7 +2241,7 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { // --max-members: members per recursive walk. exav default 100000 (see // `Limits::max_members` for why it is higher than ClamAV's); compat 10000. if let Some(f) = cli.max_members.or_else(|| compat.then_some(10_000)) { - opts.limits.max_members = f; + opts.limits.max_members = no_limit_at_zero(f); } // Narrowing the unpacking reach to stock ClamAV's, and naming signatures the @@ -2629,19 +2655,24 @@ fn guard_not_empty(db: Scanner, source: &str, allow_no_db: bool) -> Result Option { + cli.max_build_memory.filter(|&b| b != 0) +} + fn load_db(cli: &Cli) -> Result { // `--clamav-compat` selects exact ClamAV naming: the `.UNOFFICIAL` suffix on // signatures from an unofficial database. Provenance is @@ -2649,7 +2680,7 @@ fn load_db(cli: &Cli) -> Result { // report time from `ScanOptions::unofficial_suffix`, so one loaded database // serves both compat and non-compat scans. let suffix = cli.clamav_compat; - let bmem = cli.max_build_memory; + let bmem = build_shard_budget(cli); let pua = detectors(cli).pua(); if let Some(path) = &cli.database { return loader::load_with_options_mem(path, pua, suffix, bmem).map_err(|e| e.to_string()); @@ -2685,9 +2716,7 @@ fn scan_target( totals.dirs += walk.dirs; for line in &walk.errors { totals.errors += 1; - if !cli.quiet { - eprintln!("{line}"); - } + errln(line); } for file in &walk.files { if is_volume_part(file) { @@ -2827,20 +2856,21 @@ impl<'a> Input<'a> { } } - /// The object's bytes, or `None` when it is larger than `cap` or was not - /// held whole. - fn bytes_capped(&self, cap: u64) -> io::Result>> { + /// Every detection in the object, or `None` when it was not held whole. + #[allow(clippy::type_complexity)] + fn all_matches( + &self, + db: &Scanner, + opts: &ScanOptions, + ) -> io::Result, exav_core::AllMatchOutcome)>> { match self { Self::File(p) => { - let mut data = Vec::new(); - io::Read::read_to_end( - &mut io::Read::take(std::fs::File::open(p)?, cap.saturating_add(1)), - &mut data, - )?; - Ok((data.len() as u64 <= cap).then_some(data)) + let file = std::fs::File::open(p)?; + let size = file.metadata()?.len(); + exav_core::analyze_all_seekable(db, file, size, opts).map(Some) } - Self::Held(held, n) if held.short.is_none() && *n <= cap => { - held.payload.bytes_capped(cap) + Self::Held(held, _) if held.short.is_none() => { + held.payload.all_matches(db, opts).map(Some) } Self::Held(..) => Ok(None), } @@ -2976,11 +3006,18 @@ fn report_error(name: &str, message: &str, cli: &Cli, totals: &mut Totals) { "file": name, "status": "ERROR", "reason": message }) ); - } else if !cli.quiet { - eprintln!("{name}: {message} ERROR"); + } else { + errln(&format!("{name}: {message} ERROR")); } } +/// Print an error line to stderr and mirror it into `--log`. Not silenced by +/// `--quiet`, which drops the `OK` lines and the summary, never an error. +fn errln(line: &str) { + eprintln!("{line}"); + log_line(line); +} + /// Scan stdin: held like any stream, then the same per-object path as a file, /// so `--all-matches`, `--profile` and the totals apply to it too. fn scan_stdin(db: &Scanner, opts: &ScanOptions, cli: &Cli, totals: &mut Totals) { @@ -3121,7 +3158,7 @@ fn run_ping(cli: &Cli) -> ExitCode { if !cli.paths.is_empty() { eprintln!( "exav: --ping takes no arguments and probes once; it does not take clamdscan's \ - attempts[:interval]. Retry around it — a HEALTHCHECK's --retries, or a loop." + attempts[:interval]. Retry around it: a HEALTHCHECK's --retries, or a loop." ); return ExitCode::from(2); } @@ -3593,9 +3630,7 @@ fn run_client(cli: &Cli) -> ExitCode { let mut totals = Totals::default(); for line in &walk_errors { totals.errors += 1; - if !cli.quiet { - eprintln!("{line}"); - } + errln(line); } if scan_stdin { @@ -3815,11 +3850,17 @@ fn print_client_summary(totals: &Totals) { println!("\n----------- SCAN SUMMARY -----------"); println!("Scanned files: {}", totals.scanned); println!("Infected files: {}", totals.infected); + print_partial_and_errors(totals); +} + +/// The summary lines past clamscan's: files not fully examined, which clamscan +/// has no word for, and errors, which it has. +fn print_partial_and_errors(totals: &Totals) { if totals.limits > 0 { - println!("Limits exceeded (unscanned, not clean): {}", totals.limits); + println!("Partial files: {}", totals.limits); } if totals.errors > 0 { - println!("Errors: {}", totals.errors); + println!("Total errors: {}", totals.errors); } } @@ -3867,7 +3908,7 @@ fn print_daemon_reply(line: &str, cli: &Cli, totals: &mut Totals) { } } else { totals.errors += 1; - eprintln!("{line}"); + errln(line); } } else if !cli.quiet { outln!("{line}"); @@ -3974,28 +4015,14 @@ fn scan_one_allmatch( // scan that skipped its contents, and is expensive to tell apart from one. totals.data_scanned += input.size(); let name = path.display().to_string(); - // Two ceilings, one fallback. `deep_analysis_max` is how much this path can - // buffer; `max_scan_size` is how much the operator said may be scanned at - // all, and `analyze_all` — which takes bytes, not a file — cannot see it. - // Whichever is lower decides, because a file over the second must reach the - // single-match path: that is where the ceiling is enforced, and without this - // `--all-matches` would answer `OK` for a file `--max-input-bytes` says was - // never fully examined. A silent clean, produced by adding a flag about how - // many signatures to report. - let cap = opts.max_scan_size.map_or(opts.deep_analysis_max, |max| { - opts.deep_analysis_max.min(max) - }); - let data = match input.bytes_capped(cap) { - Ok(d) => d, - Err(e) => { - report_error(&name, &e.to_string(), cli, totals); - return; - } - }; - let Some(data) = data else { - // Too big for all-match; fall back to a single-match scan, which scans - // the budgeted prefix before reporting the limit, so a detection in the - // part that did fit still wins. + // `max_scan_size` is how much the operator said may be scanned at all, and + // the all-match walk does not enforce it: a file over it must reach the + // single-match path, where the ceiling is, or `--all-matches` would answer + // `OK` for a file `--max-input-bytes` says was never fully examined. So + // must an input not held whole. + let fall_back = |totals: &mut Totals| { + // The single-match scan scans the budgeted prefix before reporting the + // limit, so a detection in the part that did fit still wins. let scanned = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| input.scan(db, opts))); match scanned { @@ -4006,11 +4033,21 @@ fn scan_one_allmatch( Ok(Err(e)) => report_error(&name, &e.to_string(), cli, totals), Err(_) => report_error(&name, "internal error while scanning", cli, totals), } - return; }; - let found = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - exav_core::analyze_all_with_outcome(db, &data, opts) - })); + if opts.max_scan_size.is_some_and(|max| input.size() > max) { + return fall_back(totals); + } + let found = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| input.all_matches(db, opts))); + let found = match found { + Ok(Ok(Some(found))) => Ok(found), + Ok(Ok(None)) => return fall_back(totals), + Ok(Err(e)) => { + report_error(&name, &e.to_string(), cli, totals); + return; + } + Err(panic) => Err(panic), + }; match found { Ok((dets, outcome)) if !dets.is_empty() => { totals.infected += 1; @@ -4308,11 +4345,7 @@ fn print_summary(db: &Scanner, totals: &Totals, elapsed: std::time::Duration, ve println!("Scanned directories: {}", totals.dirs); println!("Scanned files: {}", totals.scanned); println!("Infected files: {}", totals.infected); - if totals.limits > 0 { - println!("Total errors: {}", totals.limits + totals.errors); - } else if totals.errors > 0 { - println!("Total errors: {}", totals.errors); - } + print_partial_and_errors(totals); println!("Data scanned: {mb:.2} MB"); println!("Data read: {mb:.2} MB (ratio 0.00:1)"); println!("Time: {secs:.3} sec ({m} m {s} s)"); @@ -4321,9 +4354,6 @@ fn print_summary(db: &Scanner, totals: &Totals, elapsed: std::time::Duration, ve println!("Engine signatures: {}", db.signature_count()); println!("Unsupported sigs skipped: {}", db.unsupported_count()); println!("Bytecode programs loaded: {}", db.bytecode_count()); - if totals.limits > 0 { - println!("Limits exceeded (unscanned, not clean): {}", totals.limits); - } } } @@ -4398,6 +4428,95 @@ fn parse_limit_secs_u32(s: &str) -> Result { u32::try_from(n).map_err(|_| format!("too many seconds: {s}")) } +/// `--max-unpack-depth`: at least 1, no `off` (see its help). +fn parse_unpack_depth(s: &str) -> Result { + match s.parse::() { + Ok(0) => Err( + "a depth of 0 would report every container as over the limit; \ + give at least 1" + .to_string(), + ), + Ok(n) => Ok(n), + Err(_) if s.eq_ignore_ascii_case("off") => Err("there is no unlimited depth: each \ + level costs stack, so a deep enough nesting would crash the scan; give a number" + .to_string()), + Err(_) => Err(format!("expected a nesting depth, got `{s}`")), + } +} + +/// `--dlp-*`: a count of findings, at least 1. `0` would alert on every file. +fn parse_dlp_count(s: &str) -> Result { + match s.parse::() { + Ok(0) => { + Err("0 would alert on every file; leave the flag out to turn this off".to_string()) + } + Ok(n) => Ok(n), + Err(_) => Err(format!("expected a number of findings, got `{s}`")), + } +} + +/// A setting whose `off` leaves it out, which is not the same as `0`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum OrOff { + Off, + On(T), +} + +impl OrOff { + /// The value, `None` for `off`. + #[cfg_attr(not(feature = "icap"), allow(dead_code))] + pub(crate) fn value(self) -> Option { + match self { + OrOff::Off => None, + OrOff::On(v) => Some(v), + } + } +} + +#[cfg_attr(not(feature = "icap"), allow(dead_code))] +fn parse_size_or_omit(s: &str) -> Result, String> { + if s.trim().eq_ignore_ascii_case("off") { + return Ok(OrOff::Off); + } + parse_size_usize(s).map(OrOff::On) +} + +#[cfg_attr(not(feature = "icap"), allow(dead_code))] +fn parse_secs_or_omit(s: &str) -> Result, String> { + if s.trim().eq_ignore_ascii_case("off") { + return Ok(OrOff::Off); + } + parse_limit_secs_u32(s).map(OrOff::On) +} + +/// A timeout that has to exist: at least one second, no `off`. +#[cfg_attr(not(feature = "icap"), allow(dead_code))] +fn parse_timeout_secs(s: &str) -> Result { + match s.trim().parse::() { + Ok(0) => Err("a timeout of 0 would drop every connection at once".to_string()), + Ok(n) => Ok(n), + Err(_) => Err(format!("expected a number of seconds, got `{s}`")), + } +} + +/// `--icap-max-header-bytes`: a size of at least 1K, no `off`. +#[cfg_attr(not(feature = "icap"), allow(dead_code))] +fn parse_header_bytes(s: &str) -> Result { + match parse_size_not_off(s)? { + n if n < 1024 => Err("below 1K no ICAP request's headers fit".to_string()), + n => usize::try_from(n).map_err(|_| format!("size too large for this platform: {s}")), + } +} + +/// A size that is not a limit, so `0` is taken as it is and `off` means +/// nothing. +fn parse_size_not_off(s: &str) -> Result { + if s.trim().eq_ignore_ascii_case("off") { + return Err("this is a size, not a limit, so it has no `off`".to_string()); + } + parse_size(s) +} + /// Parse a size with optional K/M/G/T suffix (base-1024), like clamscan. /// [`parse_size`] for a flag whose field is a `usize`. /// @@ -4595,7 +4714,7 @@ mod tests { // Scanning and serving belong to the daemon. for a in [ "--max-input-bytes", - "--max-extracted-bytes", + "--max-object-bytes", "--max-unpack-depth", "--max-members", "--detect", @@ -4806,14 +4925,9 @@ mod tests { /// `--max-object-bytes` is parsed on its own because it also sets the /// deep-analysis cap (one knob for the largest single allocation), which /// would mask the extracted-bytes assertion. - fn assert_limit_flags(label: &str, argv: [&str; 11], buffer_argv: [&str; 3]) { + fn assert_limit_flags(label: &str, argv: [&str; 9], buffer_argv: [&str; 3]) { let opts = build_scan_options(&Cli::parse_from(argv)); assert_eq!(opts.max_scan_size, Some(101), "{label}: input bytes"); - assert_eq!(opts.deep_analysis_max, 102, "{label}: extracted (deep)"); - assert_eq!( - opts.limits.max_extracted_bytes, 102, - "{label}: extracted (total)" - ); assert_eq!(opts.limits.max_scanned_bytes, 103, "{label}: scanned bytes"); assert_eq!(opts.limits.max_recursion, 104, "{label}: recursion"); assert_eq!(opts.limits.max_members, 105, "{label}: members"); @@ -4837,8 +4951,6 @@ mod tests { "exav", "--max-input-bytes", "101", - "--max-extracted-bytes", - "102", "--max-matcher-bytes", "103", "--max-unpack-depth", @@ -4864,6 +4976,19 @@ mod tests { ); } + // `off` on --build-shard-bytes is no cap, not one anchor per shard. + for (argv, want) in [ + (&["exav"][..], None), + (&["exav", "--build-shard-bytes", "off"][..], None), + (&["exav", "--build-shard-bytes", "0"][..], None), + ( + &["exav", "--build-shard-bytes", "1M"][..], + Some(1024 * 1024), + ), + ] { + assert_eq!(build_shard_budget(&Cli::parse_from(argv)), want, "{argv:?}"); + } + // With no flags, the defaults the --help text quotes. let opts = build_scan_options(&Cli::parse_from(["exav"])); assert_eq!(opts.max_scan_size, None); @@ -4884,6 +5009,55 @@ mod tests { assert_eq!(opts.limits.max_members, 10_000); } + /// `--max-extracted-bytes` is gone, and saying it, as a flag or in the + /// environment, is an error naming what replaced it rather than a setting + /// dropped without a word. + #[test] + fn the_removed_extraction_flag_is_named() { + let _env = env_guard(); + let args = |a: &[&str]| a.iter().map(|s| s.to_string()).collect::>(); + for argv in [ + &["--max-extracted-bytes", "1G"][..], + &["--max-extracted-bytes=1G"], + ] { + let msg = removed_flag(&args(argv)).expect("refused"); + assert!(msg.contains("--max-process-bytes"), "{msg}"); + } + assert_eq!(removed_flag(&args(&["--max-object-bytes", "1G"])), None); + std::env::set_var("EXAV_MAX_EXTRACTED_BYTES", "1G"); + let msg = removed_flag(&[]); + std::env::remove_var("EXAV_MAX_EXTRACTED_BYTES"); + assert!(msg.is_some_and(|m| m.contains("EXAV_MAX_EXTRACTED_BYTES"))); + } + + /// Counts that are limits take `off`; the depth and the DLP thresholds, + /// where no limit or a zero means nothing useful, refuse them. + #[test] + fn counts_read_off_as_no_limit_or_refuse_it() { + let _env = env_guard(); + for v in ["0", "off"] { + let opts = build_scan_options(&Cli::parse_from(["exav", "--max-members", v])); + assert_eq!(opts.limits.max_members, u64::MAX, "--max-members {v}"); + } + for bad in [ + &["--max-unpack-depth", "0"][..], + &["--max-unpack-depth", "off"], + &["--dlp-ssns", "0"], + &["--dlp-credit-cards", "0"], + &["--spill-threshold-bytes", "off"], + &["--update-interval-secs", "0"], + ] { + assert!( + Cli::try_parse_from(["exav"].iter().chain(bad.iter()).copied()).is_err(), + "{bad:?}" + ); + } + assert_eq!( + Cli::parse_from(["exav", "--spill-threshold-bytes", "0"]).spill_threshold, + Some(0) + ); + } + /// `off` (and `0`) on a limit is "no limit", never a limit of zero. #[test] fn off_lifts_a_limit() { @@ -5291,7 +5465,38 @@ mod tests { // Naming services replaces the defaults rather than adding to them, so // exav never answers on a name nobody configured. assert_eq!(cfg.services, ["one", "two"]); - assert_eq!(cfg.preview_size, 512); + assert_eq!(cfg.preview_size, Some(512)); + + // `0` and `off` are different settings where both mean something, and + // where no limit would leave a client able to hold a connection, there + // is no `off`. + let icap = |args: &[&str]| { + icap::config_from_cli(&Cli::parse_from( + std::iter::once("exav").chain(args.iter().copied()), + )) + .unwrap() + }; + assert_eq!(icap(&["--icap-preview-bytes", "0"]).preview_size, Some(0)); + assert_eq!(icap(&["--icap-preview-bytes", "off"]).preview_size, None); + assert_eq!(icap(&["--icap-options-ttl-secs", "0"]).options_ttl, Some(0)); + assert_eq!(icap(&["--icap-options-ttl-secs", "off"]).options_ttl, None); + for v in ["0", "off"] { + assert_eq!( + icap(&["--icap-max-requests", v]).keepalive_requests, + u64::MAX + ); + } + for bad in [ + &["--icap-idle-secs", "0"][..], + &["--icap-idle-secs", "off"], + &["--icap-max-header-bytes", "off"], + &["--icap-max-header-bytes", "100"], + ] { + assert!( + Cli::try_parse_from(["exav"].iter().chain(bad.iter()).copied()).is_err(), + "{bad:?}" + ); + } // The ICAP listener has no size ceiling of its own to configure: an // object's size is `--max-input-bytes`, the same on every surface. assert!( diff --git a/crates/exav/src/policy.rs b/crates/exav/src/policy.rs index 2524140c..666f2489 100644 --- a/crates/exav/src/policy.rs +++ b/crates/exav/src/policy.rs @@ -64,8 +64,8 @@ pub(crate) fn apply(report: &mut ScanReport, policy: PartialAs) { // Loud, per object. A pass an operator configured is a risk they // accepted; a pass they cannot count is one they cannot review. eprintln!( - "exav: reporting an object that could not be fully examined as OK ({tag}: {}) \ - — --partial-as says so", + "exav: reporting an object that could not be fully examined as OK ({tag}: {}), \ + as --partial-as says", report.verdict.detail().unwrap_or_default() ); report.verdict = Verdict::Clean; diff --git a/crates/exav/src/signatures.rs b/crates/exav/src/signatures.rs index 26f02e09..3e0699f7 100644 --- a/crates/exav/src/signatures.rs +++ b/crates/exav/src/signatures.rs @@ -16,8 +16,7 @@ //! * pull a prebuilt `.exavdb` over HTTP (`--db-url`) as an alternative to //! fetching signature files, with the same change detection and reload. //! -//! Unix only: the reload it drives belongs to the prefork supervisor, and the -//! fetching half needs a build with `--features http-update`. +//! Unix only; the fetching half needs a build with `--features http-update`. #![cfg(unix)] use std::path::{Path, PathBuf}; @@ -274,15 +273,14 @@ fn redact_url(url: &str) -> String { url } -/// Whether a reload has anything to act on: only the prefork supervisor re-forks -/// its pool on request. Elsewhere the change is picked up by the mtime watch on -/// the same path, and raising `SIGHUP` in a process with no handler for it would -/// kill the server instead of reloading it. +/// Whether a reload has anything to act on in this process. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum Reload { - /// Signal the prefork supervisor as soon as an update lands. + /// A listener's supervisor runs here: ask it to reload as soon as an + /// update lands. Signal, - /// Leave it to the mtime watch on the signature path. + /// Nothing is served here (`--auto-update` alone); whoever serves the + /// directory picks the change up from its mtime watch. Watch, } @@ -314,8 +312,8 @@ pub(crate) struct AutoUpdate { /// The directory the sources are fetched into. `None` in prebuilt-database /// mode, where [`start`] already owns the polling. dir: Option, - /// Time between source re-checks. - interval: Duration, + /// Time between source re-checks; `None` when they are off. + interval: Option, /// How an installed update reaches the running server. reload: Reload, /// Per-source HTTP validators learned by the initial fetch. @@ -323,26 +321,35 @@ pub(crate) struct AutoUpdate { validators: std::collections::HashMap>, } -/// How often the sources are re-checked, from `--update-interval-secs`. -fn interval(cli: &Cli) -> Duration { - Duration::from_secs( +/// How often the sources are re-checked, from `--update-interval-secs`; +/// `None` for `off`, which fetches at startup only. +fn interval(cli: &Cli) -> Option { + every( cli.update_interval_secs - .unwrap_or(DEFAULT_UPDATE_INTERVAL_SECS) - .max(MIN_UPDATE_INTERVAL_SECS), + .unwrap_or(DEFAULT_UPDATE_INTERVAL_SECS), ) } /// How often a `--db-url` prebuilt database is re-checked, from the same flag. /// A conditional `HEAD`, so the default cadence is minutes rather than the day -/// a full source fetch gets — but the same 60-second floor applies: a short -/// poll aimed at someone else's server is the same loop whatever it checks. +/// a full source fetch gets. The same 60-second floor applies: a short poll +/// aimed at someone else's server is the same loop whatever it checks. #[cfg(feature = "http-update")] -fn db_poll_interval(cli: &Cli) -> Duration { - Duration::from_secs( - cli.update_interval_secs - .unwrap_or(DEFAULT_DB_URL_POLL_SECS) - .max(MIN_UPDATE_INTERVAL_SECS), - ) +fn db_poll_interval(cli: &Cli) -> Option { + every(cli.update_interval_secs.unwrap_or(DEFAULT_DB_URL_POLL_SECS)) +} + +/// `secs` as a polling period, floored; `None` for `0`, which `off` parses to. +fn every(secs: u64) -> Option { + (secs != 0).then(|| Duration::from_secs(secs.max(MIN_UPDATE_INTERVAL_SECS))) +} + +/// How a polling period reads in a log line. +fn describe(every: Option) -> String { + match every { + Some(d) => format!("every {}s", d.as_secs()), + None => "at startup only (--update-interval-secs off)".to_string(), + } } /// Seconds to wait for a sidecar to fill an empty signature dir. `--auto-update` @@ -398,9 +405,9 @@ pub(crate) fn start(cli: &mut Cli, reload: Reload) -> Result, let interval = interval(cli); eprintln!( - "exav: {} signature source(s), refreshed every {}s", + "exav: {} signature source(s), fetched {}", sources.len(), - interval.as_secs() + describe(interval) ); // A source is saved under the filename its URL ends in, and the loader routes @@ -416,7 +423,7 @@ pub(crate) fn start(cli: &mut Cli, reload: Reload) -> Result, if !exav_core::loader::loads_by_extension(&name) { eprintln!( "exav: {url} saves as `{name}`, which the loader does not recognise as a \ - database — it will be fetched and not loaded. Signatures are routed by \ + database: it will be fetched and not loaded. Signatures are routed by \ extension: name the URL after the file it serves (.cvd, .cld, .ndb, .hdb, \ .ldb, .yar, …)." ); @@ -426,7 +433,7 @@ pub(crate) fn start(cli: &mut Cli, reload: Reload) -> Result, #[cfg(not(feature = "http-update"))] { eprintln!( - "exav: signature source URLs are set but this build has no updater — rebuild \ + "exav: signature source URLs are set but this build has no updater: rebuild \ with `--features http-update`, or populate {} from a sidecar. Continuing.", dir.display() ); @@ -465,7 +472,7 @@ fn start_prebuilt_db( ) -> Result<(), String> { if !sources.is_empty() { eprintln!( - "exav: --db-url is set — ignoring --sig-sources \ + "exav: --db-url is set, ignoring --sig-sources \ (serving a prebuilt database, not signature files)" ); } @@ -505,10 +512,11 @@ fn start_prebuilt_db( } }; - eprintln!( - "exav: polling {shown} for database changes every {}s", - poll.as_secs() - ); + eprintln!("exav: database from {shown} fetched {}", describe(poll)); + let Some(poll) = poll else { + cli.database = Some(dest); + return Ok(()); + }; let url = url.to_string(); let watched = dest.clone(); std::thread::spawn(move || loop { @@ -517,7 +525,7 @@ fn start_prebuilt_db( Ok(f) => { validator = f.validator().map(String::from); if f.is_updated() { - eprintln!("exav: database updated from {shown} — reloading"); + eprintln!("exav: database updated from {shown}, reloading"); reload.request(); } } @@ -536,7 +544,7 @@ fn start_prebuilt_db( _sources: &[String], _reload: Reload, ) -> Result<(), String> { - Err("--db-url needs the updater — build with `--features http-update`".to_string()) + Err("--db-url needs the updater: build with `--features http-update`".to_string()) } impl AutoUpdate { @@ -553,7 +561,9 @@ impl AutoUpdate { reload, mut validators, } = self; - let Some(dir) = dir else { return }; + let (Some(dir), Some(interval)) = (dir, interval) else { + return; + }; std::thread::spawn(move || loop { std::thread::sleep(interval); if update_signatures(&sources, &dir, &mut validators) { @@ -580,6 +590,8 @@ impl AutoUpdate { mut validators, } = self; let dir = dir.expect("a prebuilt database poller is never an updater-only run"); + let interval = + interval.expect("`updater_only_error` refuses an updater that never polls"); eprintln!( "exav: updater-only: refreshing {} every {}s", dir.display(), @@ -629,11 +641,18 @@ pub(crate) fn updater_only_error(cli: &Cli) -> Option { if cli.db_url.is_some() { return Some( "an updater fetches signature files; --db-url pulls a prebuilt \ - database, which is served rather than written for another process — \ + database, which is served rather than written for another process: \ add --listen" .to_string(), ); } + if interval(cli).is_none() { + return Some( + "--auto-update with no listener is an updater, and with \ + --update-interval-secs off it would fetch once and then do nothing" + .to_string(), + ); + } None } @@ -741,23 +760,23 @@ mod tests { let cli = |args: &[&str]| Cli::parse_from(std::iter::once("exav").chain(args.iter().copied())); + let secs = |args: &[&str]| interval(&cli(args)).map(|d| d.as_secs()); + assert_eq!(secs(&[]), Some(24 * 3600), "once a day is the default"); assert_eq!( - interval(&cli(&[])).as_secs(), - 24 * 3600, - "once a day is the default" - ); - assert_eq!( - interval(&cli(&["--update-interval-secs", "3600"])).as_secs(), - 3600, + secs(&["--update-interval-secs", "3600"]), + Some(3600), "an interval in seconds is used as given" ); - for too_short in ["0", "1", "59"] { + for too_short in ["1", "59"] { assert_eq!( - interval(&cli(&["--update-interval-secs", too_short])).as_secs(), - MIN_UPDATE_INTERVAL_SECS, + secs(&["--update-interval-secs", too_short]), + Some(MIN_UPDATE_INTERVAL_SECS), "--update-interval-secs {too_short} must not become a delay-free loop" ); } + // `off` stops the re-checks; it used to poll every minute. + assert_eq!(secs(&["--update-interval-secs", "off"]), None); + assert!(Cli::try_parse_from(["exav", "--update-interval-secs", "0"]).is_err()); } #[cfg(feature = "http-update")] @@ -767,18 +786,20 @@ mod tests { let cli = |args: &[&str]| Cli::parse_from(std::iter::once("exav").chain(args.iter().copied())); + let secs = |args: &[&str]| db_poll_interval(&cli(args)).map(|d| d.as_secs()); assert_eq!( - db_poll_interval(&cli(&[])).as_secs(), - 300, + secs(&[]), + Some(300), "a pulled database re-checks every five minutes by default" ); - for too_short in ["0", "1", "59"] { + for too_short in ["1", "59"] { assert_eq!( - db_poll_interval(&cli(&["--update-interval-secs", too_short])).as_secs(), - MIN_UPDATE_INTERVAL_SECS, + secs(&["--update-interval-secs", too_short]), + Some(MIN_UPDATE_INTERVAL_SECS), "--update-interval-secs {too_short} must not hammer the database host" ); } + assert_eq!(secs(&["--update-interval-secs", "off"]), None); } /// Waiting for a sidecar is what `--auto-update` brings with it. Without the diff --git a/crates/exav/src/spill.rs b/crates/exav/src/spill.rs index bc461b72..b5fc81f3 100644 --- a/crates/exav/src/spill.rs +++ b/crates/exav/src/spill.rs @@ -25,8 +25,9 @@ //! sharing that filesystem. //! //! Exceeding a budget is not an I/O failure and must not be reported as one. The -//! object is simply one exav could not examine, which is a verdict every surface -//! already knows how to say ([`SpillError::Budget`] → `UNSCANNABLE`). Dropping +//! object is one exav could examine only as far as it held, which is a verdict +//! every surface already knows how to say ([`SpillError::Budget`]: what was held +//! is scanned, and without a detection it is `LIMITS-EXCEEDED`). Dropping //! the connection instead would leave the client with no answer at all, and a //! client with no answer is a client that decides for itself. @@ -58,8 +59,9 @@ pub(crate) struct SpillConfig { /// /// Off, nothing is ever written to a temp file and /// [`threshold`](Self::threshold) becomes a hard per-object memory ceiling: - /// an object that outgrows it is `UNSCANNABLE`, because there is nowhere - /// left to put it. That is the shape a read-only root filesystem needs, and + /// an object that outgrows it has what was held scanned and is + /// `LIMITS-EXCEEDED` unless something is found. That is the shape a + /// read-only root filesystem needs, and /// the one a deployment picks when it would rather refuse a large object /// than let a scanned payload touch a disk at all. pub enabled: bool, @@ -260,6 +262,54 @@ impl Drop for SpillFile { } } +/// Spill space for the scan itself: what it makes of an object too large to +/// hold (normalised text, a member too large to buffer) goes to the same temp +/// files, under the same budgets, as the objects the listeners spill. +pub(crate) struct ScanSpill; + +impl exav_core::spill::Spill for ScanSpill { + fn create(&self) -> Result, String> { + if !config().enabled { + return Err("spilling to disk is off (--spill-dir off)".to_string()); + } + let file = SpillFile::create().map_err(|e| e.to_string())?; + Ok(Box::new(file)) + } +} + +impl exav_core::spill::SpillWriter for SpillFile { + fn write(&mut self, bytes: &[u8]) -> Result<(), String> { + self.write_all(bytes).map_err(|e| e.to_string()) + } + + fn finish(self: Box) -> Result, String> { + let file = self.reopen().map_err(|e| format!("temporary file: {e}"))?; + Ok(Box::new(ReadBack { + file, + _spill: *self, + })) + } +} + +/// A spill file being read back. Its share of the budget is held until this is +/// dropped. +struct ReadBack { + file: std::fs::File, + _spill: SpillFile, +} + +impl io::Read for ReadBack { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.file.read(buf) + } +} + +impl io::Seek for ReadBack { + fn seek(&mut self, to: io::SeekFrom) -> io::Result { + self.file.seek(to) + } +} + /// Render a byte count the way an operator writes one, so a message about a /// 64 KiB budget does not read "0 MiB". pub(crate) fn human_bytes(n: u64) -> String { diff --git a/crates/exav/tests/daemon_client.rs b/crates/exav/tests/daemon_client.rs index 5a2d4df5..3cb12fc6 100644 --- a/crates/exav/tests/daemon_client.rs +++ b/crates/exav/tests/daemon_client.rs @@ -224,7 +224,7 @@ fn the_thread_model_socket_is_permissioned_too() { } /// A mode is octal, and a mode that is not one is refused rather than applied. -/// `666` read as decimal is 0o1232 — a setgid socket nobody asked for. +/// `666` read as decimal is 0o1232, a setgid socket nobody asked for. #[test] fn a_mode_that_is_not_a_mode_is_refused() { // Under the test's own directory rather than a fixed `/tmp` name: the @@ -304,7 +304,7 @@ fn the_daemon_logs_what_it_answered() { } /// `--send-as contents` sends the file's bytes (`INSTREAM`), so the daemon needs -/// no access to the path — the case a milter or a container deployment is in. +/// no access to the path: the case a milter or a container deployment is in. /// /// The daemon's own log is the proof of which verb ran: it records a streamed /// scan as `stream:` and a path scan under the path it was given. @@ -415,7 +415,7 @@ fn a_split_set_is_rejoined_when_streamed() { /// `--verbose` in client mode. The daemon's reply is a verdict and nothing /// else, so what `-v` has to add here is which daemon answered and what it was -/// asked — accepting the flag and printing nothing is the defect. +/// asked. Accepting the flag and printing nothing is the defect. #[test] fn verbose_names_the_daemon_and_the_command() { let d = Daemon::start("077", &[]); @@ -440,7 +440,7 @@ fn verbose_names_the_daemon_and_the_command() { ); } -/// A stream the daemon could not examine comes back as `PARTIAL`, exit 3 — the +/// A stream the daemon could not examine comes back as `PARTIAL`, exit 3: the /// same answer a local scan of the same object gives. /// /// The wire grammar is `: ERROR`, because clamd has no @@ -452,7 +452,7 @@ fn verbose_names_the_daemon_and_the_command() { #[test] fn an_unexaminable_stream_is_partial_over_the_wire_not_a_hard_error() { // Nowhere to spill and almost no room in RAM, so any real object is one the - // daemon cannot examine — the condition, reached the quickest way. + // daemon cannot examine: the condition, reached the quickest way. let d = Daemon::start( "077", &["--spill-dir", "off", "--spill-threshold-bytes", "1M"], @@ -492,6 +492,43 @@ fn a_timed_out_job_gets_an_answer() { ); } +/// The CPU limit is per job. `RLIMIT_CPU` counts a process's whole life, so set +/// once per worker it killed whichever job ran when the jobs before it had used +/// the budget between them, and that job got no reply. +#[test] +fn the_cpu_limit_is_per_job_not_per_worker() { + use std::io::{Read, Write}; + let d = Daemon::start("077", &["--workers", "1", "--max-scan-secs", "1"]); + // Random bytes, so the scan is CPU work rather than a cache hit. + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let body: Vec = (0..8 << 20) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect(); + let f = d.file("noise.bin", &body); + let start = Instant::now(); + let mut jobs = 0; + while start.elapsed() < Duration::from_secs(4) || jobs < 5 { + let mut s = std::os::unix::net::UnixStream::connect(&d.sock).unwrap(); + s.write_all(format!("zSCAN {}\0", f.display()).as_bytes()) + .unwrap(); + s.set_read_timeout(Some(Duration::from_secs(30))).unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + let reply = String::from_utf8_lossy(&out); + assert!( + reply.trim_end_matches('\0').ends_with("OK"), + "job {jobs} after {:?}: {reply:?}", + start.elapsed() + ); + jobs += 1; + } +} + /// `EXINSTREAM` answers a stream it could not hold as `INSTREAM` does: partial, /// over a limit, not an error. `MULTI` answers it for that file alone. #[test] @@ -663,7 +700,7 @@ fn verbose_stays_out_of_the_json_stream() { /// `clamdscan` has no `-r`, so a command line migrated from it hands the client /// a bare directory. Answering for one file in it and exiting 0 is a clean /// verdict over a tree that holds a detection, which is the one thing a scanner -/// may never report — so the default has to be the whole tree. +/// may never report, so the default has to be the whole tree. /// /// Asking for the top level is a different matter: `--no-recursive` is exav's /// own flag and the caller typed it. The client does its own walking, so it is @@ -725,7 +762,7 @@ fn the_summary_counts_every_file_the_daemon_answered_for() { } /// A directory and a file in one run. Both are answered on the same session, so -/// a reply left unread by the first command is read as the second's answer — +/// a reply left unread by the first command is read as the second's answer, /// which loses the file's verdict and misattributes the tree's. #[test] fn a_directory_beside_a_file_keeps_every_answer() { @@ -767,6 +804,21 @@ fn allmatch_answers_for_every_file_in_a_tree() { assert_eq!(code, 1, "{stdout}"); } +/// `--all-matches` on a file the daemon could not fully examine is partial, as +/// without the flag; it used to exit 0. +#[test] +fn allmatch_reports_a_partial_file() { + let d = Daemon::start("077", &[]); + let dir = TempDir::new().unwrap(); + let mut gz = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]; + gz.extend_from_slice(&[0xff; 64]); + std::fs::write(dir.path().join("damaged.gz"), gz).unwrap(); + + let (code, stdout) = d.client(&["--all-matches"], &[dir.path()]); + assert!(stdout.contains("UNSCANNABLE"), "{stdout}"); + assert_eq!(code, 3, "{stdout}"); +} + /// `--all-matches` over a byte-split set. No part decodes on its own, so scanning /// the parts as the files they are finds nothing; the set has to be rejoined /// whichever flag was passed, or the answer is clean over an archive nobody @@ -795,7 +847,7 @@ fn allmatch_rejoins_a_split_set() { /// A daemon of the test's own answers one `SCAN` with two messages, which is /// what a real one does whenever the path it was handed names more than one /// file. A client that reads a fixed single line reports the first and drops -/// the second — and the second is the detection here. +/// the second, and the second is the detection here. #[test] fn every_reply_message_of_a_scan_is_reported() { use std::io::Write; @@ -874,7 +926,7 @@ fn every_reply_message_of_a_scan_is_reported() { /// The premise the client's session framing rests on: one command, any number /// of replies. `SCAN` over a directory answers once per file, every message -/// tagged with the same command id and nothing marking the last — so a client +/// tagged with the same command id and nothing marking the last, so a client /// that reads a fixed line count drops verdicts, and the one it sends behind /// the scan (`PING`) is what tells it the scan has finished talking. #[test] diff --git a/crates/exav/tests/detect.rs b/crates/exav/tests/detect.rs new file mode 100644 index 00000000..eb3d87a3 --- /dev/null +++ b/crates/exav/tests/detect.rs @@ -0,0 +1,34 @@ +//! `--detect`: the heuristic detectors a default build carries run when asked. +//! +//! A detector compiled out of the binary accepts its flag and finds nothing, so +//! the only check that it is really there is a scan from outside. + +use std::process::Command; + +#[path = "../src/tmpfile.rs"] +mod tmpfile; +use tmpfile::TempDir; + +/// A link whose text names one site and whose target is another. +#[test] +fn detect_phishing_runs_in_a_default_build() { + let dir = TempDir::new().unwrap(); + let page = dir.path().join("mail.html"); + std::fs::write( + &page, + br#"https://www.paypal.com/signin"#, + ) + .unwrap(); + let out = Command::new(env!("CARGO_BIN_EXE_exav")) + .env("EXAV_ALLOW_NO_DB", "1") + .args(["--detect", "phishing"]) + .arg(&page) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + stdout.contains("Heuristics.Phishing.Email.SpoofedDomain FOUND"), + "{stdout}\n{}", + String::from_utf8_lossy(&out.stderr) + ); +} diff --git a/crates/exav/tests/docs_match_cli.rs b/crates/exav/tests/docs_match_cli.rs index 3d21d3eb..9034d585 100644 --- a/crates/exav/tests/docs_match_cli.rs +++ b/crates/exav/tests/docs_match_cli.rs @@ -348,6 +348,8 @@ fn every_flag_named_in_a_message_exists() { "alert-phishing-cloak", "alert-partition-intersection", "alert-encrypted", + "alert-encrypted-archive", + "alert-encrypted-doc", "alert-exceeds-max", "structured-ssn-count", "structured-cc-count", @@ -361,6 +363,8 @@ fn every_flag_named_in_a_message_exists() { // Spellings named to explain why they are not the ones exav uses. "max-depth", "password", + // Removed from exav, named in the error that says what replaced it. + "max-extracted-bytes", ]; let real = flags_in_help(); diff --git a/crates/exav/tests/listeners.rs b/crates/exav/tests/listeners.rs index 7a1b516b..8a78313f 100644 --- a/crates/exav/tests/listeners.rs +++ b/crates/exav/tests/listeners.rs @@ -459,6 +459,157 @@ fn a_rewritten_signature_directory_is_reloaded() { } } +/// Rewrite a signature file in place and put its mtime back, so the directory +/// watch sees no change and only an explicit reload can pick it up. +fn rewrite_unseen(dir: &Path, name: &str, body: &str) { + let p = dir.join(name); + let before = std::fs::metadata(&p).unwrap().modified().unwrap(); + std::fs::write(&p, body).unwrap(); + std::fs::File::options() + .write(true) + .open(&p) + .unwrap() + .set_modified(before) + .unwrap(); +} + +fn signal(server: &Server, sig: libc::c_int) { + // SAFETY: signals the child this test started and has not reaped. + assert_eq!( + unsafe { libc::kill(server.child.id() as libc::pid_t, sig) }, + 0 + ); +} + +/// Scan `malwareD` until `Exav.Test.Delta` detects it: the reload that brought +/// it in has taken effect. +fn until_delta(scan: impl Fn() -> String, server: &Server) { + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let got = scan(); + if got.contains("Exav.Test.Delta") { + return; + } + assert!( + Instant::now() < deadline, + "the reload never took effect: {got:?}\n{}", + server.stderr() + ); + std::thread::sleep(Duration::from_millis(100)); + } +} + +const WITH_DELTA: &str = "Exav.Test.Alpha:0:*:6d616c7761726541\n\ + Exav.Test.Beta:0:*:6d616c7761726542\n\ + Exav.Test.Delta:0:*:6d616c7761726544\n"; + +/// As in clamd: `SIGUSR2` reloads the signatures and `SIGHUP` reopens the +/// `--log` file, whatever serves. Neither stops the daemon. +#[test] +fn signals_reload_and_reopen_the_log_like_clamd() { + for workers in ["2", "threads"] { + let sigs = sig_dir(); + let dir = TempDir::new().unwrap(); + let log = dir.path().join("scans.log"); + let clamd = free_port(); + let server = Server::start( + dir, + &[ + "--workers", + workers, + "--log", + log.to_str().unwrap(), + "--listen", + &format!("clamd://127.0.0.1:{clamd}"), + "--sig-dir", + sigs.path().to_str().unwrap(), + ], + ); + let scan = || until_answered("scan", || try_clamd_instream(clamd, b"malwareD")); + assert!(scan().contains(": OK"), "workers={workers}"); + + // Rotation: the log moves away, SIGHUP, and the next line lands in a + // new file at the configured path. + std::fs::rename(&log, log.with_extension("1")).unwrap(); + signal(&server, libc::SIGHUP); + let deadline = Instant::now() + Duration::from_secs(60); + while !std::fs::read_to_string(&log).is_ok_and(|s| s.contains(": OK")) { + assert!( + Instant::now() < deadline, + "workers={workers}: the log was never reopened:\n{}", + server.stderr() + ); + scan(); + std::thread::sleep(Duration::from_millis(100)); + } + assert_eq!( + clamd_ping(clamd), + "PONG", + "workers={workers}: SIGHUP stopped it" + ); + + rewrite_unseen(sigs.path(), "test.ndb", WITH_DELTA); + signal(&server, libc::SIGUSR2); + wait_for_log(&server, "reloading signatures"); + until_delta(scan, &server); + } +} + +/// `RELOAD` reloads in the thread model too; it used to answer `RELOADING` +/// and do nothing. +#[test] +fn reload_works_in_the_thread_model() { + let sigs = sig_dir(); + let clamd = free_port(); + let server = Server::start( + TempDir::new().unwrap(), + &[ + "--workers", + "threads", + "--listen", + &format!("clamd://127.0.0.1:{clamd}"), + "--sig-dir", + sigs.path().to_str().unwrap(), + ], + ); + let scan = || until_answered("scan", || try_clamd_instream(clamd, b"malwareD")); + assert!(scan().contains(": OK")); + rewrite_unseen(sigs.path(), "test.ndb", WITH_DELTA); + let mut s = dial(clamd, "clamd"); + s.write_all(b"zRELOAD\0").unwrap(); + let mut out = Vec::new(); + let _ = s.read_to_end(&mut out); + assert_eq!( + String::from_utf8_lossy(&out).trim_end_matches('\0'), + "RELOADING" + ); + wait_for_log(&server, "reloading signatures"); + until_delta(scan, &server); +} + +/// ICAP served alone reloads on `SIGUSR2`, and `SIGHUP` does not stop it. +#[test] +fn icap_alone_reloads_on_sigusr2() { + let sigs = sig_dir(); + let icap = free_port(); + let server = Server::start( + TempDir::new().unwrap(), + &[ + "--listen", + &format!("icap://127.0.0.1:{icap}"), + "-d", + sigs.path().to_str().unwrap(), + ], + ); + wait_for_log(&server, "serving ICAP on tcp:"); + assert!(!icap_respmod(icap, b"malwareD").contains("Exav.Test.Delta")); + signal(&server, libc::SIGHUP); + rewrite_unseen(sigs.path(), "test.ndb", WITH_DELTA); + signal(&server, libc::SIGUSR2); + wait_for_log(&server, "reloading signatures"); + until_delta(|| icap_respmod(icap, b"malwareD"), &server); +} + /// A reload does not cut short a scan already in progress: the worker holding /// it finishes the job on the database it started with, then retires. #[test] diff --git a/crates/exav/tests/self_scan.rs b/crates/exav/tests/self_scan.rs index c5b8e7fc..2b96e309 100644 --- a/crates/exav/tests/self_scan.rs +++ b/crates/exav/tests/self_scan.rs @@ -89,7 +89,8 @@ fn source_files(root: &Path) -> Vec { p.extension().and_then(|s| s.to_str()), Some("rs" | "py" | "sh" | "toml" | "md" | "mdx" | "yml" | "yaml" | "js" | "ts") ); - !in_fixtures && is_text + // A tracked file deleted in the working tree ships nowhere. + !in_fixtures && is_text && p.exists() }) .collect(); diff --git a/crates/exav/tests/value_vocabulary.rs b/crates/exav/tests/value_vocabulary.rs index 017d9a2d..3979e9e6 100644 --- a/crates/exav/tests/value_vocabulary.rs +++ b/crates/exav/tests/value_vocabulary.rs @@ -22,13 +22,19 @@ fn exav() -> Command { /// /// A flag may still be refused later for needing a listener it was not given. /// That is a different check, and it means the value itself parsed, which is -/// what is under test. +/// what is under test. A flag the binary does not have fails the test rather +/// than passing it: an unknown flag is not an invalid value. fn parses(flag: &str, value: &str) -> bool { let out = exav() .args([flag, value, "/etc/hosts"]) .output() .expect("run exav"); - !String::from_utf8_lossy(&out.stderr).contains("invalid value") + let err = String::from_utf8_lossy(&out.stderr); + assert!( + !err.contains("unexpected argument"), + "{flag} is not a flag: {err}" + ); + !err.contains("invalid value") } /// Byte-valued flags all read the same size vocabulary. @@ -37,23 +43,38 @@ fn parses(flag: &str, value: &str) -> bool { /// plain integer parser to match, so `4K` was a parse error on them and fine on /// every other flag measured in bytes. The field's width is not something the /// person typing can see. +/// +/// `0` and `off` are where they differ, on purpose: a limit takes both; a size +/// that is not a limit (`--spill-threshold-bytes`) takes `0` and refuses `off`; +/// `--icap-max-header-bytes` has a floor of `1K` and no `off`. #[test] fn every_byte_flag_takes_the_same_sizes() { - const BYTE_FLAGS: &[&str] = &[ + const LIMITS: &[&str] = &[ "--max-input-bytes", - "--max-extracted-bytes", "--max-object-bytes", "--max-matcher-bytes", "--max-process-bytes", "--max-spill-bytes", "--max-total-spill-bytes", - "--spill-threshold-bytes", "--build-shard-bytes", "--icap-preview-bytes", - "--icap-max-header-bytes", ]; - for flag in BYTE_FLAGS { - for good in ["4096", "45K", "45M", "45G", "45m", "0", "off"] { + const SIZES: &[&str] = &["--spill-threshold-bytes", "--icap-max-header-bytes"]; + for flag in LIMITS { + for good in ["0", "off"] { + assert!(parses(flag, good), "{flag} must accept `{good}`"); + } + } + assert!(parses("--spill-threshold-bytes", "0")); + for flag in SIZES { + assert!(!parses(flag, "off"), "{flag} is a size, with no `off`"); + } + assert!( + !parses("--icap-max-header-bytes", "0"), + "below the 1K floor" + ); + for flag in LIMITS.iter().chain(SIZES) { + for good in ["4096", "45K", "45M", "45G", "45m"] { assert!(parses(flag, good), "{flag} must accept `{good}`"); } // `M` is binary here, so `MB` would be a 4.9% lie at that suffix and a @@ -70,30 +91,35 @@ fn every_byte_flag_takes_the_same_sizes() { /// Duration flags all disable themselves with the same word. /// -/// `off` reaches the disabled state on every one of them. `0` additionally works -/// wherever zero is a meaningful bound, and is refused on the two that describe a -/// recurring interval, where a zero-second period is a loop rather than "never". -/// That refusal explains itself; it is the one difference in this family and it -/// is not arbitrary. +/// `off` reaches the disabled state on every one that can be disabled. `0` +/// additionally works wherever zero is a meaningful bound, and is refused on the +/// ones that describe a recurring interval, where a zero-second period is a loop +/// rather than "never". `--icap-idle-secs` has neither: with no idle timeout a +/// client could hold a connection forever, and with a zero one no request +/// would get through. #[test] fn every_duration_flag_disables_with_off() { const BOUNDS: &[&str] = &[ "--max-scan-secs", "--startup-wait-secs", - "--update-interval-secs", - "--icap-idle-secs", "--icap-options-ttl-secs", ]; - const PERIODS: &[&str] = &["--metrics-secs", "--slow-scan-secs"]; + const PERIODS: &[&str] = &[ + "--metrics-secs", + "--slow-scan-secs", + "--update-interval-secs", + ]; - for flag in BOUNDS.iter().chain(PERIODS) { + for flag in BOUNDS.iter().chain(PERIODS).chain(&["--icap-idle-secs"]) { assert!( parses(flag, "120"), "{flag} must accept a number of seconds" ); - assert!(parses(flag, "off"), "{flag} must accept `off`"); assert!(!parses(flag, "45M"), "{flag} is seconds, not a size"); } + for flag in BOUNDS.iter().chain(PERIODS) { + assert!(parses(flag, "off"), "{flag} must accept `off`"); + } for flag in BOUNDS { assert!(parses(flag, "0"), "{flag}: `0` is a bound of zero"); } @@ -103,6 +129,8 @@ fn every_duration_flag_disables_with_off() { "{flag}: a zero-second period is a loop, not `off`" ); } + assert!(!parses("--icap-idle-secs", "off")); + assert!(!parses("--icap-idle-secs", "0")); } /// Counting flags take counts, and nothing that looks like a size. @@ -110,21 +138,33 @@ fn every_duration_flag_disables_with_off() { /// The separation is what lets a reader tell the families apart at a glance: a /// suffix means bytes. `--max-members 10K` quietly meaning ten thousand members /// would remove that signal from the whole surface. +/// +/// A limit on a count takes `0` and `off` for no limit. The nesting depth has +/// no unlimited setting (each level costs stack), and a threshold of findings +/// at `0` would fire on every file, so those refuse both. #[test] fn every_counting_flag_refuses_size_suffixes() { - const COUNTS: &[&str] = &[ + const LIMITS: &[&str] = &[ "--max-members", - "--max-unpack-depth", "--max-jobs-per-worker", "--icap-max-requests", - "--dlp-ssns", - "--dlp-credit-cards", ]; - for flag in COUNTS { - for good in ["10", "0", "10000"] { + const AT_LEAST_ONE: &[&str] = &["--max-unpack-depth", "--dlp-ssns", "--dlp-credit-cards"]; + for flag in LIMITS.iter().chain(AT_LEAST_ONE) { + for good in ["10", "10000"] { + assert!(parses(flag, good), "{flag} must accept `{good}`"); + } + for bad in ["10K", "10M", "banana"] { + assert!(!parses(flag, bad), "{flag} must refuse `{bad}`"); + } + } + for flag in LIMITS { + for good in ["0", "off"] { assert!(parses(flag, good), "{flag} must accept `{good}`"); } - for bad in ["10K", "10M", "off", "banana"] { + } + for flag in AT_LEAST_ONE { + for bad in ["0", "off"] { assert!(!parses(flag, bad), "{flag} must refuse `{bad}`"); } } diff --git a/crates/exav/tests/walk_errors.rs b/crates/exav/tests/walk_errors.rs index 5978499d..2714248a 100644 --- a/crates/exav/tests/walk_errors.rs +++ b/crates/exav/tests/walk_errors.rs @@ -88,6 +88,48 @@ fn an_unreadable_subdirectory_is_reported_and_fails_the_run() { ); } +/// `--quiet` drops the `OK` lines and the summary, never an error, and an error +/// reaches `--log` as a detection does. +#[test] +fn an_error_is_printed_under_quiet_and_logged() { + let dir = TempDir::new().expect("temp dir"); + let log = dir.path().join("scan.log"); + let missing = dir.path().join("missing.bin"); + let out = exav() + .arg("--quiet") + .arg("--log") + .arg(&log) + .arg(&missing) + .output() + .expect("run exav"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(2), "{stderr}"); + assert!( + stderr.contains("missing.bin") && stderr.contains("ERROR"), + "{stderr}" + ); + let logged = std::fs::read_to_string(&log).expect("read the log"); + assert!( + logged.contains("missing.bin") && logged.contains("ERROR"), + "{logged}" + ); +} + +/// The summary counts a file not fully examined as partial, not as an error: +/// the two exit differently, and the summary has to agree with the exit code. +#[test] +fn the_summary_counts_a_partial_file_apart_from_errors() { + let dir = TempDir::new().expect("temp dir"); + let mut gz = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]; + gz.extend_from_slice(&[0xff; 64]); + std::fs::write(dir.path().join("damaged.gz"), gz).expect("write"); + let out = exav().arg(dir.path()).output().expect("run exav"); + let stdout = String::from_utf8_lossy(&out.stdout); + assert_eq!(out.status.code(), Some(3), "{stdout}"); + assert!(stdout.contains("Partial files: 1"), "{stdout}"); + assert!(!stdout.contains("Total errors"), "{stdout}"); +} + /// The counterweight to the above: a tree with nothing wrong still exits 0. /// A test that only checks the failure direction is satisfied by a scanner that /// always fails. diff --git a/docker-compose.yml b/docker-compose.yml index 5d15f71c..782d914e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,11 @@ # container as the sidecar; its NotifyClamd RELOAD is protocol-compatible # with exav's daemon. # -# Run with: EXAV_SIG_SOURCES=https://database.clamav.net/ docker compose up -d +# Run with: EXAV_SIG_SOURCES=https://clamav-mirror.internal/ docker compose up -d +# +# The source is a mirror you run (`cvdupdate serve`, or any web server over a +# `cvdupdate` directory). Cisco's database.clamav.net serves only freshclam and +# cvdupdate, and blocks other clients. services: exav: @@ -41,6 +45,9 @@ services: EXAV_SIG_SOURCES: "${EXAV_SIG_SOURCES:?set EXAV_SIG_SOURCES to one or more signature sources you trust — a CVD mirror base ends in /}" volumes: - exav-db:/var/lib/exav + # The image's health check pings a listener, and this container has none. + healthcheck: + disable: true restart: unless-stopped volumes: diff --git a/docs/BYTECODE.md b/docs/BYTECODE.md index 58c42d8c..01d60e56 100644 --- a/docs/BYTECODE.md +++ b/docs/BYTECODE.md @@ -34,8 +34,8 @@ A `.cbc` file is line-oriented: The instruction set is an LLVM-IR-like SSA form: arithmetic, bitwise, casts, integer comparisons, branches, calls, `GEP` pointer math, `load`/`store`, `memcpy`/`memset`/`memcmp`, and intrinsics (`bswap`, …). Programs cannot make -syscalls; they reach the outside world only through a fixed **API** (~96 -functions): `read`/`seek`/`file_find` to inspect the file, PE/PDF/JSON +syscalls; they reach the outside world only through a fixed **API** (107 +functions in ClamAV's table): `read`/`seek`/`file_find` to inspect the file, PE/PDF/JSON accessors, hashing, `disasm_x86`, and `setvirusname` to report a hit. ## 2. What's actually in the live database @@ -63,20 +63,22 @@ routine* that reads bytes (`read`/`seek`/`file_find`), maybe checks PE section info, and calls `setvirusname`. The exotic, heavy APIs (`disasm_x86`, PDF/JSON, the compression codecs) appear in only a handful of programs. -## 3. Minimal scope - -Targeting the file-access + detection + info/debug APIs covers the bulk: - -- **In scope (implemented host API surface):** `setvirusname`, `read`, `seek`, - `file_byteat`, `file_find`/`file_find_limit`, `engine_functionality_level`, - `bytecode_rt_error`, the `debug_*` family, `read_number`, `memstr`. -- With exav's current parser, **66 of 85 (78%)** of the live programs use *only* - APIs in this set — i.e. they're within reach of the minimal interpreter once - instruction lowering is complete. -- **Out of scope for now (safe stubs / not executed):** `disasm_x86` (an x86 - decoder), the PDF/JSON object APIs, `inflate`/`lzma`/`bzip2`, - `jsnorm`, `matchicon`. A program needing these is loaded but not run — it can - never false-positive or crash. +## 3. Scope + +- **Implemented:** the file-access, detection and PE APIs (`setvirusname`, + `read`, `seek`, `file_byteat`, `file_find`/`file_find_limit`, `read_number`, + `memstr`, `get_pe_section`, `pe_rawaddr`, `malloc`, `write`, `extract_new`), + `disasm_x86` (over `exav-x86`), the PDF object accessors, `matchicon`, + `engine_functionality_level`, and the math/util helpers. The `debug_*` family + and `bytecode_rt_error` are accepted and do nothing. +- **Stubbed:** every other API in ClamAV's table (buffer pipes, maps, + hashsets, the inflate/bzip2/LZMA and `jsnorm` contexts, JSON accessors, trace + and environment calls) returns a fail-safe value (0, -1 or a null pointer), + so a program that calls one still runs and takes its failure path. See + [Host APIs](#host-apis-34-of-107). +- A program whose body does not decode, or that names an API outside ClamAV's + table, is loaded but never run. All 85 programs in the live database run to + completion. ## 4. Why exav's interpreter is safer than ClamAV's @@ -84,8 +86,9 @@ This is the part that matters. ClamAV's bytecode subsystem has a **documented RCE history**, because it executes DB-supplied programs in memory-unsafe C and historically via an LLVM JIT: -- **CVE-2020-37167** — heap issue in the bytecode interpreter's function-name - handling, **CVSS 9.8 (RCE)**. +- **CVE-2020-37167**: weak validation of bytecode function names in ClamBC + before 0.103.0, a code-injection flaw (CWE-94), **CVSS 3.1 8.4**, local + vector. - **ClamAV < 0.102 `bytecode_vm` code execution** (exploit-db 47687) — the bytecode VM/JIT path. - The optional **LLVM JIT** generated and ran native code from bytecode: a large @@ -95,7 +98,7 @@ exav's design removes these failure modes by construction: | Risk in ClamAV's C VM | exav | |---|---| -| Buffer overflow / UAF in the VM (the CVE-2020-37167 class) | **Pure safe Rust, no `unsafe`** — every memory access is a bounds-checked slice; an out-of-range index panics into isolation, it cannot corrupt memory | +| Memory corruption or code injection in the VM (the class of the two above) | **Pure safe Rust, no `unsafe`** — every memory access is a bounds-checked slice; an out-of-range index panics into isolation, it cannot corrupt memory | | Native code generation from bytecode (JIT spray, W^X issues) | **No JIT, ever** — interpret only | | Untrusted program escaping the sandbox (syscalls, host memory) | Program sees only bounded `Vec`s and a fixed read-only file API; no syscalls, no host pointers | | Runaway program (CPU/memory exhaustion) | **Instruction budget**, scratch-memory cap, and call-depth limit | @@ -109,8 +112,8 @@ that trade-off** — a real reason to switch. ## 5. Dangers identified (and how they're handled) -- **Decompression/alloc bombs via `malloc`/codecs** → scratch-memory cap; codecs - out of scope; allocation bounded. +- **Decompression/alloc bombs via `malloc`/codecs** → scratch-memory cap; the + codec APIs are stubs; allocation bounded. - **Infinite loops** → instruction budget (programs are not guaranteed to halt). - **Pointer math (`GEP`) out of bounds** → modeled as `(region, offset)` with checked access; never a raw pointer. @@ -141,7 +144,7 @@ that trade-off** — a real reason to switch. **live in the normal scan path**, trigger-gated per program (`lib.rs` calls `db.bytecode.scan(...)`), producing real `Method::Bytecode` detections. The gating is per-program (a program runs only when its trigger lsig matches and - its host APIs are implemented), not a global off switch. + every API it names is in ClamAV's table), not a global off switch. - Confirmed: opcode enum + `operand_counts` table; CALL/GEPN read their arg count inline; inline-constant operands marked by a `0x4N`/`0x50` lead byte. @@ -155,18 +158,17 @@ that trade-off** — a real reason to switch. constant globals (with component counting). **Decodes all 85** (1,257 globals) with no errors. -**Execution — live, with partial coverage.** `bytecode::exec` is a bounded -interpreter over the decoded IR: value array, integer -arithmetic/bitwise/compare/cast/select, `branch`/`jmp`/`ret`, host-API dispatch, -and the `(region, offset)` pointer addressing model — and it runs in the live -scan path (`Scanner::bytecode.scan`), trigger-gated per program, emitting -`Method::Bytecode` detections. The remaining work is **coverage, not gating**: -programs needing not-yet-implemented host APIs (`disasm_x86`, the PDF/JSON -object APIs, the inflate/lzma/bzip2 codecs) are skipped rather than executed, and -the disasm/codec-dependent detections that do run are not yet -detection-validated against `clamscan` (see `BYTECODE_VALIDATION.md`). - -Known limitation (0.0.1): functionality-level handling is unenforced for +**Execution: live.** `bytecode::exec` is a bounded interpreter over the decoded +IR: value array, integer arithmetic/bitwise/compare/cast/select, +`branch`/`jmp`/`ret`, host-API dispatch, and the `(region, offset)` pointer +addressing model. It runs in the live scan path (`Scanner::bytecode.scan`), +trigger-gated per program, emitting `Method::Bytecode` detections. A program +that calls a stubbed API runs and gets the stub's fail-safe value; a program +that runs out of its instruction budget makes the scan `LIMITS-EXCEEDED` unless +something else is found. Detections that depend on `disasm_x86` or a stub are +not yet validated against `clamscan` (see `BYTECODE_VALIDATION.md`). + +Known limitation: functionality-level handling is unenforced for bytecode. Programs observe `FLEVEL=167` via `engine_functionality_level` (`bytecode/runtime.rs`), while signature loading gates on `EXAV_FLEVEL=213` (`engine/parse.rs`); per-program `min/max_flevel` and `format_level` are parsed @@ -196,43 +198,12 @@ priority. And note the bigger picture — the **detection delta** from 85 mostly legacy programs is modest, so exav's headline win here is the **memory-safe, non-JIT sandbox** (removing the RCE class), more than the raw extra coverage. -## 7. Roadmap to 100% of the 85 programs - -The decisive tool throughout is a **self-validating oracle**: decoding is correct -iff every `B` record consumes exactly to its end *and* the per-function -instruction total equals the header's `numInsts`, across all 85 files. Each -phase is "done" only when that holds. - -- **Phase A — instruction decode (done).** Pin the exact `B`-record framing: - the `T` terminator layout, the trailing marker (the observed `@d E` tail), the - `GEP1`/`GEPZ` operand reads, and confirm the opcode field encoding. *Gate: - 100% of B records consume exactly; Σinsts == numInsts for all 85.* (~1–2 wk) -- **Phase B — types + SSA value model.** Decode the `T` type table - (struct/array/pointer/function) and the value-numbering so register sizes, - constants, and pointer types are correct. (~1–2 wk) -- **Phase C — lower to executable IR.** Map all ~50 opcodes to the VM IR with a - bounds-checked `(region, offset)` pointer model (covers `load`/`store`/`GEP`/ - `memcpy`/`select`/casts/branches/calls). (~2–3 wk) -- **Phase D — host APIs (→78%).** Implement the API ladder over the existing - file buffer + `pe.rs`: `read`/`seek`/`file_byteat`/`file_find(_limit)`, - `setvirusname`, `engine_functionality_level`, `debug_*`, `read_number`, - `memstr`, bounded `malloc`, `get_pe_section`/`pe_rawaddr`. This set already - covers **66/85 (78%)** of the live programs. (~1–2 wk) -- **Phase E — gated execution + differential testing.** Trigger lsig match → run - under budget + `catch_unwind` → `setvirusname` → detection; run only when a - program fully lowers and all its APIs are implemented (else safe no-op). - Differential-test verdicts against `clamscan` on the same inputs. (~1 wk) -- **Phase F — long tail (→100%).** The remaining ~19 need `disasm_x86` (an x86 - decoder — the single biggest API, ~4 programs), the PDF/JSON object APIs, the - `inflate`/`lzma`/`bzip2` codecs (reuse exav's unpack decoders), `jsnorm`, - `matchicon`, `extract_new`. Each is an isolated, on-demand addition. (~3–6 wk) - -**Cross-cutting:** fuzz the decoder/VM every phase; keep execution gated so -partial support can never false-positive or crash; the memory-safe, non-JIT, -strictly-bounded sandbox is invariant throughout. - -Phases A–E (~6–8 weeks) reach ~78% coverage with real, safe detections; the -`disasm_x86` long tail (Phase F) is the bulk of the remainder. +## 7. How decoding was validated + +The decisive tool was a **self-validating oracle**: decoding is correct iff +every `B` record consumes exactly to its end *and* the per-function instruction +total equals the header's `numInsts`, across all 85 files. That holds for the +whole live database, and the decoder and VM are fuzzed. ## Host APIs: 34 of 107 diff --git a/docs/FUZZING.md b/docs/FUZZING.md index 3657ee0a..09ba4b4e 100644 --- a/docs/FUZZING.md +++ b/docs/FUZZING.md @@ -155,7 +155,7 @@ For each artifact (`fuzz/artifacts//{crash,timeout,oom}-*`): before slicing, `saturating_*` arithmetic, `.min(len)` clamps on attacker-controlled offsets/sizes. A parser must tolerate any byte string. - **A third-party decoder** (e.g. `cab`, which panics instead of returning - `Err`) → contain it at the **`extract_each` `catch_unwind` boundary**, which + `Err`) → contain it at the **`walk` `catch_unwind` boundary**, which turns any decoder panic into a clean `LimitHit`. (We can't easily fix the dep in place; the boundary makes hostile input a non-event for *every* decoder uniformly.) @@ -179,7 +179,7 @@ Even with every known panic fixed, new ones can hide in deps. The layers: | layer | contains | mechanism | |-------|----------|-----------| | Root-cause fixes in our parsers | our own panics | bounds checks, saturating math | -| `extract_each` `catch_unwind` | third-party decoder panics | panic → `LimitHit` (never a silent Clean) | +| `walk` `catch_unwind` | third-party decoder panics | panic → `LimitHit` (never a silent Clean) | | Daemon prefork pool | OOM, stack overflow, hangs | per-job `RLIMIT_AS` / `RLIMIT_CPU` / `SIGALRM` → worker `_exit`, recycled | | Verdict taxonomy | "couldn't fully scan" | `LimitsExceeded`/`Unscannable` — a not-fully-scanned file is never reported Clean | @@ -194,7 +194,7 @@ boundary is a backstop, not an excuse to leave our own parsers panicky. | input | site | root cause | fix | |------------------|-----------------------------------|---------------------------------------------------------|-----| -| `MSCF` cabinet | `cab-0.6.0` `folder.rs:134` | dep indexes an empty vector on a crafted cabinet (panics instead of `Err`) | `catch_unwind` boundary in `extract_each` → `Unscannable` | +| `MSCF` cabinet | `cab-0.6.0` `folder.rs:134` | dep indexes an empty vector on a crafted cabinet (panics instead of `Err`) | `catch_unwind` boundary in `walk` → `Unscannable` | | `0o070707` cpio | `formats/cpio.rs` (bin/newc/odc) | `data_start` from attacker `namesize`, unclamped → `data[start..end]` with start > len | clamp `data_start` to `data.len()` (all 3 variants) | | ISO9660 | `formats/iso.rs:54/57` | dir record's self-declared `rec_len < 33`, then `rec[2..32]` indexed | require `rec_len >= 33` before slicing | | UPX (NRV2B/D/E) | `formats/upx.rs` (4 gamma loops) | corrupt bitstream doubles the gamma value until `usize` overflow (and downstream `(m-3)*256`) | cap gamma at `NRV_GAMMA_MAX` (`u32::MAX`) → `Unscannable` | diff --git a/docs/PARTIAL.md b/docs/PARTIAL.md index 8dfff39b..e051bc58 100644 --- a/docs/PARTIAL.md +++ b/docs/PARTIAL.md @@ -1,8 +1,6 @@ # PARTIAL: the third verdict, and how it reaches every protocol -**Status: implemented.** Supersedes the decided parts of -[`VERDICT_PROTOCOL_PLAN.md`](VERDICT_PROTOCOL_PLAN.md), which asked the question -this answers. +**Status: implemented.** Every claim about ClamAV was measured against a local **ClamAV 1.4.3**, and every claim about exav against the built binary. Commands at the end. @@ -11,16 +9,16 @@ claim about exav against the built binary. Commands at the end. ## 1. One vocabulary -**Status** — four words, each naming the exit code it produces: +**Status**: four words, each naming the exit code it produces: | Status | Exit | Meaning | |---|---|---| | `OK` | 0 | fully scanned, nothing matched | | `FOUND` | 1 | a signature matched | -| `ERROR` | 2 | exav could not do its job — unreadable path, database that would not load | +| `ERROR` | 2 | exav could not do its job: unreadable path, database that would not load | | `PARTIAL` | 3 | exav worked, and something could not be fully examined | -**Category** — only under `PARTIAL`: +**Category**, only under `PARTIAL`: | Category | Raised when | |---|---| @@ -28,7 +26,7 @@ claim about exav against the built binary. Commands at the end. | `UNSCANNABLE` | the container could not be decoded | | `PASSWORD-PROTECTED` | the content is encrypted and no configured password opened it | -**Reason** — free text. +**Reason**: free text. One line grammar, everywhere: **`path: [reason ][CATEGORY ]STATUS`**, status last. @@ -41,7 +39,7 @@ path: Can't open file ERROR ### Why `PARTIAL` -Work happened in all three cases and stopped short of the end — a real prefix was +Work happened in all three cases and stopped short of the end: a real prefix was scanned, or the container was parsed and only its contents were out of reach. `SKIPPED` was rejected because `--exclude` already skips files at **exit 0**, so one word would have had two meanings with opposite codes; and because it claims @@ -58,7 +56,7 @@ exactly what it means in ClamAV. ### Precedence -`1` > `2` > `3` > `0`. A detection outranks everything — finding malware is +`1` > `2` > `3` > `0`. A detection outranks everything: finding malware is conclusive, and a limit hit elsewhere does not make the match less true. An error outranks a partial, because it casts doubt on the whole run where a partial is a fact about one object. @@ -106,7 +104,7 @@ as every value in that preset does. **Refused with `--connect`.** The policy belongs to whatever scans. A client only sees the reply the daemon already decided, so the flag would have parsed, looked -in force, and done nothing — and it cannot be made to work, since once the daemon +in force, and done nothing. It cannot be made to work, since once the daemon has reported `OK` for something it folded, the fact is gone from the wire. --- @@ -130,10 +128,10 @@ has reported `OK` for something it folded, the fact is gone from the wire. {"category":"LIMITS-EXCEEDED","file":"big.bin","reason":"file size 200000 exceeds max-input-bytes 1024…","status":"PARTIAL"} ``` -`category` is present only under `PARTIAL` — the other statuses have nothing to +`category` is present only under `PARTIAL`: the other statuses have nothing to sub-classify. Key order is not part of the contract; keys serialise sorted. -### clamd wire — `SCAN` / `CONTSCAN` / `MULTISCAN` / `INSTREAM` / `FILDES` / `SCANURL` +### clamd wire: `SCAN` / `CONTSCAN` / `MULTISCAN` / `INSTREAM` / `FILDES` / `SCANURL` ``` path: file size 200000 exceeds max-input-bytes 1024 LIMITS-EXCEEDED ERROR @@ -148,7 +146,7 @@ Same grammar as stdout; **only the status word differs**, and it has to. See §4 ``` The same three names the CLI JSON uses. This replaced `verdict`/`tag`/`message`, -whose `"verdict":"unscannable"` collided with the *category* of that name — a +whose `"verdict":"unscannable"` collided with the *category* of that name: a `{"verdict":"unscannable","tag":"PASSWORD-PROTECTED"}` read as a contradiction. ### ICAP @@ -160,7 +158,7 @@ whose `"verdict":"unscannable"` collided with the *category* of that name — a | `ok` | `204` (or `200` echoing the message without `Allow: 204`) | the `X-Exav-*` trio, **never** `X-Infection-Found` | `X-Exav-Status` is `PARTIAL` even under `--partial-as error`. ICAP has no exit -code — the only thing those two differ in — and answering `ERROR` would tell a +code (the only thing those two differ in), and answering `ERROR` would tell a proxy the *scanner* failed. Squid counts service failures and eventually bypasses the service, so relabelling an encrypted archive could take the scanner out of rotation and start failing open. @@ -168,7 +166,7 @@ rotation and start failing open. `X-Infection-Found` on a *block* defaults on: a large class of ICAP client greps that header alone and reads a `200` without it as a pass. `--icap-infection-header detections` restricts it to database hits, at that cost. -On a *pass* it is always withheld — the object is being delivered, so claiming an +On a *pass* it is always withheld: the object is being delivered, so claiming an infection would be false and would make those clients block it anyway. ### Library (`exav-core`) @@ -188,7 +186,7 @@ reply ends in PARTIAL -> clamdscan exit 0, prints "f.txt: OK" ``` **It rewrites an unknown status to `OK`.** No `FOUND`, no `ERROR`, therefore -clean — it does not even echo the text. +clean. It does not even echo the text. So `PARTIAL` on this wire would make every existing clamd client report a file exav could not scan as clean: the silent-clean failure exav exists to prevent, @@ -197,11 +195,18 @@ protocol's vocabulary is `OK` / `FOUND` / `ERROR`, and `ERROR` is the only word it that fails closed. The category is still in the text for anything that reads further. +The cost is known. In clamd, `ERROR` means the scanner failed (a vanished file, a +dead socket), and clients act on that: `clamdscan` exits 2, which wrappers read +as "the scanner is broken", and a mail gateway typically temp-fails the message, +so a password-protected ZIP is retried rather than quarantined. A client that +needs to tell the categories apart uses `EXINSTREAM`, or `--partial-as found` +for names it already handles. + --- ## 5. ClamAV compatibility -ClamAV **does** report these — as detections, opt-in: +ClamAV **does** report these, as detections, opt-in: | Situation | ClamAV default | With its alert flag | |---|---|---| @@ -227,11 +232,59 @@ That last part was a real gap: the top-level size check produced the one limit ClamAV *does* name. It now passes the limit kind as a type into the engine's own lookup, as every other budget already did. -ClamAV has **no structured category anywhere** — `--gen-json` emits file metadata +ClamAV has **no structured category anywhere**: `--gen-json` emits file metadata only (`Magic`, `RootFileType`, `FileName`, `FileType`, `FileSize`, `FileMD5`), no verdict. Its only categorisation is the dotted signature name. exav keeps its own flat categories for `PARTIAL`, and uses ClamAV's dotted namespace under -`--partial-as found`, where the object genuinely is being reported as a detection. +`--partial-as found`, where the object is being reported as a detection. + +--- + +## 6. Why a third verdict: what ClamAV's `OK` covers + +The question that led here: run with every heuristic and `--allmatch`, does +ClamAV miss nothing silently? It does, and no flag changes that. + +Method: extract each container's members with exav, hash every member into an +`.hdb`, then scan the **untouched** container with `clamscan` and every alert +flag it has. If clamscan reports `OK`, it never reached the member. + +``` +--allmatch --heuristic-alerts=yes --alert-broken=yes --alert-broken-media=yes +--alert-encrypted=yes --alert-encrypted-archive=yes --alert-encrypted-doc=yes +--alert-macros=yes --alert-exceeds-max=yes --alert-phishing-ssl=yes +--alert-phishing-cloak=yes --alert-partition-intersection=yes +``` + +Every row is a plain `OK` from clamscan on a file whose payload exav reads: + +| Container | Members exav reaches | ClamAV reaches | +|---|---|---| +| LZW `.Z` (10 fixtures, 12- and 16-bit) | 1 each | **0** | +| WIM: LZX, LZMS, XPRESS, uncompressed (5 images) | 1-3 each | **0** | +| UDF-only ISO (no ISO 9660 descriptor) | 3 | **0** | +| VHDX, dynamic | 1 | **0** | +| QCOW2 with deflate clusters (2 fixtures) | 1 each | **0** | +| VMDK sparse, and streamOptimized (the shape inside an OVA) | 1 each | **0** | +| EGG with LZMA members | 2 | **0** | +| EGG solid archive | 1 | **0** | + +ClamAV does reach EGG store/deflate members, ALZ, ISO 9660 (including the ISO +side of a bridge image), and anything behind gzip. + +- **The EGG rows are inside a format ClamAV supports.** Its EGG code is on by + default and still returns `OK` when the members are LZMA or solid. Supporting + a format is not reaching its members. +- **No flag changes any of it.** There is no alert for "a container I have no + code for". ClamAV's `--help` says as much for its limits: `--max-filesize`, + *"Files larger than this will be skipped and assumed clean"*, and the same + for `--max-scantime`. `--alert-exceeds-max` covers file size, scan size and + recursion; there is no alert for scan time. + +With the documented behaviour past 2 GB (read, scanned as zero bytes, `OK`), +ClamAV's `OK` means "nothing matched in what I examined". exav's `OK` means "I +examined it". No combination of flags turns the first into the second, which is +why exav needs a status ClamAV does not have. --- @@ -255,4 +308,12 @@ printf 'nSCAN %s\n' "$PWD/big.bin" | nc 127.0.0.1 3310 # ICAP codes and headers: crates/exav/tests/icap_server.rs asserts them cargo test -p exav --test icap_server + +# Section 6: exav extracts, members are hashed into an .hdb, clamscan +# rescans the untouched container with every alert flag. +scripts/clamav-reach.sh crates/exav-unpack/tests/fixtures/{lzw,wim,egg,alz}/* ``` + +The `.gz`-wrapped fixtures under `diskimage/` and `udf/` must be gunzipped +first: otherwise the measurement tests gzip, not the inner format, and reports +"reached all" for images ClamAV never opens. diff --git a/docs/STREAMING_FORMATS.md b/docs/STREAMING_FORMATS.md index b5433e27..c1afcfe6 100644 --- a/docs/STREAMING_FORMATS.md +++ b/docs/STREAMING_FORMATS.md @@ -1,23 +1,19 @@ -# Streaming the last three container formats +# Streaming the last container formats -exav optimises for memory, not bandwidth. A streamable format is walked member by -member straight off the seekable source, so peak memory is one member's window -rather than the whole decoded object. Most containers already work that way (see -`is_streamable`), including tar, zip, 7z and cab. Three do not, for different -reasons, and only two of them are worth changing. +exav optimises for memory, not bandwidth. `walk` hands a format's members over +straight off the seekable source, each decoded as it is read, so peak memory is +one member's window rather than the whole decoded object. Most containers +already work that way (every arm of `dispatch` in `stream.rs` but the last), +including tar, zip, 7z, cab, lz4, `.Z` and DMG; the rest are read whole +(`read_whole`). Two of those matter, for different reasons, and only one of them +is worth changing. -**DMG: the tractable one, and the next concrete step.** `decompress_udif` -materialises the entire virtual disk into one `Vec` because the HFS+/APFS -readers need random access over it, so a 4 GB image costs 4 GB of resident -memory (bounded by the peak-buffer limit, which means large images are refused -rather than scanned). The reader is already the right shape: `DmgReader` is -position-based, mapping an offset to a BLKX run and decompressing that run. Two -pieces are missing. First a decompressed-run cache: `read` currently -re-decompresses the containing run on every call, which is fine for a linear -pass and unusable under the seek storms a filesystem crate generates. Then -`impl Seek`, after which the reader can be handed to the filesystem crates -directly instead of a materialised image. Roughly 150 lines, and it converts a -whole-image allocation into one cached run. +**DMG: done.** The HFS+/APFS readers get the disk as a `Read + Seek` over +`DmgReader`, which decompresses a BLKX run when the filesystem first reaches it +and keeps the last 32 MiB of decoded runs. The virtual disk is never held +whole. Each file is still returned whole by the filesystem crates, so a file is +bounded by the peak-buffer limit. An encrypted image is decrypted whole, under +the same limit. **RAR: real work, deliberately not rushed.** `rar3_unpack.rs` and `rar5_unpack.rs` are vendored decoders that build the complete output `Vec`. diff --git a/docs/VERDICT_PROTOCOL_PLAN.md b/docs/VERDICT_PROTOCOL_PLAN.md deleted file mode 100644 index 858d0931..00000000 --- a/docs/VERDICT_PROTOCOL_PLAN.md +++ /dev/null @@ -1,567 +0,0 @@ -# Verdicts on the wire: the investigation - -How exav should report *"I could not look at this"* — to a human, to a clamd -client, and to a new integration — and what that costs in ClamAV compatibility. - -Every claim here that could have been asserted was measured instead; the commands -are given. That evidence is why this file is kept. - -> **This is the question, not the answer.** [`PARTIAL.md`](PARTIAL.md) is the -> spec: read it for what exav does. Sections 2 and 4 below describe a design that -> was weighed and not taken — the `NotScanned` category and the `--compat=` -> flag do not exist. Section 1 (what ClamAV can and cannot express) and section 3 -> (what real clients do with `ERROR`) are the measurements that decided it, and -> they still hold. - ---- - -## 1. The question that motivates it - -> If ClamAV is run with full heuristics and `--all-matches`, does it actually -> have the same capabilities as exav — no silent misses? - -**No.** Not close, and not for want of flags. - -Method (the rigorous one, not a signature-database comparison): extract each -container's members with exav, hash every member into an `.hdb`, then scan the -**untouched** container with `clamscan` and every alert flag it has. If clamscan -reports `OK`, it never reached the member. Flags used: - -``` ---allmatch --heuristic-alerts=yes --alert-broken=yes --alert-broken-media=yes ---alert-encrypted=yes --alert-encrypted-archive=yes --alert-encrypted-doc=yes ---alert-macros=yes --alert-exceeds-max=yes --alert-phishing-ssl=yes ---alert-phishing-cloak=yes --alert-partition-intersection=yes -``` - -Result — every row is a plain `OK` from clamscan on a file whose payload exav -reads: - -| Container | Members exav reaches | ClamAV reaches | -|---|---|---| -| LZW `.Z` (10 fixtures, 12- and 16-bit) | 1 each | **0** | -| WIM — LZX, LZMS, XPRESS, uncompressed (5 images) | 1–3 each | **0** | -| UDF-only ISO (no ISO 9660 descriptor) | 3 | **0** | -| VHDX, dynamic | 1 | **0** | -| QCOW2 with deflate clusters (2 fixtures) | 1 each | **0** | -| VMDK sparse, and streamOptimized (the shape inside an OVA) | 1 each | **0** | -| **EGG with LZMA members** | 2 | **0** | -| **EGG solid archive** | 1 | **0** | - -For contrast, ClamAV *does* reach: EGG store/deflate members, ALZ, ISO 9660 -(including the ISO side of a bridge image), and anything behind gzip. - -Two things this shows that a format list would not: - -- **The last two rows are inside a format ClamAV claims.** Its EGG submodule is - on by default, and it still returns `OK` on an EGG whose members are LZMA or - solid. Format-level support is not member-level reach, so "does it support - EGG?" is the wrong question to audit with. -- **No flag changes any of it.** These are not heuristics that are off by - default; there is no name for "a container I have no code for". ClamAV's own - `--help` says the quiet part for the limit cases — `--max-filesize`: *"Files - larger than this will be skipped and **assumed clean**"*, `--max-scantime`: - same wording. `--alert-exceeds-max` covers filesize/scansize/recursion; there - is no `--alert-exceeds-scantime`. - -Add the documented >2 GB behaviour (read, scanned as zero bytes, reported `OK`) -and the shape is clear: **ClamAV's `OK` means "nothing matched in what I -examined". exav's `OK` is intended to mean "I examined it".** Those are different -claims, and no combination of flags converts the first into the second. - -This is the whole reason the vocabulary problem below exists. If ClamAV had a -verdict for "not examined", exav would just use it. - ---- - -## 2. Where exav is today - -**Verdicts** (`exav_core::Verdict`), collapsing to three categories: - -| Verdict | Category | Exit | -|---|---|---| -| `Clean` | Clean | 0 | -| `Infected { signature, offset, method }` | Infected | 1 | -| `LimitsExceeded { reason }` | NotScanned | 2 | -| `Unscannable { reason }` | NotScanned | 2 | -| `PasswordProtected { reason }` | NotScanned | 2 | - -The three-way split inside `NotScanned` is real and worth keeping: - -- `LimitsExceeded` — **the operator can fix this.** Raise a limit, rescan. -- `PasswordProtected` — **the user can fix this.** Supply a password, rescan. -- `Unscannable` — **nobody can fix this today.** exav has no decoder. - -Each points at a different person, which is the only justification a verdict -distinction ever needs. - -**On the clamd wire** (`crates/exav/src/daemon.rs`), all three become: - -``` -: () ERROR -``` - -and `EXINSTREAM`, exav's own command, returns one line of JSON: - -```json -{"v":1,"verdict":"unscannable","tag":"LIMITS-EXCEEDED","message":"…"} -``` - ---- - -## 3. The problem with `ERROR` - -The clamd protocol has exactly three reply shapes: `OK`, ` FOUND`, -` ERROR`. exav maps "not scanned" to `ERROR` because it is the only reply -that is not a lie. - -But `ERROR` already means something to every existing client, and it is not -this. In clamd, `ERROR` is an *infrastructure* failure — file vanished, socket -died, out of memory. Clients act accordingly: - -- **Mail gateways** (amavisd, MailScanner, rspamd) typically **temp-fail** the - message: a 4xx and a retry. A password-protected ZIP arriving as `ERROR` - therefore gets retried forever and never delivered or quarantined — the worst - outcome for a condition where the right answer is "quarantine and tell the - user to send the password". -- **Storage/backup scanners** log and move on, so the finding is invisible. -- **`clamdscan`** prints it and sets exit 2, which most wrappers treat as "the - scanner is broken", not "this file needs attention". - -So the honest reply produces, in practice, the same operational outcome as -silence — sometimes worse. **The invariant is preserved on the wire and lost in -the client.** That is the actual bug, and it is a UX bug rather than a protocol -one. - -Meanwhile ClamAV expresses the *same* conditions as ordinary detections: -`Heuristics.Encrypted.Zip`, `Heuristics.Limits.Exceeded`, -`Heuristics.Broken.Executable` arrive as `FOUND`, and every client already knows -what to do with `FOUND`. - ---- - -## 4. The plan - -### 4.1 Keep the model, change only the projection - -The internal model (three NotScanned verdicts, each with a reason) is right and -should not move. What is wrong is that there is one lossy projection of it onto -the wire, chosen for us. Make the projection a policy. - -**`--compat=`**, on both the CLI and the daemon: - -| Mode | `OK`/`FOUND`/`ERROR` mapping | For | -|---|---|---| -| `clamav` | Not-scanned → `Heuristics.* FOUND`, gated by the same `--alert-*` flags ClamAV uses, **including staying silent when the flag is off** | Drop-in replacement behind existing clients | -| `strict` *(recommended default)* | Not-scanned → `Heuristics.* FOUND` always, no flag needed | Anyone who wants the invariant to survive contact with a real client | -| `exav` | Current behaviour: ` () ERROR` | Operators who have already built on it | - -`strict` differs from `clamav` in exactly one way — it does not let a flag turn -a known gap into an `OK`. That is the difference the whole project is about, and -making it one axis rather than a rewrite is the point. - -### 4.2 The names - -Reuse ClamAV's name wherever the condition is the same, so existing allowlists, -dashboards and alert rules keep working: - -| exav condition | Name emitted | -|---|---| -| Encrypted archive member | `Heuristics.Encrypted.Zip` / `.7Zip` / `.RAR` / `.EGG` … | -| Encrypted document | `Heuristics.Encrypted.PDF` / `.Doc` | -| Any limit hit | `Heuristics.Limits.Exceeded.` | -| Broken PE/ELF | `Heuristics.Broken.Executable` | -| Broken media | `Heuristics.Broken.Media.` | -| Overlapping partitions | `Heuristics.MBRPartitionnIntersect` (the doubled `n` is upstream's, reproduced deliberately) | - -For the categories ClamAV has **no name for** — §1's eight rows — exav needs its -own, and they must not collide with anything Talos may ship: - -``` -Heuristics.Exav.Unopened. container recognised, no decoder -Heuristics.Exav.Unopened.. member codec not implemented -``` - -e.g. `Heuristics.Exav.Unopened.Ace`, `Heuristics.Exav.Unopened.Egg.Azo`. The -`Exav.` infix is the namespace; `Unopened` states the fact without claiming -maliciousness, which matters because these fire on entirely benign files. - -**Open question for you:** whether `Heuristics.Exav.*` should be on in `clamav` -mode. Emitting a name ClamAV never emits is a compatibility divergence; *not* -emitting it reintroduces the exact silent miss the project exists to remove. My -recommendation is on-by-default with `--no-alert-unopened` to suppress, because -a name a client does not recognise still surfaces, whereas silence does not. - -### 4.3 Structured detail is the real answer — via `EXINSTREAM` - -Squeezing "why" into a signature name is a workaround for a protocol with three -reply shapes. New integrations should not have to. - -`EXINSTREAM` already returns JSON and already carries `tag` + `message`. Extend -it rather than the string protocol — this is where the earlier -"shouldn't we return structured metadata?" question lands: - -```json -{"v":1,"verdict":"unscannable", - "tag":"UNSCANNABLE", - "reasons":[ - {"format":"Egg","codec":"AZO","member":"a.zip/inner.egg/x.bin", - "why":"no-decoder","recoverable":false}], - "examined_bytes":12345,"total_bytes":99999} -``` - -Three properties worth having, none expressible in a name: - -- **`member`** — the nesting path, so an operator can find the thing. -- **`recoverable`** — machine-readable version of the "who can fix this" split - in §2. A gateway can route `recoverable:true` to a user-facing bounce and - `false` to quarantine. -- **`examined_bytes` / `total_bytes`** — turns "not fully scanned" into a - proportion. A 4 GiB archive with one 12-byte undecodable member is not the - same risk as one where nothing was read, and today both report identically. - -Add `--format=json` to the CLI with the same schema, so the daemon and the CLI -answer the same question the same way. - -### 4.4 The invariant must not be defeasible - -Whatever the wire says, the **process exit code and the scan summary keep -telling the truth**. In `clamav` mode a suppressed heuristic may produce `OK` on -the socket, but: - -- the run still exits `2` if anything went unexamined; -- the summary still prints the count and the reasons; -- `--fail-on-unscanned=no` is the single explicit opt-out, and it is loud in - `--help`. - -This is what keeps `clamav` mode from being a hole: the compatibility surface is -the reply line, not the truth. - -### 4.5 Sequencing - -1. `Heuristics.*` emission behind `--compat`, reusing ClamAV's names — no new - vocabulary, immediate interop win. Diff-testable against clamscan directly. -2. `Heuristics.Exav.Unopened.*` for the eight categories in §1, with the - `docs/` gap list generated from the same table so they cannot drift. -3. `EXINSTREAM` structured reasons + CLI `--format=json`. -4. Revisit `--compat` default once (1)–(3) have run against a real corpus. - -Steps 1 and 2 are what make exav's advantage *visible to an existing client*. -Today that advantage is real — §1 measures it — and largely invisible, because -`ERROR` is the one thing clients are trained to ignore. - ---- - -## 5. Multi-file archives: the one-buffer assumption - -Everything above concerns *what a verdict says*. This section is about a case -where exav cannot reach the content at all, and the cause is an API shape rather -than a missing decoder. - -### 5.0 The governing principle: a scan unit is a *set*, not a file - -Everything in this section is one gap wearing several hats, and it is worth -stating generally before the special cases. - -**At every recursion level, the members produced by the level above are a -cohort — and exav's API hands them over one at a time.** `Sink` receives one -`Entry` with no view of its siblings, so no extractor and no recursion step can -observe a relationship *between* files: - -| Relationship | Invisible today because | -|---|---| -| Multi-volume set (`x.part1.rar` …) | the member's stream continues in a sibling | -| Multi-cabinet CAB | the folder continues in the cabinet the header names | -| `x.exe` + `x.dat` sidecar | the relationship is in code, but both files are right there | -| Split archive inside a ZIP | all the parts arrive in one pass and are then separated | - -The same shape recurs at the outside edge: a directory (`CONTSCAN`), a -client-supplied manifest (`EXINSTREAM`), and a container's member list are all -**a set of named byte sources**. Treating them as three problems is what made -this look like three features. - -**The constraint that shapes the fix.** `extract_each` yields one member at a -time on purpose: it is why the working set stays flat regardless of file size, -which is the property exav has over an engine that reads a >2 GB file as zero -bytes. Passing every member's *bytes* would trade that away — a 10 GB archive -with 10,000 members would be resident at once. - -**But do not enumerate first.** An "list the cohort, then decide" pass is the -obvious design and it is wrong here: - -- `tar` has no index — listing it means walking the whole file. -- A stream (gzip, `INSTREAM`) cannot be listed at all without decompressing. -- It forces a complete pass *before any scanning starts*, delaying detection — - and [`crate::stream_members`] exists precisely so a large container never needs - one. -- A container declaring ten million members makes the metadata the memory - problem. - -That would trade the streaming property away to serve a case that almost never -arises. - -**React instead of enumerate.** The one-at-a-time visitor already sees every -member's *name* as it arrives. The moment a name parses as a volume member -(`volume::parse`), a set may exist — and only then is anything gathered. The cost -on a container with no volume members is one string match per name, and the -streaming default is untouched. - -Ordering is the one wrinkle: `part2` can arrive before `part1`. A seekable source -seeks back. A pure stream cannot, so members whose names parse as volumes are -held (bounded by the existing `max_members` / `max_buffer`) until the set completes -or the container ends. A set that never completes reports exactly as it does -today. - -Peak memory therefore becomes one *set* rather than one *file*, and only where a -set exists — never a whole member list. - -**Where the change lives.** Not in the sixty-odd format modules; their `Sink` -signature is unchanged. The layer sits *above* `extract_each` — enumerate, group, -dispatch — in exav-core's recursion, plus a metadata-enumeration entry point for -the buffered path. The seekable `Archive` type already has the primitive. - -This also supplies the in-memory VFS of §5.5 for free: that VFS *is* the cohort -at one recursion level. The sidecar case then falls out of the same design rather -than needing one of its own. - -### 5.1 The gap - -A multi-volume archive is one logical archive spread over several files — -`x.part1.rar`/`x.part2.rar`…, `x.rar`/`x.r00`/`x.r01`…, `x.7z.001`, -`x.zip`/`x.z01`, `x.a01`, and multi-cabinet CAB. A member's compressed stream -runs off the end of one file and continues in the next, so **no single file -decodes it**. - -Every entry point exav has takes one buffer: - -| Entry point | Shape | -|---|---| -| `exav_unpack::extract(fmt, data, budget)` | one `&[u8]` | -| `exav_unpack::extract_each(fmt, data, …)` | one `&[u8]` | -| `exav_unpack::stream_members(fmt, source, …)` | one `Read + Seek` | -| `Archive::open(reader)` | one reader | -| clamd `INSTREAM` / `EXINSTREAM` | one chunked stream | -| clamd `SCAN` | one path | -| `CONTSCAN` / `MULTISCAN` | a directory — but **each file scanned independently** | - -So the capability was absent at every layer, and `CONTSCAN` is the interesting -one: it already receives a whole directory and threw the grouping away. - -Since resolved for byte-split sets — see §5.4 for what each surface does now and -what is still open. - -### 5.2 Measured - -`unar` reports the fixture set as `RAR (4 volumes)` containing `text.bin`. -clamscan, with `--allmatch` and the alert flags, on each part in turn: - -``` -c4.part1.rar: OK c4.part2.rar: OK c4.part3.rar: OK c4.part4.rar: OK -``` - -Four `OK`s. A payload spanning a volume set is invisible to it, and there is no -flag that changes this — the same shape as §1. - -**exav does not share the silent part.** `formats/rar.rs` reads the -`LHD_SPLIT_BEFORE`/`LHD_SPLIT_AFTER` flags and emits *both* the readable prefix -and an `Entry::unsupported` saying `"RAR member continues in another volume; -only the part in this volume was scanned"`. So exav reports the gap plainly and -cannot close it. That is the correct failure mode, and it is still -a capability gap against 7-Zip, WinRAR and The Unarchiver — which is the bar -[the parity principle](/concepts/archive-extraction/#the-parity-principle) sets, -because the attacker picks the format the *victim* can open. - -### 5.3 Design - -Prior art exists and should be the starting point: an earlier worktree holds a -248-line `volumes.rs` plus a RAR volume-set implementation, unmerged. Its -security model is the part worth keeping verbatim. - -**`exav-unpack` gains a trait, not a filesystem dependency.** The crate is -`#![forbid(unsafe_code)]`, WASM-targetable, and has no business opening files: - -```rust -/// Supplies the sibling volumes of a multi-volume archive. -pub trait VolumeSet { - /// Volume `n` (0 is the scanned file), or `None` when the set ends. - fn volume(&mut self, n: usize) -> Option<&[u8]>; - fn len(&self) -> usize; -} -``` - -It is a **byte supplier, not a callback into the scanner**, and that distinction -is what makes it safe to add. exav-core is already inside a `Sink` callback from -exav-unpack when a member is being decoded; a trait that re-entered the scan -would close a cycle. A dumb accessor does not, so there is no reentrancy to -reason about and no borrow tangle. - -Only **format-aware** volumes need it. For a byte-split set the logical stream is -a plain concatenation, so a `Read + Seek` that stitches the parts is enough — and -[`crate::stream_members`] already takes exactly that, meaning **no decoder -changes at all**. RAR needs the trait because a member's data continues *past the -next volume's headers*, so the join is not concatenation and the skipping needs -format knowledge. - -`extract_each_multi(fmt, set, budget, visit)` joins a split member's stream -across volumes; the single-buffer entry points stay as they are and keep -working. In a build with no volume source, behaviour is exactly today's — report -the split member. - -**Resolution belongs to the caller, and only ever to a path.** `exav-core` -implements the trait against the filesystem under rules that are the whole -security argument: - -- **Same directory only.** Sibling names are *generated* from the scanned file's - own name by a parsed pattern — never read from archive content. -- **Strict naming.** Only the schemes the writers actually produce, preserving - the scanned name's digit width. -- **`symlink_metadata`, regular files only.** A symlink named like the next - volume is refused, not followed. -- **Capped** at `MAX_VOLUMES` (100) and the per-volume peak-buffer limit. - -The one format that inverts this is **CAB**, whose header names the next cabinet -(`szCabinetNext`) *inside the file*. That string is attacker-controlled and must -be treated as a name to **match against a directory listing**, never as a path -to open — otherwise a cabinet chooses what exav reads. Worth stating explicitly -because it is the obvious implementation and it is a directory-traversal bug. - -### 5.4 Protocol — *implemented for byte-split sets* - -Three surfaces, and the ordering mattered: - -1. **`CONTSCAN`/`MULTISCAN` grouping — no protocol change, biggest win.** The - command already gets a directory. **Done.** Sets are grouped per directory and - the rejoined archive is scanned once; the verdict is attributed to *every* - part, so the reply stays exactly one line per file and existing clients get - multi-volume support with no changes at all. - - The alternative — reporting under a synthetic set name — was rejected: clamd - emits one line per file and a client counting lines against files would break. - A part of an infected archive is not a clean file anyway, and it is the file - the operator has to act on. -2. **`SCAN `** — resolve siblings from the path, exactly as §5.3. Not yet - done for the scanner; `exav-unpack list|extract` already does it. -3. **`INSTREAM` cannot be fixed, and should not be.** Its framing is - `…`: one stream, no names, no boundaries. Retrofitting a - manifest would break every existing client's framing. **`EXINSTREAM MULTI` - was added instead** — a separate verb, so bare `EXINSTREAM` is untouched: - - ``` - EXINSTREAM MULTI - … per file - ends the request - ``` - - The JSON manifest this section originally proposed was dropped. It would have - put a parser in front of the framing for no gain: the name belongs *with* its - chunk sequence, which the length-prefix above expresses directly, and a - malformed manifest becomes a framing desync rather than a bad request. - - Names here are **labels for reporting and for volume ordering only** — they - are never opened, so the traversal question does not arise. - - The reply is one line: `{"v":1,"files":[{"name":N,…verdict…},…]}`, one entry - per file, with `"set"` present when the verdict came from a rejoined archive - rather than from the file itself. - -**What is not covered.** Only *byte-split* sets (`.001`, `.002`) rejoin, because -concatenation is the whole of their format. Format-aware volumes (RAR `.partN`, -ZIP `.zNN`) pass through unchanged and are still reported accurately per §5.2 — -each carries its own headers and a member's data resumes *past* the next -volume's header, so their join belongs inside the format's decoder. - -**The join cannot happen mid-stream.** Nothing in a set's names records how many -parts it has, so `.001`+`.002` looks contiguous even when `.003` follows. -Joining on arrival emits a truncated prefix that *still parses as the archive* -and is then scanned as if it were whole — a silent partial scan wearing a valid -archive's costume. `volume::Collector` therefore always answers `Held` and -resolves only in `finish()`, when no further member can arrive. The cost is that -a split set is not scanned until its container ends; it is unavoidable. - -### 5.5 The sidecar case, and why the emulator is not the answer - -A related shape: a dropper `x.exe` that reads `x.dat` beside it and decrypts the -payload at runtime. Neither file is detectable alone — the executable carries no -payload, and the `.dat` is indistinguishable from noise. It is the same -"one logical object, several files" problem as a volume set, but the relationship -lives in *code* rather than in a naming convention. - -**Would a PE-packer emulator reach the sidecar?** No, and it should not be -extended to. - -exav's emulator (`exav-pe-emu`) runs an unpacking *stub*: self-contained code -that inflates a section of the file it is already in. Such a stub needs a -handful of APIs — allocate, protect, and enough `LoadLibrary`/`GetProcAddress` -to rebuild an import table — and **no file I/O at all**. - -A dropper is a different program by orders of magnitude: `GetModuleFileNameW`, -`CreateFileW`, `ReadFile`, then arbitrary decryption, and commonly registry and -network before it does anything interesting. Emulating that is a sandbox, not a -stub emulator, and every scanner in the comparison set has the same blind spot. - -**The obvious safe shape is an in-memory VFS over exactly the request's files**, -resolved by basename, with no syscall reaching a real filesystem — and that -disposes of most of the security worry rather than merely bounding it. The guest -cannot reach anything the caller did not already hand the scanner; a read -"succeeds" only into bytes exav is holding anyway. There is no escalation there, -just the same bytes by another route. It is the same discipline as §5.3: a name -from untrusted input is matched against a set the caller assembled, never turned -into a path. - -exav is also an unusually good host for this. `#![forbid(unsafe_code)]`, no JIT, -budgets on every axis, and `catch_unwind` around each decoder mean an emulator -bug is a contained panic rather than the RCE class that has repeatedly hit C -engines with JITs. The historical reason to fear scanner emulators does not -transfer. - -So the objection is **not safety, it is API breadth**. A packer stub needs about -a dozen calls; a dropper reaches for file, registry and network APIs, and dies at -the first one that is missing. That is an engineering-cost question with a known -mitigation — emulation need not run to completion. It runs until the program -*produces bytes*, and those bytes get scanned; the same checkpoint that makes -stub unpacking work. Partial execution with a "what did it write" trap is the -standard technique and degrades visibly: what it reaches is scanned, and what it -does not is reported. - -The prerequisite is the same either way: the VFS can only contain the request's -files if a request can *carry* several files, which is what §5.4 builds. The -emulator stays a separate decision on top of that, and a defensible one. - -### 5.6 What it does not change - -The never-silent invariant is unaffected either way: an unresolvable volume is -reported now and would still be reported then. This is purely about turning a -reported gap into read content — worth doing because a split archive is a -completely ordinary way to move a large payload, and today every scanner in the -comparison set except the extractors themselves misses it. - ---- - -## 6. What this does not solve - -- A client that ignores `FOUND` for `Heuristics.*` names (some sites allowlist - the whole prefix to cut noise) is back to silence. Nothing exav emits can fix - a site that filters it out; the `--format=json` path is the answer for anyone - who cares. -- `Heuristics.Exav.*` names are not in anyone's threat feed, so they will look - like false positives until documented. The docs page must lead with "these are - coverage reports, not detections". -- None of this narrows the gap itself. The eight rows in §1 shrink by writing - decoders — which is what tasks #24 (AZO) and the remaining gap list are for. - This plan only ensures the gap is *legible* while it exists. - ---- - -## Reproducing the measurements - -```sh -cargo build --release -p exav && cargo build -p exav-unpack -# For each container: exav extracts, members are hashed into an .hdb, -# clamscan rescans the untouched container with every alert flag. -scripts/clamav-reach.sh crates/exav-unpack/tests/fixtures/{lzw,wim,egg,alz}/* -``` - -The `.gz`-wrapped fixtures under `diskimage/` and `udf/` must be gunzipped -first — otherwise the measurement tests gzip, not the inner format, and reports -"reached all" for images ClamAV never opens. That mistake was made and caught -while producing the table above. diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index db1fabc1..aabee7ef 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -333,7 +333,7 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "exav-core" -version = "0.0.1" +version = "0.0.2" dependencies = [ "aho-corasick", "daachorse", @@ -345,7 +345,10 @@ dependencies = [ "md-5", "memchr", "regex", + "regex-automata", + "regex-syntax", "rmp-serde", + "rustc-hash", "rustdct", "serde", "sha1", @@ -369,14 +372,15 @@ dependencies = [ [[package]] name = "exav-pe-emu" -version = "0.0.1" +version = "0.0.2" dependencies = [ "exav-x86", + "rustc-hash", ] [[package]] name = "exav-unpack" -version = "0.0.1" +version = "0.0.2" dependencies = [ "aes", "apfs", @@ -423,7 +427,7 @@ dependencies = [ [[package]] name = "exav-x86" -version = "0.0.1" +version = "0.0.2" [[package]] name = "explode" @@ -964,6 +968,12 @@ dependencies = [ "serde", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustdct" version = "0.7.1" diff --git a/fuzz/fuzz_targets/sigs.rs b/fuzz/fuzz_targets/sigs.rs index 648e2910..b16530c8 100644 --- a/fuzz/fuzz_targets/sigs.rs +++ b/fuzz/fuzz_targets/sigs.rs @@ -1,15 +1,13 @@ #![no_main] -//! Signature-text parsers (.ndb, simple Name=HEX, hash db, fuzzy db) on -//! arbitrary UTF-8. +//! Signature-text parsers (simple Name=HEX, hash db, fuzzy db) on arbitrary +//! UTF-8. `.ndb` text is fuzzed through the engine by `ndb_compile`. use libfuzzer_sys::fuzz_target; use exav_core::fuzzy::FuzzyDb; use exav_core::hashes::HashDb; -use exav_core::patterns::PatternSet; fuzz_target!(|data: &[u8]| { if let Ok(text) = std::str::from_utf8(data) { - let _ = PatternSet::parse_ndb(text); - let _ = PatternSet::parse_simple(text); + let _ = exav_core::patterns::parse_simple(text); let mut h = HashDb::new(); h.extend_from_text(text); let mut f = FuzzyDb::new(); diff --git a/fuzz/fuzz_targets/unpack.rs b/fuzz/fuzz_targets/unpack.rs index 02e5c6c2..be484e85 100644 --- a/fuzz/fuzz_targets/unpack.rs +++ b/fuzz/fuzz_targets/unpack.rs @@ -84,6 +84,6 @@ fuzz_target!(|data: &[u8]| { tight_limits(), vec!["infected".to_string(), "hunter2".to_string()], ); - let _ = extract(fmt, data, &mut budget); + let _ = extract(fmt, &data, &mut budget); } }); diff --git a/packaging/exav-clamd.service b/packaging/exav-clamd.service deleted file mode 100644 index c2fb8d1a..00000000 --- a/packaging/exav-clamd.service +++ /dev/null @@ -1,72 +0,0 @@ -# A clamd-compatible scanning daemon, as a drop-in for clamav-daemon. -# -# It serves the SAME Unix socket clamd uses, so `clamdscan`, mail milters, and -# clamd client libraries talk to exav unchanged. Enable it *instead of* -# clamav-daemon (don't run both on the same socket): -# -# sudo systemctl stop clamav-daemon # free the socket -# sudo systemctl disable clamav-daemon -# sudo systemctl enable --now exav-clamd -# clamdscan --ping 1 # verify it answers -# -# The socket path below is the Debian/Ubuntu default. On Fedora/RHEL clamd uses -# /run/clamd.scan/clamd.sock — edit ExecStart + RuntimeDirectory to match, or -# copy your existing clamd.conf's LocalSocket value. -[Unit] -Description=exav clamd-compatible malware scanning daemon -Documentation=https://exav.org/guides/daemon/ -After=network.target - -[Service] -Type=simple -# --workers defaults to one prefork process per CPU core (kernel-enforced per-job -# limits + hard-kill isolation). Reads signatures from the freshclam DB dir. -ExecStart=/usr/bin/exav --listen /run/clamav/clamd.ctl -d /var/lib/exav -# -# The socket is created 0600, reachable only by the clamav user this runs as. -# A client under another UID — Amavis, Rspamd, a web server — needs it widened, -# which is what clamd.conf spells LocalSocketMode. The mode rides on the address, -# since it is a property of that socket: -# ExecStart=... --listen 'clamd:///run/clamav/clamd.ctl?mode=660' -# with the client's user in the clamav group. Every user the mode admits can -# submit scans and read the verdicts, so grant the narrowest set that works -# rather than 666. -# -# --- Pulling a prebuilt database over HTTP (build once, serve many) -------------- -# To instead fetch an `.exavdb` built by another server, point -d at a WRITABLE -# file and give the URL. Requires an `http-update`-featured build. Swap the -# ExecStart above and replace the `ReadOnlyPaths` line in the hardening block -# with `ReadWritePaths` on the database's directory: -# ExecStart=/usr/bin/exav --listen /run/clamav/clamd.ctl --auto-update \ -# --db-url https://build.internal/exav/live.exavdb \ -# -d /var/lib/exav/live.exavdb -# ReadWritePaths=/var/lib/exav # the daemon writes the pulled database + temp file here -# (Basic auth: --db-url https://user:pass@build.internal/... — kept out of logs.) -# See https://exav.org/guides/prebuilt-database/ -# ------------------------------------------------------------------------------ -# -# `systemctl reload exav-clamd` re-reads the signature database without dropping -# the socket: the prefork supervisor reloads and re-forks its workers on SIGHUP. -# (Use this after swapping in a rebuilt prebuilt database.) -ExecReload=/bin/kill -HUP $MAINPID -Restart=on-failure -RestartSec=2 - -# Run as the existing clamav user (present from a ClamAV install). -User=clamav -Group=clamav -RuntimeDirectory=clamav -RuntimeDirectoryMode=0755 - -# Hardening — the daemon only needs to read files and the signature DB. -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=read-only -PrivateTmp=true -ReadOnlyPaths=/var/lib/exav -ProtectControlGroups=true -ProtectKernelTunables=true -RestrictSUIDSGID=true - -[Install] -WantedBy=multi-user.target diff --git a/packaging/exav.service b/packaging/exav.service new file mode 100644 index 00000000..73060565 --- /dev/null +++ b/packaging/exav.service @@ -0,0 +1,76 @@ +# exav serving the clamd protocol, in place of clamav-daemon. +# +# It serves the Unix socket clamd uses, so `clamdscan`, mail milters and clamd +# client libraries talk to exav unchanged, and it reads the signatures freshclam +# keeps in /var/lib/clamav. Enable it *instead of* clamav-daemon: both cannot +# hold the socket. On Debian/Ubuntu the socket unit has to go too, or socket +# activation starts clamd again on the first connection: +# +# sudo systemctl disable --now clamav-daemon.socket clamav-daemon.service +# sudo systemctl enable --now exav +# clamdscan --ping 1 # verify it answers +# +# The socket path below is the Debian/Ubuntu default. On Fedora/RHEL clamd uses +# /run/clamd.scan/clamd.sock: edit ExecStart and RuntimeDirectory to match, or +# copy your existing clamd.conf's LocalSocket value. +[Unit] +Description=exav malware scanning daemon (clamd protocol) +Documentation=https://exav.org/guides/daemon/ +After=network.target + +[Service] +Type=simple +# --workers defaults to one prefork process per CPU core (kernel-enforced per-job +# limits, and a job that will not stop is killed without touching the others). +ExecStart=/usr/bin/exav --listen /run/clamav/clamd.ctl -d /var/lib/clamav +# +# The socket is created 0600, reachable only by the clamav user this runs as. +# A client under another UID (Amavis, Rspamd, a web server) needs it widened, +# which is what clamd.conf spells LocalSocketMode. The mode rides on the address, +# since it is a property of that socket: +# ExecStart=... --listen 'clamd:///run/clamav/clamd.ctl?mode=660' +# with the client's user in the clamav group. Every user the mode admits can +# submit scans and read the verdicts, so grant the narrowest set that works +# rather than 666. +# +# --- Pulling a prebuilt database over HTTP (build once, serve many) ---------- +# To fetch an `.exavdb` built by another server instead, point -d at a writable +# file and give the URL. Needs a build with the `http-update` feature. Replace +# ExecStart above, and the `ReadOnlyPaths` line below with the two lines after +# it: +# ExecStart=/usr/bin/exav --listen /run/clamav/clamd.ctl --auto-update \ +# --db-url https://build.internal/exav/live.exavdb \ +# -d /var/lib/exav/live.exavdb +# StateDirectory=exav # creates /var/lib/exav for the clamav user +# ReadWritePaths=/var/lib/exav # the pulled database and its temp file +# (Basic auth: --db-url https://user:pass@build.internal/... is kept out of logs.) +# See https://exav.org/guides/prebuilt-database/ +# ----------------------------------------------------------------------------- +# +# As with clamd: `systemctl reload exav` (SIGUSR2) re-reads the signatures +# without dropping the socket, and SIGHUP reopens the --log file, for logrotate: +# postrotate systemctl kill -s HUP exav.service +# freshclam's NotifyClamd reloads it through the socket, as it does clamd. +ExecReload=/bin/kill -USR2 $MAINPID +Restart=on-failure +RestartSec=2 + +# Run as the existing clamav user (present from a ClamAV install). +User=clamav +Group=clamav +RuntimeDirectory=clamav +RuntimeDirectoryMode=0755 + +# Hardening: the daemon only needs to read the files it scans and the +# signatures. +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=read-only +PrivateTmp=true +ReadOnlyPaths=/var/lib/clamav +ProtectControlGroups=true +ProtectKernelTunables=true +RestrictSUIDSGID=true + +[Install] +WantedBy=multi-user.target diff --git a/scripts/clamav-reach.sh b/scripts/clamav-reach.sh index a0311950..b52cbd7d 100755 --- a/scripts/clamav-reach.sh +++ b/scripts/clamav-reach.sh @@ -20,7 +20,7 @@ # # GOTCHA: gunzip `.gz`-wrapped fixtures first. Otherwise this measures gzip # rather than the inner format and reports "reached all" for images ClamAV never -# opens — a mistake made and caught while producing docs/VERDICT_PROTOCOL_PLAN.md. +# opens (see docs/PARTIAL.md, section 6). # # REQUIREMENTS: clamscan on PATH; `cargo build --release -p exav` and # `cargo build -p exav-unpack`. diff --git a/scripts/difftest.sh b/scripts/difftest.sh index b9deee48..8496850c 100755 --- a/scripts/difftest.sh +++ b/scripts/difftest.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # exav vs ClamAV differential testing, in three separable phases. # -# 1. clamd over the corpus -> results-clam.tsv (CACHED — see below) +# 1. clamd over the corpus -> results-clam.tsv (CACHED, see below) # 2. exav over the corpus -> results-exav.tsv # 3. compare the two tables # @@ -9,7 +9,7 @@ # # * **ClamAV does not change.** Its verdicts are a function of the signature # database, which is pinned here. Re-scanning 18k files through it on every -# exav change is pure waste — so phase 1 writes a cache and later runs skip +# exav change is pure waste, so phase 1 writes a cache and later runs skip # straight to phase 2. Phase 1 is the slow one; you pay it once. # * **Only one engine is resident at a time.** The old harness ran both # daemons together. On a small host that is ~2 GB of clamd plus ~1 GB of @@ -24,7 +24,7 @@ # difference in results cannot come from a difference in how they were asked. # Every scan is ALL-MATCH: a verdict is the SET of signatures that matched, not # whichever one an engine reached first. Comparing first-matches is the single -# largest source of fake disagreement — both engines find the malware, each +# largest source of fake disagreement: both engines find the malware, each # names a different signature, and the diff calls it a conflict. # # USAGE @@ -46,7 +46,7 @@ # phase 1 has run for a given manifest only exav is re-paid. # # LIMIT picks a deterministic sample (fixed seed), and BOTH engines scan exactly -# that list — the manifest is generated once and shared. A sampled run that gave +# that list: the manifest is generated once and shared. A sampled run that gave # each engine its own random subset would compare different files and call the # result a difference. set -u @@ -62,7 +62,7 @@ MAXSZ="${MAXSZ:-20971520}" # skip files > 20 MB # Per-file cap. Generous on purpose: under all-match neither engine may stop at # the first hit, so a single xlsx or 7z is a full walk of every member, and the # disk here reads cold data at ~8.6 MB/s. A tight cap does not make the run -# faster — it converts slow files into ERROR rows, which is worse than waiting +# faster; it converts slow files into ERROR rows, which is worse than waiting # because they then look like a compliance difference. TMO="${TMO:-120}" CLAMD_START_TMO="${CLAMD_START_TMO:-900}" # clamd DB load can take minutes @@ -71,8 +71,8 @@ FRESH_CLAM="${FRESH_CLAM:-0}" COMPAT="${COMPAT:-1}" # exav at --clamav-compat (apples-to-apples) HEURISTICS="${HEURISTICS:-1}" # opt-in alert classes, BOTH engines SHOW="${SHOW:-15}" # examples per bucket in the report -# Concurrent scans. Both engines run at the SAME value — that is what keeps the -# two runs comparable at all — and clamd's own thread pool is sized to match. +# Concurrent scans. Both engines run at the SAME value (that is what keeps the +# two runs comparable at all), and clamd's own thread pool is sized to match. # # Default: 2x CPUs, capped at 8. Measured on cold data (see difftest-scan.py), # concurrency is worth ~1.75x rather than the job count, because the disk is the @@ -116,7 +116,7 @@ PROFILE="COMPAT=$COMPAT HEURISTICS=$HEURISTICS JOBS=$JOBS" # The FILE LIST is part of that identity, not just the flags. The cached clam # table is skipped on row count alone, so a cache holding the right NUMBER of -# rows for the wrong files would be reused and then joined against exav's — and +# rows for the wrong files would be reused and then joined against exav's, and # the join is by path, so the overlap is empty and the run reports on nothing. # LIMIT/SEED are not enough on their own either: the corpus itself changes as # samples are added. Fingerprint the manifest that was actually scanned. @@ -130,7 +130,7 @@ say() { echo "[$(date '+%H:%M:%S')] $*"; } cleanup() { # Save the container's log BEFORE removing it. Removing first is what turned # a startup failure into "FATAL: clamd socket never appeared" with no way to - # find out why — the evidence was deleted by the handler reporting the error. + # find out why: the evidence was deleted by the handler reporting the error. if docker inspect "$CONTAINER" >/dev/null 2>&1; then docker logs "$CONTAINER" >"$TMPROOT/difftest-clamd.log" 2>&1 || true fi @@ -143,13 +143,13 @@ trap cleanup EXIT # ── Signatures ─────────────────────────────────────────────────────────────── # daily.cvd ONLY, deliberately: both engines must load exactly the same set or # the comparison is meaningless. `freshclam` also fetches main.cvd, so it is -# removed again — which also keeps the exav database build to ~340k signatures +# removed again, which also keeps the exav database build to ~340k signatures # instead of 3.6M, the difference between building and being OOM-killed here. # # Called by the phases that need it rather than run up front. Two of them do: # clamd loads this directory, and building the exav database reads it. Neither -# applies to `PHASE=exav` against an already-built `$DB` — the documented way to -# iterate on exav — and fetching there would make the fast path depend on docker +# applies to `PHASE=exav` against an already-built `$DB` (the documented way to +# iterate on exav), and fetching there would make the fast path depend on docker # and a network it has no use for. need_signatures() { [ -f "$DBDIR/daily.cvd" ] && return 0 @@ -167,8 +167,8 @@ build_manifest() { # `shuf` with a fixed random source so a sampled run is reproducible: the same # LIMIT and SEED select the same files, which is what makes two runs # comparable at all. - # `find -printf` gives size and path in ONE pass. The obvious version — pipe - # paths into a shell loop and `stat` each — spawns a process per file and took + # `find -printf` gives size and path in ONE pass. The obvious version (pipe + # paths into a shell loop and `stat` each) spawns a process per file and took # 3m47s over 18.5k files, which is longer than scanning a good few of them. find "$CORPUS" -type f \ ! -name '*.py' ! -name '*.sh' ! -name '*.json' ! -name '*.md' \ @@ -188,7 +188,7 @@ build_manifest() { # An empty manifest is not an empty run: every later phase "succeeds" over # nothing and the report reads as a clean sweep. Fail where the cause is # visible (wrong CORPUS, everything over MAXSZ, a find that matched nothing). - [ "$n" -gt 0 ] || { echo "FATAL: manifest is empty — is CORPUS=$CORPUS right, and MAXSZ=$MAXSZ not excluding everything?"; exit 1; } + [ "$n" -gt 0 ] || { echo "FATAL: manifest is empty. Is CORPUS=$CORPUS right, and MAXSZ=$MAXSZ not excluding everything?"; exit 1; } say "manifest: $n files" } @@ -198,7 +198,7 @@ check_cache() { local tsv=$1 stamp="$1.profile" now now="$(current_profile)" if [ -f "$tsv" ] && [ -s "$stamp" ] && [ "$(cat "$stamp")" != "$now" ]; then - say "WARNING: $tsv was built under [$(cat "$stamp")], now [$now] — discarding" + say "WARNING: $tsv was built under [$(cat "$stamp")], now [$now]: discarding" rm -f "$tsv" fi printf '%s' "$now" > "$stamp" @@ -221,10 +221,10 @@ phase_clam() { if [ -f "$CLAM_TSV" ]; then have=$(awk -F'\t' 'NR>1 && $3!="ERROR"' "$CLAM_TSV" | wc -l) if [ "$have" -ge "$todo" ]; then - say "phase 1: clam results already cached ($have answered rows) — skipping" + say "phase 1: clam results already cached ($have answered rows), skipping" return 0 fi - say "phase 1: cache has $have answered of $todo — rescanning the rest" + say "phase 1: cache has $have answered of $todo, rescanning the rest" fi say "phase 1: starting clamd" @@ -241,7 +241,7 @@ MaxScanSize 2000M MaxFileSize 2000M CONF # Without this clamd REFUSES ALLMATCHSCAN, and the refusal looks like a clean - # file — every reply would read OK and the whole run would be silently void. + # file: every reply would read OK and the whole run would be silently void. echo "AllowAllMatchScan yes" >> "$conf" if [ "$HEURISTICS" = 1 ]; then cat >> "$conf" <<'HEUR' @@ -269,7 +269,7 @@ HEUR "$IMAGE" --foreground --config-file=/etc/clamav/clamd.conf >/dev/null || { echo "FATAL: could not start the clamd container"; exit 1; } # clamd loads ~340k signatures before it binds the socket, and with the alert - # classes on it took 7 MINUTES here — a 3-minute wait failed the run for no + # classes on it took 7 MINUTES here, and a 3-minute wait failed the run for no # reason. Wait generously: the cost of waiting is nothing next to the cost of # a wasted phase. local waited=0 @@ -284,12 +284,12 @@ HEUR [ -S "$CSOCK" ] || { echo "FATAL: clamd socket never appeared after ${CLAMD_START_TMO}s" docker logs "$CONTAINER" 2>&1 | tail -20; exit 1; } - say "phase 1: clamd ready after ${waited}s — scanning" + say "phase 1: clamd ready after ${waited}s, scanning" python3 scripts/difftest-scan.py --socket "$CSOCK" --manifest "$MANIFEST" \ --out "$CLAM_TSV" --timeout "$TMO" --jobs "$JOBS" --label clam - # Save the log BEFORE removing the container — `cleanup()` also does this, but + # Save the log BEFORE removing the container. `cleanup()` also does this, but # only on script EXIT, by which time this `rm` has already destroyed the # evidence. The log is the sole record of clamd's own bail-outs (a single run # showed 119 bytecode-interpreter timeouts on one program, plus hundreds of @@ -297,10 +297,10 @@ HEUR # that are otherwise unattributable. docker logs "$CONTAINER" >"$TMPROOT/difftest-clamd.log" 2>&1 || true # A daemon that died mid-run turns every remaining file into an ERROR row, and - # a stale socket makes those fail instantly — so the run *finishes*, fast, with + # a stale socket makes those fail instantly, so the run *finishes*, fast, with # a table full of holes. Say so plainly; the row count alone looks like success. if ! docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null | grep -q true; then - say "WARNING: clamd was NOT running at the end of the phase — it died mid-run." + say "WARNING: clamd was NOT running at the end of the phase: it died mid-run." say " Re-run PHASE=clam to retry the ERROR rows; see $TMPROOT/difftest-clamd.log" say " If it was OOM-killed (log ends mid-line, no shutdown message), raise MEMORY=$MEMORY." fi @@ -312,7 +312,7 @@ HEUR phase_exav() { [ -x "$EXAV" ] || { echo "FATAL: $EXAV not built (cargo build --release -p exav)"; exit 1; } # A rebuilt exav can bump the database format, making a stale database - # unloadable — but `--build-db` needs several GB of RAM, so do NOT rebuild + # unloadable, but `--build-db` needs several GB of RAM, so do NOT rebuild # merely because the binary is newer. Rebuild when it is missing or fails to # LOAD, which is the failure that actually matters. local need=0 @@ -320,7 +320,7 @@ phase_exav() { elif [ "$EXAV" -nt "$DB" ]; then mkdir -p "$SOCK_DIR"; : > "$SOCK_DIR/.probe" "$EXAV" -d "$DB" "$SOCK_DIR/.probe" >/dev/null 2>&1 \ - && { say "phase 2: database older than the binary but loads — reusing"; touch "$DB"; } \ + && { say "phase 2: database older than the binary but loads, reusing"; touch "$DB"; } \ || need=1 fi if [ "$need" = 1 ]; then @@ -334,8 +334,8 @@ phase_exav() { echo "FATAL: exav database build failed (it needs several GB of FREE RAM)"; exit 1; } fi - # exav's results are NOT cached across binaries — that is the entire reason for - # re-running it — so the table is rebuilt unless the caller kept it on purpose. + # exav's results are NOT cached across binaries (that is the entire reason for + # re-running it), so the table is rebuilt unless the caller kept it on purpose. [ "${KEEP_EXAV:-0}" = 1 ] || rm -f "$EXAV_TSV" check_cache "$EXAV_TSV" @@ -343,12 +343,12 @@ phase_exav() { # the same budget the client waits. Both matter, and getting either wrong is # invisible in the results: # - # * exav preforks one worker PER CORE by default — 2 here. Sending 4 + # * exav preforks one worker PER CORE by default: 2 here. Sending 4 # concurrent scans meant half of them sat in the accept queue behind a # long job and timed out having never been looked at. That records as # ERROR, which reads as a compliance difference and is not one. Measured: # 27 of 40 files "failed" this way, with 13 worker deaths. - # * `--max-scan-time` defaults to 120s. If the client's own timeout is + # * `--max-scan-secs` defaults to 120. If the client's own timeout is # longer, a killed worker looks like a slow file; if shorter, the client # gives up on work the daemon then completes for nobody. Keep them equal. # `--max-scan-secs` takes whole seconds, so a fractional TMO is rounded @@ -360,7 +360,7 @@ phase_exav() { local flags="--workers $JOBS --max-scan-secs $max_scan_secs" [ "$COMPAT" = 1 ] && flags="$flags --clamav-compat" # Cover every alert class clamd was configured with. Anything clamd is asked - # to alert on and exav is not becomes a fake FN — the file is bucketed as + # to alert on and exav is not becomes a fake FN: the file is bucketed as # "clam detected, exav did not" when exav was never asked to look. Measured # before these were added: 150 of 258 FNs, dominated by 119 # Heuristics.Broken.Executable and 33 Heuristics.Limits.Exceeded.*. @@ -373,7 +373,7 @@ phase_exav() { # AlertEncryptedDoc and AlertExceedsMax actually are: ClamAV reports both as # `Heuristics.* FOUND`, so exav has to as well or the same file is a PARTIAL # here and a detection there. `unscannable` is deliberately left at the - # default — ClamAV has no flag for it, so making it a detection would invent a + # default: ClamAV has no flag for it, so making it a detection would invent a # disagreement rather than remove one. # # Keep both in step with the AlertX lines in the clamd conf above; they are @@ -390,7 +390,7 @@ phase_exav() { for _ in $(seq 1 60); do [ -S "$ESOCK" ] && break; sleep 2; done [ -S "$ESOCK" ] || { echo "FATAL: exav socket never appeared:"; tail -5 "$TMPROOT/difftest-exav-daemon.log"; exit 1; } - say "phase 2: exav ready — scanning" + say "phase 2: exav ready, scanning" python3 scripts/difftest-scan.py --socket "$ESOCK" --manifest "$MANIFEST" \ --out "$EXAV_TSV" --timeout "$TMO" --jobs "$JOBS" --label exav @@ -400,9 +400,12 @@ phase_exav() { # got to answer", and they are only visible here. # `grep -c` prints 0 AND exits 1 when nothing matches, so `|| echo 0` would # make this "0\n0" and the numeric test below an error. Assign on failure. + # A worker recycled at its job limit, or retired by a reload, answered + # everything it was given. local dead - dead=$(grep -c "worker .* exited" "$TMPROOT/difftest-exav-daemon.log" 2>/dev/null) || dead=0 - [ "$dead" -gt 0 ] && say "NOTE: $dead worker exits (timeout/OOM) — see $TMPROOT/difftest-exav-daemon.log" + dead=$(grep "worker .* exited" "$TMPROOT/difftest-exav-daemon.log" 2>/dev/null \ + | grep -vc "exited: recycled\|exited: retired") || dead=0 + [ "$dead" -gt 0 ] && say "NOTE: $dead worker deaths (timeout/OOM/abort), see $TMPROOT/difftest-exav-daemon.log" } # ── Phase 3: compare ───────────────────────────────────────────────────────── diff --git a/scripts/wasm-format-sizes.sh b/scripts/wasm-format-sizes.sh index 0e4db15b..95671a14 100755 --- a/scripts/wasm-format-sizes.sh +++ b/scripts/wasm-format-sizes.sh @@ -26,11 +26,12 @@ TARGET="${TARGET:-wasm32-unknown-unknown}" BUILDSTD="${BUILDSTD:-0}" mkdir -p "$WORK/src" -# Format list: exav-unpack's `all-formats` array (kept in sync automatically). +# Format list: exav-unpack's `all-formats-no-emu` array plus `pe-emu`, which +# together make `all-formats` (kept in sync automatically). if [ -z "${FMTS:-}" ]; then - FMTS=$(awk '/^all-formats = \[/{f=1} f{print} f&&/\]/{exit}' \ - "$ROOT/crates/exav-unpack/Cargo.toml" | grep -oE '"[a-z0-9]+"' | tr -d '"' \ - | grep -v '^all-formats$' | tr '\n' ' ') + FMTS="$(awk '/^all-formats-no-emu = \[/{f=1} f{print} f&&/\]/{exit}' \ + "$ROOT/crates/exav-unpack/Cargo.toml" | grep -oE '"[a-z0-9-]+"' | tr -d '"' \ + | grep -v '^all-formats' | tr '\n' ' ')pe-emu" fi # Generate the minimal wrapper crate (all format features forwarded). diff --git a/www/astro.config.mjs b/www/astro.config.mjs index fa5890bb..bd6c8e90 100644 --- a/www/astro.config.mjs +++ b/www/astro.config.mjs @@ -47,6 +47,7 @@ export default defineConfig({ items: [ { label: 'Scanning', slug: 'guides/scanning' }, { label: 'Daemon mode', slug: 'guides/daemon' }, + { label: 'Sizing a server', slug: 'guides/sizing' }, { label: 'ICAP server', slug: 'guides/icap' }, { label: 'Docker', slug: 'guides/docker' }, { label: 'Signatures', slug: 'guides/signatures' }, diff --git a/www/src/content/docs/concepts/architecture.md b/www/src/content/docs/concepts/architecture.md index 2220fa10..bee2cba5 100644 --- a/www/src/content/docs/concepts/architecture.md +++ b/www/src/content/docs/concepts/architecture.md @@ -11,44 +11,91 @@ high-level picture; implementation detail lives in the repository's `docs/`. Every input is scanned through a seekable source (`Read + Seek`): a local file, an HTTP range reader, or a buffered stream. Stdin, `INSTREAM` and ICAP bodies are buffered first (in memory, then in a temporary file), because container formats -need to seek. `scan_seekable` types the input, walks containers through the -reader (fetching only the directory and the members it scans), and runs the -engine. See [Streaming & memory](/concepts/streaming-memory/) for what is held in -memory and what is streamed. +need to seek. `scan_seekable` hands the input to the one pipeline every object +takes, the input and whatever is unpacked, decoded or carved out of it, at any +size and depth. See [Streaming & memory](/concepts/streaming-memory/) for what +is held in memory and what is read as it is needed. -`exav` and the WASI build are the front ends, `exav-core` is the engine, and -`exav-unpack` does extraction (and backs the standalone `exav-grep`). See -[How the crates compose](#how-the-crates-compose) below, and -[Subprojects](/subprojects/) for what each one owns. +## How a scan flows -## Not one monolithic engine + + How a scan flows + An object enters identify(), which types it from its start, + its end, or a search of it. An archive has its members walked one at a time + under a single shared Budget, then its own bytes matched. Anything else has + its own bytes matched first, then what is inside it analysed: a packed or + installer executable's payload, carved images, decoded payloads, heuristics. + Everything found inside re-enters identify(), bounded at --max-unpack-depth of 16. + Both paths end in OK, FOUND, or PARTIAL with its category: LIMITS-EXCEEDED, + UNSCANNABLE or PASSWORD-PROTECTED; an input that cannot be read, even part + way, is ERROR. + + + + + + + + + + + + + + object + identify() + archive + anything else + OK · FOUND · PARTIAL · ERROR + PARTIAL: LIMITS-EXCEEDED · UNSCANNABLE · PASSWORD-PROTECTED + + + start · end · a search + members one at a time, sharing + one Budget; then its own bytes + its own bytes; then what is inside: + payloads, carved images, heuristics + not fully examined is PARTIAL, never OK; input unreadable is ERROR + everything found inside re-enters + identify() · --max-unpack-depth (16) + + + + + + + + + + + -Different ClamAV signature types compile to different runtime structures rather -than one matcher: +A file is `OK`, `FOUND`, or `PARTIAL` when it could not be fully examined, with +a category saying why: a limit (`LIMITS-EXCEEDED`), content with no decoder +(`UNSCANNABLE`) or encryption (`PASSWORD-PROTECTED`). Anything that stops a scan +early is `PARTIAL`, never `OK`. `ERROR` is exav failing rather than a fact +about the file: a path it cannot open, a database that will not load, or a +source that fails part way through a scan, whose unread bytes are not taken for +the end of the file (see [Verdicts & exit codes](/reference/verdicts/)). -| Source | Runtime structure | Matching | -|---|---|---| -| `.ndb` bodies + `.ldb` literal subsignatures | a shared **Aho-Corasick automaton** keyed on a literal anchor per body | an automaton hit fans out to every body sharing that anchor; each candidate is then verified (wildcards / gaps / nibbles / alternation / offset / nocase) | -| `.ldb` PCRE subsignatures | regexes compiled lazily, gated by the trigger expression | linear-time `regex`; patterns with backreferences use a backtracking engine under a step bound | -| `.hsb` / `.hdb` | a size-keyed hash table | whole-file digest lookup | -| `.mdb` / `.msb` | a section-hash table | per-PE-section digest lookup | -| `.cdb` | container-metadata matchers | matched on archive members (name/size/encryption/position) | -| `.imp` | a size-constrained import-hash map | PE imphash lookup | -| `.cbc` | a [sandboxed bytecode interpreter](/concepts/bytecode-sandbox/) (no JIT) | trigger-gated programs run on extracted buffers | -| `.yar` / `.yara` | a [native YARA engine](/guides/yara/) | near-full YARA, no runtime codegen | +### Content-based typing -So the engine is one Aho-Corasick automaton (fed by `.ndb` and `.ldb` literal -subsignatures), several cheap hash tables, lazy regexes and interpreters. Almost -all of the memory cost is the automaton; see the -[prebuilt database](/guides/prebuilt-database/). +File type is decided by magic bytes, not by extension: an executable renamed +`.jpg` is still typed and scanned as an executable. + +### Recursive unpacking, bounded + +See [Archive extraction](/concepts/archive-extraction/) for the budgets every +extraction runs under and what a member exav cannot read becomes. ## How the crates compose -exav is a small Cargo workspace, not one binary. Four front-ends drive one -engine: the CLI (which is also the daemon), the WASI build, the archive grep, -and the WebAssembly bindings published to npm. `exav-core` is the engine; -`exav-unpack` does extraction; `exav-update` sits to the side, feeding fresh -signatures out of band. +exav is a small Cargo workspace, not one binary. Two front ends drive the +engine, `exav-core`: the CLI (which is also the daemon) and the WASI build. Two +more use only the extractor, `exav-unpack`: the archive grep, and the +WebAssembly bindings published to npm (built outside the workspace, with their +own profile). `exav-update`, enabled by the `http` feature, sits to the side, +feeding fresh signatures out of band. Every arrow points down. Nothing below calls anything above it, which is what lets the extraction crates be taken on their own. @@ -59,8 +106,9 @@ lets the extraction crates be taken on their own. wasm32-wasip1, exav-grep and exav-unpack-wasm. exav and the wasi binary depend on exav-core; exav-core, exav-grep and exav-unpack-wasm all depend on exav-unpack, which depends on exav-pe-emu, which depends on exav-x86. - exav-update hangs off exav alone and feeds signature files out of - band. + exav-core also depends on exav-x86 directly, for the bytecode disassembly + API. exav-update, optional behind the http feature, hangs off exav alone and + feeds signature files out of band. @@ -92,7 +140,7 @@ lets the extraction crates be taken on their own. clamscan + clamd the only unsafe: libc - --wasi-bin + feature wasi-bin wasm32-wasip1 grep inside archives @@ -104,7 +152,8 @@ lets the extraction crates be taken on their own. #![forbid(unsafe_code)] x86-32 sandbox decoder - signatures, out of band + optional: http feature + bytecode disasm runs a packer's own stub to recover the image it rebuilds decode only, @@ -119,86 +168,331 @@ lets the extraction crates be taken on their own. + -A scan flows through it like this: +Extraction touches hostile bytes first and hardest, so it lives in its own +`#![forbid(unsafe_code)]` crate with its own budget and panic containment. It +also means one piece can be used alone: a build system that needs to look inside +archives does not need a virus scanner, and a reverse engineer who wants to +unpack a packed executable needs neither. - - How a scan flows - A file enters identify(), which types the first 4 KiB. A - container has its members walked one at a time under a single shared Budget, - and each member re-enters identify() bounded at max_recursion of 16. A flat - file goes to pattern, hash and heuristic matching. Both paths end in one of - five verdicts: Clean, Infected, LimitsExceeded, Unscannable or - PasswordProtected. +`exav-pe-emu` sits below the extractor for the same reason. Running a packer's stub +is the one place where the scanner executes attacker-authored *control flow* +rather than parsing attacker-authored data, so it is its own crate with its own +budgets, its own `#![forbid(unsafe_code)]`, and no way to reach a syscall. + +Each crate has its own page under [Subprojects](/subprojects/), with install +instructions, API, and examples. + +## Signature types and runtime structures + +Different ClamAV signature types compile to different runtime structures rather +than one matcher: + +| Source | Runtime structure | Matching | +|---|---|---| +| `.ndb` bodies + `.ldb` literal subsignatures | a set of **Aho-Corasick automata**, one per (target type, case) pair, keyed on a literal anchor per body | an automaton hit fans out to every body sharing that anchor; each candidate is then verified (wildcards / gaps / nibbles / alternation / offset / nocase) | +| `.ldb` PCRE subsignatures | regexes compiled lazily, gated by the trigger expression | linear-time `regex`; patterns with backreferences use a backtracking engine under a step bound | +| `.hsb` / `.hdb` | a size-keyed hash table | whole-file digest lookup | +| `.mdb` / `.msb` | a section-hash table | per-PE-section digest lookup | +| `.cdb` | container-metadata matchers | matched on archive members (name/size/encryption/position) | +| `.imp` | a size-constrained import-hash map | PE imphash lookup | +| `.cbc` | a [sandboxed bytecode interpreter](/concepts/bytecode-sandbox/) (no JIT) | trigger-gated programs run on extracted buffers | +| `.yar` / `.yara` | a [native YARA engine](/guides/yara/) | near-full YARA, no runtime codegen | + +So the engine is a set of Aho-Corasick automata (fed by `.ndb` and `.ldb` literal +subsignatures), several cheap hash tables, lazy regexes and interpreters. Almost +all of the memory cost is the automata; see the +[prebuilt database](/guides/prebuilt-database/). + +## How a body signature is matched + + + From anchor hit to detection + An object's bytes, and a lowercased view of them for + case-insensitive anchors, go through the partitions whose target fits the + file type. One Aho-Corasick sweep reports every anchor hit as a group and an + offset. Each hit fans out to every body sharing the anchor, and each is + verified around the hit. A verified .ndb body is a detection. A verified .ldb + subsignature is counted, and the logical expressions of the .ldb signatures + whose subsignatures hit are evaluated after the sweep; a PCRE subsignature + runs only when its trigger holds. - + - - - - - + + + + + + + - - file - identify() - container - flat - Clean · Infected · LimitsExceeded - Unscannable · PasswordProtected + + object + partitions, by (target, case) + one Aho-Corasick sweep + fan out, then verify each body + .ndb body verified + .ldb subsignature verified + logical expression - types the first 4 KiB - walk members one at a time, - sharing one Budget - pattern + hash + heuristics - every scan ends in exactly one verdict - each member re-enters identify() - bounded at max_recursion (16) + its bytes, and a lowercased view + only those whose target fits the file type run + every anchor at once: hits as (group, offset) + every body sharing the anchor: prefix back, suffix + forward, gaps as intervals, offset, ::f word bounds + a detection; a normal scan stops here, + --all-matches keeps verifying + counted, with the offset of its first match + (byte comparisons are taken from there) + after the sweep, only for the signatures whose + subsignatures hit; a PCRE runs if its trigger holds - - - - - - - + + + + + + + -The five verdicts are the whole output vocabulary. Anything that stops a scan -early produces one of the last three rather than `Clean` (see -[Verdicts & exit codes](/reference/verdicts/)). +An anchor is a literal run each body must contain; everything else in the body +(wildcards, nibbles, alternations, gaps, the offset) is checked only where an +anchor hit. Anchors are split by target type and case, so a scan runs only the +automata that can matter for the object's type, and a PE never walks the +HTML-only anchors. The automata that apply advance together over one read of +the object, whatever the number of signatures in them, so the processor waits on +their memory at the same time; case-insensitive ones read each byte lowercased. -Extraction touches hostile bytes first and hardest, so it lives in its own -`#![forbid(unsafe_code)]` crate with its own budget and panic containment. It -also means one piece can be used alone: a build system that needs to look inside -archives does not need a virus scanner, and a reverse engineer who wants to -unpack a packed executable needs neither. +Verification never backtracks: a body is a sequence of elements, and the +positions it can have reached after each one are kept as a set of intervals, so +a `{10-}` gap widens an interval instead of multiplying attempts. Backtracking +is left to PCRE subsignatures with lookaround or backreferences, which are not +regular and so cannot run on the linear engine: they run on a backtracking +engine under a step bound, after a linear prefilter (the pattern with its +lookarounds removed, which matches at least as much) has ruled most objects out. -`exav-pe-emu` sits below the extractor for the same reason. Running a packer's stub -is the one place where the scanner executes attacker-authored *control flow* -rather than parsing attacker-authored data, so it is its own crate with its own -budgets, its own `#![forbid(unsafe_code)]`, and no way to reach a syscall. +### How far a check reads around a hit -Each crate has its own page under [Subprojects](/subprojects/), with install -instructions, API, and examples. + + How far a check reads around an anchor hit + An object drawn as a strip of 64 KiB blocks. The sweep reads + it in order, a chunk at a time, carrying the automaton's state across chunk + seams. At an anchor hit, verification reads the body's prefix backward from + the anchor and its suffix forward, as far as the body needs; a gap with no + upper bound can reach the end of the object. A block no longer in the cache + is read again. The cache bounds what is held, not how far a check reaches. + + + + + + + THE OBJECT, IN 64 KIB BLOCKS + + + + + + + + + + + + + anchor + + + + + + + + + + + + sweep: in order, a chunk at a time; the automaton's state carries across seams, so nothing is read twice + prefix: read backward from the anchor + suffix: read forward + a {n-} gap can reach the end of the object + not in the cache: read again + Offsets are checked against where the match starts: EOF-n everywhere, EP+n and Sn+n + only on an object held in memory. A ::f body is checked against the byte on each side. + Held: 8 MiB of 64 KiB blocks (the sweep reads 1 MiB chunks), least recently used dropped first. + + -## Recursive unpacking, bounded +Neither the sweep nor a check has a window to fall out of. The automata read +the object once, in order, and their state carries from one chunk to the next, +so an anchor split across a seam is found without re-reading anything. A check +asks for the bytes it needs at their offset, before or after the hit, from +memory or, past `--max-object-bytes`, from the +[block cache](/concepts/streaming-memory/#past-it-through-a-block-cache). -The `exav-unpack` crate recursively extracts archives and structured documents -(see [Supported formats](/reference/formats/)). Every path runs under -decompression-bomb defenses: output-byte, ratio, file-count, recursion-depth, -cumulative scan-byte and emulation-step budgets. A bomb is `LIMITS-EXCEEDED`; a -member with an unsupported codec or encryption is `UNSCANNABLE` or -`PASSWORD-PROTECTED`. +Over an object not held in memory, regular expressions (PCRE subsignatures, +YARA strings) run as lazy DFAs stepped through it, forward to find where a match +ends and backward from there to find where it starts. A PCRE with lookaround +runs its lookaround-free superset that way: when that finds nothing, the PCRE +cannot match either; when it finds something, the scan is `LIMITS-EXCEEDED` +unless something else is found, as it is when a lazy DFA meets a construct it +cannot follow (a Unicode word boundary next to a non-ASCII byte). -## Content-based typing +## Reading an object not held in memory -File type is decided by magic bytes, not by extension: an executable renamed -`.jpg` is still typed and scanned as an executable. + + One source, read through a block cache + A file, an HTTP range reader or a spill file is read through a + block cache of 64 KiB blocks holding 8 MiB. Every check asks it for bytes at + an offset: the signature sweep a chunk at a time in order, verification and + the container walk at any offset, the regexes stepped through, the hashes, + typing's search and carving in one shared pass. The checks that parse an object whole run only when it fits within + --max-object-bytes. + + + + + + + SOURCES + WHAT READS IT + + + + + + + + + + + + + + + file + HTTP range + spill file + BlockCache + signature sweep + verification + regexes + hashes · typing · carving + container walk + whole-object checks + + + past --max-object-bytes + a request per block + a large member or view + 64 KiB blocks, 8 MiB held, + least recently used dropped; + bytes at any offset, or in + 1 MiB chunks in order + chunks in order, one pass + windows at any offset + lazy DFAs stepped through + one shared pass; start and end + headers and members, by offset + held whole only: PE, RAR, OLE + + + + + + + + + + + + + +What the cache holds, and which checks parse an object whole and so do not run +past `--max-object-bytes`, is in +[Streaming & memory](/concepts/streaming-memory/#past-it-through-a-block-cache). + +## Where the spill fits + + + The two uses of spill files + Before a scan, a stream (stdin, INSTREAM, ICAP) is held in + memory up to --spill-threshold-bytes and then in a temporary file up to + --max-spill-bytes, and becomes a seekable source for the scan. Inside a scan, + a streamed archive member that decodes past --max-object-bytes, and the normalised + text views of a large text file, are written to spill files and read back + through a block cache. A stream past its budget, or past the threshold with + --spill-dir off, has what was held scanned; with --spill-dir off, such a + member or view is not scanned. Either way a scan that finds nothing is + LIMITS-EXCEEDED. + + + + + + + BEFORE THE SCAN: A STREAM + INSIDE THE SCAN: WHAT IT MAKES + + + + + + + + + + + + + stdin · INSTREAM + RAM + temp file + the scan + member past the limit + text views, large file + spill file + scanned from there + + + ICAP bodies + --spill-threshold-bytes + --max-spill-bytes + a seekable source + decoded past --max-object-bytes + HTML, text, JS normalised + written as it is made + through a block cache + + + + + + + + + + + Past --max-spill-bytes, or past the threshold with --spill-dir off, what was held is + scanned and the stream is LIMITS-EXCEEDED unless that finds something. + --max-total-spill-bytes bounds every spill file of a process together. + + + +The library never writes to disk itself: the host hands a scan somewhere to +spill (the `exav` binary passes its `--spill-dir` files and budgets). Only a +member decoded as it is read can spill; one decoded whole is bounded by +`--max-object-bytes` (see [Archive members](/concepts/streaming-memory/#archive-members)). +Without a spill, a member that decodes past `--max-object-bytes` is not scanned, +the normalised views of a text file that large are skipped, and a scan that +found nothing is `LIMITS-EXCEEDED` rather than `OK`. diff --git a/www/src/content/docs/concepts/archive-extraction.md b/www/src/content/docs/concepts/archive-extraction.md index 797da285..791d7e69 100644 --- a/www/src/content/docs/concepts/archive-extraction.md +++ b/www/src/content/docs/concepts/archive-extraction.md @@ -4,8 +4,8 @@ description: How exav unpacks containers, from the parity principle and bounded --- Most malware does not arrive as a bare executable. It arrives inside something: a -ZIP, an ISO, a 7z, a Word document, an installer. A scanner's real job is less -"match bytes" than "reach the bytes", and the unpacker is where that happens. +ZIP, an ISO, a 7z, a Word document, an installer. The unpacker is what gets the +scanner to the bytes it has to match. exav's extraction lives in `exav-unpack`, a standalone crate with no dependency on the scanning engine, `#![forbid(unsafe_code)]`, and usable on its own as a safe @@ -17,10 +17,9 @@ If the software on the target's machine can open a container and exav cannot, that container is an evasion vector: the payload lands intact and the scanner said nothing useful about it. -The attacker picks the delivery format, so the target is the union of what real -extractors handle: 7-Zip, WinRAR, WinZip, The Unarchiver, Windows Explorer, macOS -Archive Utility, libarchive. Not the intersection, and not what showed up in last -month's corpus. +An attacker picks the delivery format, so exav targets everything real extractors +open (7-Zip, WinRAR, WinZip, The Unarchiver, Windows Explorer, macOS Archive +Utility, libarchive), not the formats seen in recent samples. [Supported formats](/reference/formats/) tracks this as a matrix: every container and codec those extractors open, exav's status against each, and @@ -28,110 +27,100 @@ and codec those extractors open, exav's status against each, and missing. Every input is extracted through a seekable source: a file, an HTTP range reader -(fetching only the directory and the members it scans), or a stream buffered -first (see [Streaming & memory](/concepts/streaming-memory/)). +(for a format with an index, such as a ZIP's central directory, the walk fetches +only the index and the members it reads), or a stream buffered first (see +[Streaming & memory](/concepts/streaming-memory/)). ## The shape of the unpacker -A member is bytes plus a budget, a container is something that yields members, -and nesting is the same step applied to what comes out. +exav finds the format, then hands each member to the scanner one at a time, all +under one set of limits (the `Budget`). A container is something that yields +members, and nesting is the same step applied to what comes out. - The two doors into extraction - A source is typed by identify(). Formats that stream natively - go through door A, stream_members and dispatch_stream, which holds one member - at a time. Everything else goes through door B, extract_each, which needs the - whole container in memory. Both doors yield a member carrying the same Budget, - and each member re-enters identify(), bounded at max_recursion of 16. One - Budget covers the whole tree, not each container. + One walk into extraction + A source is typed by detect(), then walk() hands its members + to a visitor one at a time. A format whose decoder reads forward yields each + member as a reader that decodes as it is read. A format whose decoder needs + random access is read whole, bounded before it is allocated. Either way a + member carries the same Budget, and each member re-enters detect(), bounded + at --max-unpack-depth of 16. One Budget covers the whole tree, not each + container. - - DOOR A: STREAMING - DOOR B: BUFFERED - - - - + + + source - identify() - stream_members → dispatch_stream - extract_each(fmt, &[u8], …) + detect() + walk(fmt, source, budget, visit) member file · reader · bytes - first 4 KiB - holds ONE member at a time, - the container is never fully resident - zip · tar · 7z · cab · iso · … - needs the WHOLE container as one slice, - bounded before it is allocated - rar · ole · pdf · chm · … - bytes + the SAME Budget + start · end · a search + decoded as it is read: a reader, + nothing resident but what is read + zip · tar · cab · iso · gzip · … + read whole, bounded before it is + allocated, up to --max-object-bytes + rar · 7z · ole · pdf · chm · … + metadata, content, same Budget one Budget for the whole tree, not per container - each member re-enters - identify() · max_recursion (16) + each member re-enters + detect() · --max-unpack-depth (16) - - - - - - + + + + -`identify()` types the bytes, and `streams_natively` decides which door opens. -Door A hands back members one at a time and never has the container resident; -door B needs the whole container as one slice before it can start. Which door a -format uses is a property of its decoder, not of how exav was invoked. - -Past the door both are the same: a member is bytes carrying the same `Budget` as -its parent, and scanning it re-enters `identify()`. So nesting needs no special -case, and the extraction budget bounds an archive tree rather than each container -in it. - -### Where the code departs from the picture - -**There are two dispatches.** `dispatch_stream` and `dispatch_extract` are -separate functions with separate format coverage. A change to one does not -appear in the other, and a format can be reachable through one door only. When a -hook or check does not fire, suspect this first. - -**The `Archive` API has its own split.** `ArchiveInner` has native arms for -`Zip`, `Tar` and `Gzip`, a `Lazy` arm for the single-stream compressors (bzip2, -xz, zstd), and a `Buffered` arm that reads the source into memory and hands it to -door B, which is where every other format sits in that API. - -**An empty `Archive::list()` means "no directory", not "no members".** Only -formats with an index can be listed without extracting: a ZIP's central -directory, a tar's headers. A gzip's single member has no name or size until it -is decompressed, which is why the wasm bindings walk the archive instead of -passing an empty list outward. - -**A ZIP has more members than its directory admits to.** Extraction also scans -the bytes the central directory does not claim (see below). Both doors do this; -a listing reports those members at indices straight after the directory's own. - -## Bounded, in memory, budget before allocation +`detect()` names the format, from the start of the object, its end, or a search +of it. `walk()` then hands the members to a visitor one at a time. A format whose +decoder reads forward (ZIP, tar, CAB, ISO, the single-stream compressors, …) +produces each member as a reader that decodes as it is read, so neither the +container nor the member is resident. A format whose decoder needs random access +over the whole container (7z, RAR, OLE, PDF, CHM, the disk images, …) is read +whole, bounded before it is allocated, up to `--max-object-bytes`; past it the +container is `LIMITS-EXCEEDED`. 7z's members still stream once the container is +in memory. Which it is is a property of the decoder, not of how exav was +invoked; [Supported formats](/reference/formats/#size-what-is-read-as-it-goes-and-what-is-read-whole) +lists both. + +Past the walk both are the same: a member carries the same `Budget` as its +parent, and scanning it re-enters `detect()`. So nesting needs no special case, +and the extraction budget bounds an archive tree rather than each container in +it. + +A member's metadata comes before its content, and a visitor that leaves a member +unread costs nothing for it: a ZIP is listed from its central directory, and a +tar's headers are read in turn with the data between them skipped, so listing +decodes nothing. + +**A ZIP has more members than its directory admits to.** The walk also scans the +bytes the central directory does not claim (see below), after the directory's +own members. + +## Bounded, budget before allocation Every extraction runs under one `Budget`, reserved before a member is read rather than checked after: | Limit | Bounds | |---|---| -| output bytes | total decompressed size across the whole tree | -| per-member size | one member's decompressed size | +| output bytes | what formats decoded whole produce across the tree | +| per-member size | a member decoded whole (a streamed member is bounded by the ratio and matcher budgets) | | compression ratio | output ÷ input, the decompression-bomb guard | | file count | members across the archive and everything nested | | recursion depth | archives inside archives | @@ -141,9 +130,11 @@ than checked after: Reserving first keeps the peak bounded: a "1 GB from 4 KB" member never allocates 1 GB and then gets rejected. -Nothing is written to disk. Members exist in memory only, so zip-slip, path -traversal and symlink attacks do not apply: there is no filesystem write for a -`../../` path to escape into. +`exav-unpack` itself never writes to disk. The scanner may write a member too +large to hold to a temporary file (see +[Streaming & memory](/concepts/streaming-memory/)). That file is created with +`O_EXCL`, mode `0600`, under a name exav chooses, never the member's path, so +zip-slip, path traversal and symlink attacks still have nowhere to write. ## Containers have two indexes @@ -188,6 +179,11 @@ and the missing tail is absent rather than hidden. Indexed formats are different a damaged ZIP index can leave member data present but never enumerated, which is why dual indexing exists. +A stream damaged part way is different: the compressed bytes after the damage +are present but were never decoded, so a clean prefix is `UNSCANNABLE`. A +checksum that fails after a full decode is not damage of that kind: everything +was decoded and scanned. + ## A member exav cannot read is reported When extraction cannot produce a member's content (an unsupported codec, @@ -225,19 +221,22 @@ turns it into a detection. An encrypted ZIP appended to a picture or document The extractor's entire input is attacker-controlled, so: -- `#![forbid(unsafe_code)]`, compiler-enforced, across every decoder. +- `#![forbid(unsafe_code)]`, compiler-enforced, across exav's own decoders (for + third-party ones, see [Dependencies](/reference/dependencies/)). - Release builds enable `overflow-checks`, so a wrapping size calculation traps instead of bypassing a limit. -- Each file is scanned under `catch_unwind`; a parser panic on one input is an - error for that file, not an aborted batch. +- Each container walk and each file runs under `catch_unwind`: a decoder panic + makes that container `UNSCANNABLE`, and a panic elsewhere is `ERROR` for that + file, not an aborted batch. - The decoders are tested on `wasm32`, where `usize` is 32-bit and overflows on parsed offsets that a 64-bit host would hide show up. It is also the build the [WASM sandbox](/guides/wasm-sandbox/) ships. - A read error from the source is reported, never taken for the end of the container. -- Checksums are not enforced by default. A wrong CRC is a corrupt archive, not a - reason to stop scanning; making it fatal would give an attacker a one-byte way - to make a member unscannable. +- Most formats scan a member whose checksum fails: a wrong CRC is a corrupt + archive, not a reason to stop scanning. RAR, WIM, ARC and EGG, where the + checksum is the only check on a decoder, report such a member `UNSCANNABLE` + instead; ZOO scans it and flags the mismatch. ## Validating a decoder @@ -259,8 +258,9 @@ So each decoder gets an external oracle: | LZ4 | `lz4 -dc` | | ARC | `arc` 5.21q and `nomarch` | | Office OOXML/XLS crypto | msoffcrypto-tool reference vectors | -| ZIP PPMd | the `ppmd-rust` encoder | -| ARJ | a real archive, mutated to produce each failure case | +| ZIP PPMd | the `ppmd-rust` encoder, an implementation independent of exav's decoder | +| bzip2 | the `bzip2` crate's encoder, a port of libbzip2 | +| ARJ | a real archive, and mutations of it for each failure case | Where the format carries one, the integrity check does the same job from inside the data: RAR CRC-32, WIM SHA-1, ARC CRC-16, UPX Adler-32, 7z AES CRC. A subtly diff --git a/www/src/content/docs/concepts/bytecode-sandbox.md b/www/src/content/docs/concepts/bytecode-sandbox.md index 94c6b51a..80c3b654 100644 --- a/www/src/content/docs/concepts/bytecode-sandbox.md +++ b/www/src/content/docs/concepts/bytecode-sandbox.md @@ -8,7 +8,8 @@ C, compiled to a custom VM bytecode, shipped as a `.cbc` file in `bytecode.cvd`) that runs against a candidate file and decides whether it is malicious. It expresses logic that pattern and logical signatures cannot, most usefully a handful of unpackers that deobfuscate packed PEs so the other signatures can -match the payload. +match the payload. What an unpacker produces is scanned like an archive member: +through the whole pipeline, under the same depth and scan budgets. exav implements a reader and a memory-safe, sandboxed interpreter for this format. The live database holds few programs, so the point is less extra @@ -18,7 +19,8 @@ ClamAV's bytecode subsystem. ## The format, in brief A `.cbc` file is line-oriented: a `ClamBC` header, a trigger line (a `.ldb` -logical signature; the program runs only when it matches), and records for types, +logical signature; the program runs only when it matches, or on every file of +one type for a program that hooks it instead), and records for types, API declarations, globals, function headers, basic-block instruction streams and strings. The instruction set is an SSA form close to LLVM IR. Programs cannot make syscalls; they reach the outside world only through a fixed host API of 107 @@ -30,7 +32,8 @@ hit. The trigger side comes for free with exav's `.ldb` engine. ClamAV's bytecode subsystem has a documented code-execution history, because it runs database-supplied programs in memory-unsafe C and, historically, through an -LLVM JIT: +LLVM JIT. An attacker needs to get a `.cbc` file loaded, from a third-party feed +for instance: - **[CVE-2020-37167](https://www.cve.org/CVERecord?id=CVE-2020-37167):** weak validation in the interpreter's function-name processing (CWE-94) lets a @@ -50,23 +53,27 @@ exav removes these failure modes by construction: | Native code generation from bytecode | No JIT: interpretation only | | A program escaping the sandbox (syscalls, host memory) | The program sees only bounded buffers and a fixed read-only file API: no syscalls, no host pointers | | A runaway program | An instruction budget, a scratch-memory cap and a call-depth limit | -| A VM bug taking down the scan | `catch_unwind` per program and per file: a bad program is skipped and the scan continues | +| A VM bug taking down the scan | `catch_unwind` per program: a bad program is stopped and the scan continues | | A malformed or hostile `.cbc` | A fallible parser with no panics; a program not fully understood is never executed | -The worst a hostile or buggy `.cbc` can do to exav is be skipped or time out. For -anyone who disables ClamAV bytecode for safety, exav offers the capability -without that trade-off. +The worst a hostile or buggy `.cbc` can do to exav is be skipped or be stopped. +A program stopped before it finished (out of instructions, or a VM panic) makes +the scan `LIMITS-EXCEEDED` unless something is found, since what it would have +found is unknown. exav can run these programs with the code-execution risk +removed, for anyone who disables ClamAV bytecode for safety. ## Trigger-gated execution, and the API subset Execution is live in the normal scan path and gated per program: a program runs -only when its logical-signature trigger matches. +only when its logical-signature trigger matches. The few programs with no +trigger (two PDF hooks in the current `bytecode.cvd`) run on every file of the +type they hook. -exav implements part of the host API. The rest are fail-safe stubs rather than a -reason to skip a program: a call returns a conservative value and is recorded, so -the program still runs to the end. A stub that fabricates a result is recorded in -the run's outcome (and warned about under `EXAV_BC_WARN`), so a verdict that -rests partly on a stub is distinguishable from one that does not. On a live +exav implements about 40 of the 107 host API functions. The rest are fail-safe +stubs rather than a reason to skip a program: a call returns a conservative value +and is recorded, so the program still runs to the end. A stub that fabricates a +result is recorded in the run's outcome and, with the `EXAV_BC_WARN` +environment variable set, printed to stderr. On a live `bytecode.cvd`, every program parses and runs to completion with no unsupported opcode, and in practice programs rarely reach a stub: a `.cbc` header declares a `maxapi` ceiling, not a call list. The per-API breakdown is in the repository's diff --git a/www/src/content/docs/concepts/design-principles.md b/www/src/content/docs/concepts/design-principles.md index 9516c3db..1d489958 100644 --- a/www/src/content/docs/concepts/design-principles.md +++ b/www/src/content/docs/concepts/design-principles.md @@ -1,11 +1,11 @@ --- title: Design principles -description: The principles exav is built around, from never a silent clean and memory safety to no runtime code generation, bounded work, drop-in compatibility and clean-room licensing. +description: The principles exav is built around, from never a silent clean and memory safety to no runtime code generation, bounded work, work done at database build time, drop-in compatibility and clean-room licensing. --- exav is built around a handful of principles. Some protect the host, some protect -the verdict, and some make exav practical to adopt. One ties them together: every -bound the engine hits is reported. +the verdict, and some make exav practical to adopt. One ties them together: a +scan that stopped short is reported as such. ## Never a silent clean @@ -18,9 +18,10 @@ that makes the scanner stop early, for any reason, is reported instead of being folded into `OK`. ClamAV, for example, reads a file over about 2 GB, scans none of it, and reports -`OK`. exav closes that gap and every other way a scan can stop early. The rule -holds in every mode, including `--clamav-compat`, which changes the names exav -reports and what it opens, but not whether an incomplete scan is reported. +`OK`. exav closes that gap and every other way a scan can stop early. The one +exception is asked for by name: `--clamav-compat`, a preset for differential +testing, answers an incomplete scan `OK` as ClamAV does, and logs every such +object on stderr so the pass stays visible. Three rules follow: @@ -32,8 +33,9 @@ Three rules follow: `LIMITS-EXCEEDED`. Otherwise `cat malware huge.pad > evil` would be a bypass. A file, stdin and every daemon verb go through that one scan. 3. **Not fully scanned is never clean, whatever the cause.** That covers external - limits and exav's own work bounds: a verification-step budget that stops a - pathological wildcard search, an emulator that runs out of steps. + limits and exav's own work bounds: the budget that stops a pathological + wildcard search, a PE emulation that runs out of instructions + (`--max-pe-emulation-steps`), a bytecode program that runs out of steps. A scan that did not complete resolves to `LIMITS-EXCEEDED`, `UNSCANNABLE` or `PASSWORD-PROTECTED`. See [Verdicts & exit codes](/reference/verdicts/) for what @@ -42,11 +44,11 @@ each means and how it is reported. ### Stricter than ClamAV `clamscan` returns `OK` after bounding its own work (its `alert-exceeds-max` and -`alert-encrypted` heuristics are off by default); exav does not. It is the one -behavioral difference a migrating user has to plan for (see -[Migrating from ClamAV](/guides/migrating-from-clamav/)). `--clamav-compat` -matches ClamAV's limit values for differential testing, and still reports the -outcome. +`alert-encrypted` heuristics are off by default); exav does not. It is the main +behavioral difference a migrating user has to plan for, along with the flags +(see [Migrating from ClamAV](/guides/migrating-from-clamav/)). `--clamav-compat` +matches ClamAV's limit values and its `OK` for incomplete scans, for +differential testing, and logs each such object. ### The verdict is not the policy @@ -57,9 +59,9 @@ rather than reject an upload. Where a deployment chooses that, with [`--partial-as ok`](/reference/cli/#what-an-unscannable-object-becomes), the -choice is named per verdict, announced at startup, logged per object, and still -reported in the response. The rule does not forbid accepting a risk; it forbids -hiding one. +choice is named per verdict and logged per object on stderr; the ICAP service +also announces it at startup and keeps `X-Exav-Status: PARTIAL` in its response. +The rule does not forbid accepting a risk; it forbids hiding one. ### Refusing empty coverage @@ -92,10 +94,21 @@ implemented from public specifications and permissively licensed Every decode and match is bounded by size or by a step count, so hostile input cannot turn a scan into an unbounded computation or allocation; the prefork -daemon adds a per-job wall-clock and CPU limit on top. Archives are walked member -by member, and every bound is a flag. Reaching one is reported, never a silent -truncation. See [Streaming & memory](/concepts/streaming-memory/) and -[Limits](/reference/limits/). +daemon adds a per-job wall-clock and CPU limit on top. Most archives are walked +member by member, and the main bounds are flags. Reaching a bound on the scan is +reported, never a silent truncation. See +[Streaming & memory](/concepts/streaming-memory/) and [Limits](/reference/limits/). + +## Work at build time + +> Whatever can be computed once from the signatures is computed when the +> database is built, not when it is loaded or when a file is scanned. + +A [prebuilt `.exavdb`](/guides/prebuilt-database/) is built once and loaded by +every CLI run, daemon start, reload and worker. So the build does the expensive +work (automatons, indexes, tables derived from the signatures) and stores the +result in a form that loads with little more than a copy. Load time and scan +speed come first; a larger file is an acceptable price. ## Minimal dependencies @@ -108,14 +121,8 @@ listed on the [Dependencies](/reference/dependencies/) page. ## Drop-in compatibility exav loads the signature databases you already have, speaks the `clamd` wire -protocol, and prints `clamscan`'s output format and exit codes, so it fits into +protocol, and prints `clamscan`'s output format and exit codes (plus exit code 3 +for `PARTIAL`), so it fits into an existing deployment without rewriting tooling. Its flags are its own. See [Comparison with ClamAV](/project/comparison-with-clamav/) and [Migrating from ClamAV](/guides/migrating-from-clamav/). - -## How they fit together - -Memory safety and no runtime code generation protect the host running exav. -Never a silent clean and bounded work protect the verdict. Minimal dependencies -and drop-in compatibility make exav practical to run and adopt. Clean-room -licensing keeps it permissively licensed and independently built. diff --git a/www/src/content/docs/concepts/differential-testing.md b/www/src/content/docs/concepts/differential-testing.md index e88e45c1..a6268219 100644 --- a/www/src/content/docs/concepts/differential-testing.md +++ b/www/src/content/docs/concepts/differential-testing.md @@ -12,13 +12,15 @@ It is a compliance harness, not a benchmark. It records timings, which help spot a wedged file or a much slower engine, but they are taken under concurrency against a cold page cache and are not performance numbers. -The harness lives in the repository's `scripts/`; this page is the concept, not -the runbook. +The harness lives in the repository's `scripts/`; this page is the concept. The +runbook is +[`docs/DIFF_TESTING.md`](https://github.com/sylvinus/exav/blob/main/docs/DIFF_TESTING.md), +and the smallest run is `LIMIT=500 scripts/difftest.sh`. ## The setup -- **Corpus:** live malware, scanned statically and never executed (not in the - repository). +- **Corpus:** live malware from MalwareBazaar, scanned statically and never + executed (not in the repository). - **Same database:** both engines load the same signature set (`daily.cvd` only, or the full `main` + `daily`), or the comparison means nothing. - **Same client:** one program drives both daemons, so a difference cannot come @@ -39,12 +41,13 @@ Results are bucketed, because "they disagreed" is rarely useful on its own: | Bucket | Meaning | |---|---| | `AGREE` / `clean` | identical signature sets, or both clean: the headline metric | -| `PARTIAL` | the sets overlap: same malware, one engine also named more | +| `PARTIAL` | the sets overlap: same malware, one engine also named more (not the `PARTIAL` verdict) | | `NAMEDIFF` | the sets are disjoint: a real disagreement about what this is | | `FN` | clamscan detected, exav did not: a real gap | | `EXAV_ONLY` | exav detected, clamscan said clean: verify before assuming a false positive | | `CAREFUL` | exav reported not fully scanned; clamscan said OK | | `CAREFUL_FN` | clamscan detected; exav reported not fully scanned | +| `CLAM_GAVE_UP` | clamscan hit its own scan limit; exav completed the scan | | `ERROR` | a scan failed, so there is nothing to compare | `EXAV_ONLY` is not called "false positive" because it has mostly been hits inside @@ -59,10 +62,11 @@ sample. Agreement is what matters. ## `--clamav-compat`: matching boundaries, not quirks -By default exav runs at full capability. For a like-for-like run, +By default exav runs at full capability, and the harness runs it with +`--clamav-compat` (`COMPAT=0` measures what exav adds). For a like-for-like run, `--clamav-compat` sets a stock ClamAV build's documented defaults: the limit -values (`--max-input-bytes 100M`, `--max-extracted-bytes 400M`, -`--max-unpack-depth 17`, `--max-members 10000`, `--decode none`), the extractor +values (`--max-input-bytes 100M`, 400M held per file, `--max-unpack-depth 17`, +`--max-members 10000`, `--decode none`), the extractor set, `.UNOFFICIAL` naming for unofficial-database signatures, and `--partial-as ok`, so a file ClamAV would call clean is answered `OK` here too. Each limit is also its own flag, and an explicit flag wins over the preset. diff --git a/www/src/content/docs/concepts/pe-emulation.md b/www/src/content/docs/concepts/pe-emulation.md index 529f8deb..8d38acca 100644 --- a/www/src/content/docs/concepts/pe-emulation.md +++ b/www/src/content/docs/concepts/pe-emulation.md @@ -9,10 +9,9 @@ memory at start-up. On disk there is nothing for a signature to match: the original code is there, but not in a form anything can read. The usual answer is a decoder per packer. exav has those for the common cases -(UPX, the aPLib family, MPRESS), and they are the fast path. But a decoder per -packer is a race the scanner cannot win: the packer's author picks the format, -and the packers that matter vary it per build to break tools that assume a -layout. +(UPX, and the aPLib family: Petite, FSG, NsPack), and they are the fast path. But +a decoder per packer cannot keep up: the packer's author picks the format, and +the packers that matter vary it per build to break tools that assume a layout. So exav also does the general thing. Whatever a packer compresses with, its stub must rebuild the original image in memory and jump to it. exav runs the stub and @@ -30,8 +29,9 @@ A bounded x86-32 interpreter, in-process, with: Instruction boundaries are where a decoder goes quietly wrong, so the decoder lives in a crate that is checked instruction by instruction against an independent decoder over the sample corpus. -* **Enough Windows to be believed.** A TEB and PEB, the loader's three module - lists, and synthetic `kernel32`/`ntdll`/`user32` images with real export +* **The Windows a stub looks for.** A TEB and PEB, the loader's three module + lists, and synthetic `kernel32`/`ntdll`/`user32`/`advapi32`/`msvcrt` images + with real export directories. Both ways a stub resolves imports work: calling `GetProcAddress`, and walking `fs:[0x30]` → PEB → loader list → export table by hand. The loader's own import binding is emulated too, because a packed @@ -81,9 +81,9 @@ and nested content in the recovered image are reached too. ## When the stub wins -Some stubs will not run to completion. The run stops and says why: an -instruction outside the implemented set, an export with no implementation (both -named), a fault nothing handled, or a budget. +Some stubs will not run to completion. The run stops on an instruction outside +the implemented set, an export with no implementation, a fault nothing handled, +or a budget; the `exav-pe-emu` triage tool names which. * **Nothing is fabricated.** A dump is emitted only when it reads back as a valid PE; a wrong guess is discarded rather than handed to the matcher. @@ -95,9 +95,11 @@ named), a fault nothing handled, or a budget. identified packer exav could not unpack into a `Heuristics.Packed.*` detection. * **Running out of budget is a limit.** The emulator has an instruction budget - per stub, and `--max-pe-emulation-steps` (1,000,000,000 by default) bounds the - total across every packed executable in one file. A scan that reaches it is - `LIMITS-EXCEEDED`. + of 120 million per stub and a 32 MiB dump cap, and + `--max-pe-emulation-steps` (1,000,000,000 by default) bounds the total across + every packed executable in one file. A scan that reaches the total is + `LIMITS-EXCEEDED`. A stub stopped by its own budget alone is treated like any + other stub that would not finish: `UNSCANNABLE` if its packer was identified. A run that stopped part-way with a substantial part of the image rebuilt is reported as a partial reconstruction: worth scanning, but not named as the @@ -108,13 +110,14 @@ original program. Virtualizing protectors (VMProtect, Themida/WinLicense, Enigma) translate the protected functions into a private bytecode when the file is built, so the original instructions never exist in memory, and no emulator can recover them. -exav does not spend the budget trying; it reports them as what they are. +exav does not spend the budget trying: it reports them `UNSCANNABLE`, and with +`--detect packed` as `Heuristics.Packed.VMProtect` and the like. ## Measured coverage Against 276 samples from 23 real packers, 12 each (ordinary Windows utilities, packed; fetched by `scripts/fetch-packed-pe.py` and checked by the corpus test in -`exav-unpack`): +`exav-unpack`; the names are the corpus's): | | Packers | |---|---| @@ -123,10 +126,11 @@ packed; fetched by `scripts/fetch-packed-pe.py` and checked by the corpus test i | **Payload recovered, image incomplete** | Amber (a .NET packer: the native stub hands the assembly to the runtime, and that assembly comes back, on 11 of 12 samples) | | **Nothing recovered**: the stub defended itself | Alienyze, TELock, Yoda-Protector | -Recovery is all-or-nothing per sample: every sample that recovered anything -recovered an image at least as large as the input. Four of the fully unpacked -packers are ones ClamAV ships a hand-written unpacker for (ASPack, MEW, NSPack, -Yoda's Cryptor); most of the rest have no decoder anywhere. +Apart from Amber, recovery is all-or-nothing per sample: every sample that +recovered anything recovered an image at least as large as the input. Seven of +the fully unpacked packers have a hand-written unpacker in ClamAV (ASPack, FSG, +MEW, NSPack, UPX, WinUpack, Yoda-Crypter), plus PEtite among the partial ones; +most of the rest have no decoder anywhere. The measure is the size of what came back, not whether the run reached the original entry point. Several packers hand control to the unpacked program in a diff --git a/www/src/content/docs/concepts/quirks.md b/www/src/content/docs/concepts/quirks.md index f1986b22..980b011c 100644 --- a/www/src/content/docs/concepts/quirks.md +++ b/www/src/content/docs/concepts/quirks.md @@ -80,15 +80,14 @@ See [Bytecode sandbox](/concepts/bytecode-sandbox/). ## The file extension is a lie, and so is the magic number File type comes from content only. Signatures are scoped by type (`Target:` and -`CL_TYPE_*` container constraints), so getting the type wrong does not just skip -a parser, it deselects a whole class of signatures. And magic bytes are not -enough either: +`CL_TYPE_*` container constraints), so getting the type wrong deselects a whole +class of signatures, not only a parser. And magic bytes are not enough either: - **`CA FE BA BE` is two formats:** the Mach-O universal ("fat") binary magic and the Java `.class` magic. exav claims a fat binary only when the architecture count is between 1 and 64, the whole table is present and every slice lies - within the file; `.class` detection also requires a plausible major version - (45 or more), which a fat header's architecture count never is. + within the file; `.class` is tried only after the fat check fails, and also + requires a major version of 45 or more. - **Short magics collide with ordinary data.** bzip2 is `BZh`, CAB is `MSCF`, gzip is two bytes. A false hit inside a PE overlay would be routed to that decoder, fail, and report the whole object `UNSCANNABLE`, so each weak magic is @@ -122,7 +121,8 @@ over about 2 GB is read, scanned as zero bytes, and reported `OK`. exav's rule is that refusing by size still scans. An input over `--max-input-bytes` has its first `--max-input-bytes` bytes scanned first, as -any smaller input would be, and only then gets a limit verdict: +any smaller input would be, and only then gets a limit verdict (here with +`--max-input-bytes 100M`): ```text file size 5368709120 exceeds max-input-bytes 104857600; scanned first 104857600 bytes only @@ -198,7 +198,8 @@ That is a double-extension detector: it catches Two more places encryption is a signal: - `--partial-as password-protected=found` turns a password-protected member into - a detection, `Heuristics.Encrypted.Zip` / `.RAR` / `.7Zip` / `.PDF` / `.Doc`. + a detection, `Heuristics.Encrypted.Zip` / `.RAR` / `.7Zip` / `.PDF` / `.OLE2`, + or `.Archive` for other formats. - exav tries a default password list on ZIPs: `infected`, `virus`, `malware`, `password`, `123456`. "infected" is the standard password for sharing samples, so a password-protected dropper opens with no configuration, as @@ -211,8 +212,10 @@ This reads like a bug in every code review: exav extracts archive members withou verifying their CRCs, and verification is off by default. A wrong CRC must never stop a member's bytes from being scanned, or an attacker could downgrade a detection by flipping one checksum byte. (ClamAV also ignores CRCs when scanning. -Verification needs the `checksums` Cargo feature and an explicit opt-in, for -extraction where a bad CRC is a real "corrupt file" signal.) +Verification is for a library embedding extracting files, where a bad CRC is a +real "corrupt file" signal: it needs the `checksums` feature of `exav-core` or +`exav-unpack` and `ScanOptions::verify_checksums`. The command line has no +switch for it.) The same instinct generalises. A CAB whose total-size field is overwritten with `0xFFFFFFFF` defeats a strict parser; exav clamps it and extracts the member. A @@ -228,7 +231,9 @@ still reported, not treated as clean. > When a sequential stream runs out of input, every byte that exists has been > scanned: the missing tail is absent, not hidden, so a clean result is a real > clean. The incomplete verdicts are for content that is present but unscanned: -> an encrypted member, an unsupported codec, a member over a limit. +> an encrypted member, an unsupported codec, a member over a limit. A stream +> damaged part way is one of those: the compressed data after the damage is +> present and was not decoded, so a clean result there is `UNSCANNABLE`. ## Identity that survives the bytes changing @@ -340,7 +345,7 @@ rendering byte for byte: HTML entity decoding, lowercasing and whitespace collapsing; quote-aware comment stripping; and, for scripts, a JavaScript normaliser that decodes string escapes, folds `"ab"+"cd"`, evaluates `String.fromCharCode()` and `unescape("…%XX…")`, and re-parses the -argument of `eval("…")` for one static layer. Nothing is executed. A single +argument of `eval("…")`, up to 32 nested layers. Nothing is executed. A single textual buffer can be scanned up to five times: raw, HTML-normalised, text-normalised, and through the light and heavy JavaScript normalisation. @@ -401,8 +406,9 @@ The `clamd` protocol itself has oddities: - **One command can answer any number of times, and nothing marks the last one.** `SCAN` over a directory, `CONTSCAN` and `ALLMATCHSCAN` reply once per file with no count and no terminator. Outside a session the closed connection is the end - marker; inside `IDSESSION` there is none, so exav's client sends a `PING` after - each scan and reads up to the `PONG`. A client that reads one line per command + marker. clamd refuses `CONTSCAN` and `ALLMATCHSCAN` inside `IDSESSION`, but + takes `SCAN` on a directory, and there nothing marks the end, so exav's client + sends a `PING` after each scan and reads up to the `PONG`. A client that reads one line per command loses a detection and attributes the leftovers to the next command. - **`INSTREAM` is `` chunks ended by a zero length:** the only binary field in a text protocol. When exav has its answer before the stream @@ -416,8 +422,8 @@ The `clamd` protocol itself has oddities: ## Smaller ones - **`(B)`, `(L)` and `(W)` are not alternations.** In `.ndb` bodies they look - exactly like one-option alternations and are boundary markers (word, line, - word). A negated alternation `!(aa|bb)` also requires every branch to be the + exactly like one-option alternations and are boundary markers (word boundary, + line boundary, non-alphanumeric byte). A negated alternation `!(aa|bb)` also requires every branch to be the same length, or the signature is rejected. - **Archive member names reach your terminal.** A hostile archive can put ANSI escapes (or megabytes of text) in a member name that ends up in the scanner's diff --git a/www/src/content/docs/concepts/streaming-memory.md b/www/src/content/docs/concepts/streaming-memory.md index 4cc28593..9ff4f72f 100644 --- a/www/src/content/docs/concepts/streaming-memory.md +++ b/www/src/content/docs/concepts/streaming-memory.md @@ -1,14 +1,10 @@ --- title: Streaming & memory -description: Where exav holds a file in memory, where it streams, what each path matches, and where the memory really goes. +description: Where exav holds a file in memory, where it streams, what each path matches, and why the database, not the file, is what uses memory. --- -exav matches a file in one of two ways, chosen by size. - -## Up to `--max-object-bytes`: the full engine, in memory - -A file up to `--max-object-bytes` (256 MiB by default) is read into memory and -gets the full engine: +Every file gets the same scan whatever its size, except the few checks that +parse it whole (below): - every `.ndb` form: wildcards, gaps, anchored offsets, file-type targets; - `.ldb` logical signatures, with their regexes and byte comparisons; @@ -17,33 +13,60 @@ gets the full engine: - YARA rules, bytecode programs, PE section hashes, whole-file hashes; - structural analysis: unpacking, embedded files, heuristics. -Verifying a wildcard or logical signature means looking back and forth around -each candidate match, which is why this path holds the file. Memory is roughly -the file's size, plus a lowercase copy for case-insensitive signatures and, for -text, one normalised copy at a time. The deobfuscated JavaScript view stops at -32 MiB; a script whose view is longer is reported `LIMITS-EXCEEDED` unless -something is found, though its raw bytes are still scanned in full. - -Archives in a streamable format (ZIP, tar, 7z, CAB, gzip, ISO and UDF images, -self-extracting executables and most others; see -[archive extraction](/concepts/archive-extraction/)) are walked member by member -at any size, and each member is treated like a file: in memory up to the same -limit. - -## Past it: the streaming core, and `LIMITS-EXCEEDED` - -A larger file, or a larger member, goes through the streaming core instead. It -reads fixed-size buffers in one forward pass and runs: - -- an Aho-Corasick automaton over the literal `.ndb` signatures (fixed bytes, any - file type, any offset), carrying state across buffer boundaries; -- MD5, SHA1 and SHA256 over the whole input, for `.hdb`/`.hsb`. - -Its memory does not grow with the input. But it covers only those two kinds of -signature, so a file scanned this way is reported `LIMITS-EXCEEDED` unless one of -them matches. Raise `--max-object-bytes` to give larger files the full engine, at -the cost of memory. A full engine that streams is on the -[roadmap](/project/roadmap/#a-streaming-full-engine). +What changes with size is where the file is while that runs. + +## Up to `--max-object-bytes`: in memory + +A file up to `--max-object-bytes` (256 MiB by default) is read into memory. +Memory is roughly the file's size, plus a lowercase copy for case-insensitive +signatures and, for text, one normalised copy at a time. The deobfuscated +JavaScript view stops at 32 MiB; a script whose view is longer is reported +`LIMITS-EXCEEDED` unless something is found, though its raw bytes are still +scanned in full. + +## Past it: through a block cache + +A larger file is read through a cache of 64 KiB blocks, of which at most 8 MiB +are held, least recently used dropped first. So are an HTTP range source and a +spill file (below). Verifying a match can still look anywhere in the file: the +cache bounds what is held at once, not how far a check may reach, and a block +that was dropped is read again when it is needed. The scan is the same one, with +the same results, only slower, since the file is read a few times over: once for +what format detection searches it for, what carving looks for and its digests +(when a hash signature has its size), once for the signature sweep (every +automaton at once), and again in the parts a check needs. +[Architecture](/concepts/architecture/#reading-an-object-not-held-in-memory) +shows what reads through the cache. + +A few things parse a file whole and are not done past the limit: + +- the structure of a PE: entry-point and section offsets, section hashes, + imports, icons, the Authenticode signature, UPX and packer unpacking; +- YARA's `pe`, `elf` and `dotnet` modules; +- containers whose format is read whole (7z, RAR, OLE, PDF, the virtual-disk and + filesystem images and others; see + [Supported formats](/reference/formats/#size-what-is-read-as-it-goes-and-what-is-read-whole)). + +A file for which one of these applied is reported `LIMITS-EXCEEDED`, naming +`--max-object-bytes`, unless something is found. Raise the limit to have them +run, at the cost of memory. + +The normalised views of a large text file are as large as the file, so they are +written to the same temporary files as a buffered stream (below), under the same +budgets. With `--spill-dir off` they are skipped and the file is +`LIMITS-EXCEEDED`. + +## Archive members + +Archives decoded as they are read (ZIP, tar, CAB, ISO/UDF, DMG, LHA, `ar`, cpio, +the single-stream compressors, self-extracting executables) are walked member by +member at any size and any depth. Such a member is held in memory up to the same +limit, and past it written to a temporary file and scanned from there. With +`--spill-dir off`, or past `--max-spill-bytes`, it is not scanned and the file is +`LIMITS-EXCEEDED`. 7z members stream the same way once the container is read. + +A member that has to be decoded whole (an encrypted ZIP member, a member of a +format read whole) is `LIMITS-EXCEEDED` past `--max-object-bytes`. ## Stdin, `INSTREAM` and ICAP bodies @@ -56,7 +79,7 @@ buffered stream then goes through the same scan as a file. A stream past `--max-input-bytes` is. See [buffering a stream](/reference/cli/#buffering-a-stream-spill). -## Where the memory really goes: the database +## The database, not the file, is what uses memory Loading raw ClamAV databases builds a double-array Aho-Corasick automaton, and for the full `main` + `daily` set that construction needs several GB for a short @@ -73,13 +96,15 @@ construction. Memory and CPU budgets are separate flags: - **`--max-object-bytes`** (256M): the most memory one object may use (a file, a - decompressed member, an LZ window, a decrypted blob), and the largest file the - full engine scans. Several such buffers are alive at once across nesting - levels, so it does not bound the total. -- **`--max-extracted-bytes`** (1G): what decompression may produce across one - top-level file, charged cumulatively. Under the daemon it is clamped to fit the - per-job address space, so reaching it is reported as a limit instead of the - worker being killed. + decompressed member, an LZ window, a decrypted blob); past it an object is read + through the block cache or a temporary file. Several such buffers are alive at + once across nesting levels, so it does not bound the total. +- **`--max-process-bytes`** (2G per worker in the pool): the memory a scan may + use. What formats decoded whole hold across one top-level file is charged + cumulatively against 1G, lowered to half of `--max-process-bytes`, so reaching + it is reported as a limit instead of the scan being killed. Streamed members + are not charged there: they are bounded by `--max-object-bytes` and the spill + budgets. - **`--max-matcher-bytes`** (10G): the most bytes fed to the matcher across one top-level file. A CPU bound, not a memory one. - **`--max-pe-emulation-steps`** (1,000,000,000): the instructions the PE diff --git a/www/src/content/docs/getting-started/installation.mdx b/www/src/content/docs/getting-started/installation.mdx index ee37b355..97d560f2 100644 --- a/www/src/content/docs/getting-started/installation.mdx +++ b/www/src/content/docs/getting-started/installation.mdx @@ -3,24 +3,36 @@ title: Installation description: Install exav with cargo, from a prebuilt binary, as a package, as a container, or build it from source. --- -import { Tabs, TabItem, Steps, Aside } from '@astrojs/starlight/components'; +import { Steps } from '@astrojs/starlight/components'; exav runs on Linux, macOS and Windows and ships as a single binary. The Linux release builds are static (musl); the macOS and Windows builds link the system -libraries. +libraries. On Windows there is no prefork worker pool and no `--auto-update`, +which are Unix only. + +## Prebuilt binaries + +Each release on [GitHub](https://github.com/sylvinus/exav/releases) carries +binaries for `{x86_64,aarch64}-unknown-linux-musl` (static, any distro), +`{x86_64,aarch64}-apple-darwin` and `x86_64-pc-windows-msvc`, and the WASI +module `exav-wasm--wasm32-wasip1.wasm` (see the +[WASM sandbox guide](/guides/wasm-sandbox/)). They are the default build, +without HTTP support: to fetch signatures with `--auto-update` or scan URLs, +build with `--features http` below. ## From crates.io +Building needs Rust 1.91 or newer (`rustc --version`; `rustup update` to +upgrade). + ```sh cargo install exav ``` That installs the `exav` binary with the default pure-Rust build, which handles -every supported archive format. The library crates it is built from (`exav-*`) -are not needed to install it. - -For a build with [HTTP support](/reference/feature-flags/#adding-http-support) -(URL scanning and signature updates over HTTP(S)): +every supported archive format. For a build with +[HTTP support](/reference/feature-flags/#adding-http-support) (URL scanning and +signature updates over HTTP(S)): ```sh cargo install exav --features http @@ -28,11 +40,6 @@ cargo install exav --features http See [Feature flags](/reference/feature-flags/) for all build options. - - ## Build from source @@ -56,15 +63,14 @@ If the build fails on syntax, check `rustc --version` first. The workspace -The default build handles every supported archive format and is already a -pure-Rust binary with no TLS stack. To add [HTTP(S) support](/reference/feature-flags/#adding-http-support) -(URL scanning and signature auto-update): +To add [HTTP(S) support](/reference/feature-flags/#adding-http-support) (URL +scanning and signature auto-update): ```sh cargo build --release -p exav --features http ``` -For an even smaller binary with fewer capabilities: +For a smaller binary without YARA, DLP, the phishing heuristic or ICAP: ```sh cargo build --release -p exav --no-default-features --features all-formats,decrypt @@ -73,40 +79,39 @@ cargo build --release -p exav --no-default-features --features all-formats,decry See [Feature flags](/reference/feature-flags/) for the full list of build-time options. -## Packages & release artifacts +## Debian / Ubuntu package + +Build a `.deb` locally with `cargo-deb`: -CI produces these on each `v*` tag: +```sh +cargo install cargo-deb +cargo build --release -p exav +cargo deb -p exav --no-build # the binary, and the systemd unit exav.service +``` - - - Attached to the GitHub Release for `{x86_64,aarch64}-linux-musl` (static, - any distro), `{x86_64,aarch64}-apple-darwin` and `x86_64-windows`. - - - ```sh - cargo install cargo-deb - cargo build --release -p exav - cargo deb -p exav --no-build # binary + optional systemd unit - ``` - - +The unit runs as the `clamav` user and reads `/var/lib/clamav`, as +`clamav-daemon` does; the package creates neither, so install it on a machine +that has ClamAV's (see [Migrating from ClamAV](/guides/migrating-from-clamav/#as-a-managed-service)). ## Container The published image is a wire-level replacement for the ClamAV Docker image: the -same clamd port 3310 and protocol. +same clamd port 3310 and protocol. It fetches signatures itself from a source you +name, such as a `cvdupdate` mirror of Cisco's database: ```sh -docker run -d -p 3310:3310 -v exav-db:/var/lib/exav ghcr.io/sylvinus/exav +docker run -d -p 3310:3310 -v exav-db:/var/lib/exav \ + -e EXAV_SIG_SOURCES=https://clamav-mirror.internal/ ghcr.io/sylvinus/exav ``` -See the [Docker guide](/guides/docker/) for signature population, environment -variables, and the dual-container updater pattern. +Without a source it waits for another container to fill the volume. See the +[Docker guide](/guides/docker/) for signature population, environment variables, +and the dual-container updater pattern. ## WASM sandbox module -To run the scanner sandboxed (for untrusted signatures), build the -`wasm32-wasip1` module: +To run the scanner sandboxed (for untrusted signatures), take the release's +`.wasm` or build the `wasm32-wasip1` module: ```sh rustup target add wasm32-wasip1 diff --git a/www/src/content/docs/getting-started/introduction.md b/www/src/content/docs/getting-started/introduction.md index 6e1d6a23..98f83a2c 100644 --- a/www/src/content/docs/getting-started/introduction.md +++ b/www/src/content/docs/getting-started/introduction.md @@ -38,21 +38,21 @@ $ exav -d ~/.cvdupdate/database suspicious.bin suspicious.bin: Win.Trojan.Agent-1234 FOUND ``` -A stream on stdin is scanned like a file. It is held in memory up to -`--spill-threshold-bytes` (16 MiB) and in a temporary file past that, up to -`--max-spill-bytes` (2 GiB): +A stream on stdin is scanned like a file: ```console -$ curl -s https://example.com/download.zip | exav -d /var/lib/exav - +$ curl -fsS https://example.com/download.zip | exav -d /var/lib/exav - stdin: OK ``` -Load the database once and serve it over either protocol, or both: +Load the database once and serve it over either protocol, or both (neither +protocol authenticates its clients, so bind to an interface only trusted clients +reach): ```console -$ exav --listen clamd://0.0.0.0:3310 --listen icap://0.0.0.0:1344 -d /var/lib/exav +$ exav --listen clamd://127.0.0.1:3310 --listen icap://127.0.0.1:1344 -d /var/lib/exav exav: prefork daemon: 4 workers; per-job limits: wall 120s, mem 2048 MiB, cpu 120s; recycle every 1000 jobs -exav: serving ICAP on tcp:0.0.0.0:1344 (services: avscan, srv_clamav, virus_scan; preview 4096 B) +exav: serving ICAP on tcp:127.0.0.1:1344 (services: avscan, srv_clamav, virus_scan; preview 4096 B) ``` One process and one loaded database replace a `c-icap` + `clamav` container @@ -63,9 +63,9 @@ coverage: ```console $ exav suspicious.bin -exav: no signature database loaded — refusing to run (it would report real -malware as clean). Load signatures with -d/--sig-dir, or pass --allow-no-db -to use the built-in EICAR-only baseline (testing only). +exav: no signature database loaded, refusing to run (it would report real +malware as clean). Load signatures with -d/--database or --sig-dir, or pass +--allow-no-db to use the built-in EICAR-only baseline (testing only). ``` ## Where to next diff --git a/www/src/content/docs/getting-started/quick-start.mdx b/www/src/content/docs/getting-started/quick-start.mdx index f37a7173..762df7b9 100644 --- a/www/src/content/docs/getting-started/quick-start.mdx +++ b/www/src/content/docs/getting-started/quick-start.mdx @@ -18,42 +18,47 @@ signatures loaded it refuses to run. Fetching signatures is the first step below 1. **Get ClamAV signatures** with Cisco's own updater: ```sh - pip install cvdupdate - cvd update # or `freshclam` + pipx install cvdupdate # or `pip install --user cvdupdate` + cvd update ``` This populates `~/.cvdupdate/database` with `main.cvd`, `daily.cvd` and - `bytecode.cvd`. See [Signatures](/guides/signatures/) for the licensing - background and third-party feeds. + `bytecode.cvd`. If `freshclam` already keeps a database current, use its + directory instead (usually `/var/lib/clamav`) in the commands below. See + [Signatures](/guides/signatures/) for the licensing background and + third-party feeds. -2. **Scan a file**, pointing `-d` at that directory: +2. **Check that detection works** with the EICAR test file, which every scanner + detects and which is harmless: ```sh - exav -d ~/.cvdupdate/database suspicious.bin + printf '%s%s' 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-' \ + 'TEST-FILE!$H+H*' > eicar.com + exav -d ~/.cvdupdate/database eicar.com ``` - The output follows `clamscan`'s format: - ```console - suspicious.bin: Win.Trojan.Agent-1234 FOUND + eicar.com: Eicar-Test-Signature FOUND ``` - or, for a clean file: +3. **Scan a file.** The output follows `clamscan`'s format: + + ```sh + exav -d ~/.cvdupdate/database suspicious.bin + ``` ```console suspicious.bin: OK ``` -3. **Scan a directory.** It is recursive by default; `--quiet` prints only errors - and detections: +4. **Scan a directory.** It is recursive by default; `--quiet` prints only + detections, partial results and errors: ```sh exav -d ~/.cvdupdate/database --quiet /var/www ``` -4. **Scan stdin** with `-`. The stream is held in memory up to - `--spill-threshold-bytes` (16 MiB), then in a temporary file up to - `--max-spill-bytes` (2 GiB), and scanned like a file: +5. **Scan stdin** with `-`: ```sh cat backup.tar.gz | exav -d ~/.cvdupdate/database - @@ -61,33 +66,11 @@ signatures loaded it refuses to run. Fetching signatures is the first step below -## Alternative: let exav fetch the signatures - -With [HTTP support](/reference/feature-flags/#adding-http-support) built in, exav -can pull Cisco's CVDs itself, replacing step 1: - -```sh -export EXAV_SIG_SOURCES=https://database.clamav.net/ -exav -d ~/.exav-db --auto-update suspicious.bin -``` - -The trailing slash makes the value a mirror base, expanded to -`main`/`daily`/`bytecode.cvd`; an exact `…/daily.cvd` URL is fetched as given. -Sources are validated into `-d`, which is then loaded for the scan, and -`--auto-update` keeps them current while the process runs. This needs a -`--features http` build (see [Installation](/getting-started/installation/)). - ## Exit codes The status is the last word on each result line, and each status has an exit -code: - -| Exit | Status | Meaning | -|---|---|---| -| `0` | `OK` | Every file was fully scanned and nothing matched. | -| `1` | `FOUND` | At least one detection. | -| `2` | `ERROR` | exav could not do its job: an unreadable path, a database that would not load. | -| `3` | `PARTIAL` | Something could not be fully examined: `LIMITS-EXCEEDED`, `UNSCANNABLE` or `PASSWORD-PROTECTED`. | +code: `0` for `OK`, `1` for `FOUND`, `2` for `ERROR` (exav could not do its job), +and `3` for `PARTIAL` (something could not be fully examined). , ldbs: Vec, pub unsupported: usize, @@ -2806,83 +2396,65 @@ impl SigEngine { self.bodies.is_empty() } - /// The partitions to run for file type `ft`: those whose `target` matches - /// (`target_ok`), each yielded as `(automaton, groups, haystack)` where the - /// haystack is the lowercased copy for a `nocase` partition and `buf` - /// otherwise. `lower` must be the lowercased `buf` when [`needs_lower`] - /// returns true (else it is never read). - fn active<'a>( - &'a self, - ft: FileType, - buf: &'a [u8], - lower: &'a [u8], - ) -> impl Iterator< - Item = ( - usize, - &'a DoubleArrayAhoCorasick, - &'a Groups, - &'a [u8], - ), - > { - // A `Target:5` (graphics) partition is normally skipped: exav doesn't - // model graphics as a `FileType`. But when the buffer IS an image, its - // Target:5 signatures legitimately apply, so run those partitions too - // (their subsignature anchors must match for an image-scoped LDB, e.g. a - // `Win.Phishing.*` raw-image sig, to become a candidate). - // The yielded partition index keys the per-scan [`GroupCost`] accounts - // (partitions shard the automaton value space, so the value alone does - // not identify a group). - let is_image = crate::fuzzy_img::looks_like_image(buf); + /// Per partition, whether a scan of file type `ft` looks for its anchors: + /// those whose `target` matches (`target_ok`). A `Target:5` (graphics) + /// partition is normally skipped, since exav doesn't model graphics as a + /// `FileType`; but when the object IS an image its signatures apply (an + /// image-scoped logical signature, e.g. a `Win.Phishing.*` raw-image sig, + /// needs its subsignature anchors to match to become a candidate). + fn active(&self, ft: FileType, is_image: bool) -> Vec { self.partitions .iter() - .enumerate() - .filter(move |(_, p)| target_ok(p.target, ft) || (p.target == 5 && is_image)) - .map(move |(i, p)| { - let hay: &[u8] = if p.nocase { lower } else { buf }; - (i, &p.ac, &p.groups, hay) - }) + .map(|p| target_ok(p.target, ft) || (p.target == 5 && is_image)) + .collect() } - /// The partitions to run for file type `ft`, as [`Self::active`] picks them, - /// each with whether it matches the lowercased object. - fn active_parts( - &self, - ft: FileType, - is_image: bool, - ) -> impl Iterator, &Groups, bool)> { - self.partitions - .iter() - .enumerate() - .filter(move |(_, p)| target_ok(p.target, ft) || (p.target == 5 && is_image)) - .map(|(i, p)| (i, &p.ac, &p.groups, p.nocase)) + /// The bodies behind value `value` of partition `pidx`; past the last + /// partition, where the pinned bodies' candidates are, body `value`. + fn group<'a>(&'a self, pidx: usize, value: &'a u32) -> &'a [u32] { + match self.partitions.get(pidx) { + Some(p) => p.groups.get(*value), + None => std::slice::from_ref(value), + } } - /// Whether any active partition for `ft` is case-insensitive, that is, whether - /// a lowercased copy of the haystack must be allocated for this scan. + /// Whether any active partition for `ft` is case-insensitive, that is, + /// whether verification may read the object lowercased. fn needs_lower(&self, ft: FileType, is_image: bool) -> bool { - self.active_parts(ft, is_image).any(|(_, _, _, nocase)| nocase) + self.pins.any_nocase() + || self + .partitions + .iter() + .zip(self.active(ft, is_image)) + .any(|(p, on)| on && p.nocase) + } + + /// Every anchor hit a scan of `buf` as file type `ft` finds, handed to `f` + /// as [`sweep`] does, the partition as the id. + fn each_hit(&self, buf: &[u8], ft: FileType, f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool) { + let active = self.active(ft, crate::fuzzy_img::looks_like_image(buf)); + sweep(&self.grams, &active, &buf, f); } - /// Serialize to the on-disk database: each double-array automaton via - /// daachorse's own format, the rest via bincode (by reference, no clone). + /// Serialize to the on-disk database: the index and the tables as binary + /// blobs, the bodies and logical signatures as MessagePack. pub(crate) fn write_cache(&self, mut w: W) -> std::io::Result<()> { - use crate::database::{enc, enc_bytes, enc_u32s}; + use crate::database::{enc, enc_bytes, enc_u32s, to_vec_indexed}; // The bodies and logical signatures first, each encoded whole into a // blob: a load decodes them on threads of their own while it reads the - // automatons behind them. - enc_bytes(&rmp_serde::to_vec(&self.bodies).map_err(std::io::Error::other)?, &mut w)?; - enc_bytes(&rmp_serde::to_vec(&self.ldbs).map_err(std::io::Error::other)?, &mut w)?; - // Each partition: target, nocase, then as binary blobs the automaton - // in daachorse's own format, its groups and its anchor runs. + // index behind them. + enc_bytes(&to_vec_indexed(&self.bodies)?, &mut w)?; + enc_bytes(&to_vec_indexed(&self.ldbs)?, &mut w)?; + // Each partition: target, nocase, then its groups as binary blobs. enc(&(self.partitions.len() as u32), &mut w)?; for p in &self.partitions { enc(&p.target, &mut w)?; enc(&p.nocase, &mut w)?; - enc_bytes(&p.ac.serialize(), &mut w)?; enc_u32s(&p.groups.start, &mut w)?; enc_u32s(&p.groups.bodies, &mut w)?; - enc_u32s(&p.runs, &mut w)?; } + self.grams.write(&mut w)?; + enc_bytes(&to_vec_indexed(&self.pins)?, &mut w)?; enc(&self.unsupported, &mut w)?; // What the build derived from those, stored so a load does not redo it. enc_u32s(&self.body_ldb, &mut w)?; @@ -2892,9 +2464,9 @@ impl SigEngine { Ok(()) } - /// Reverse of [`write_cache`]. The daachorse bytes come from a database file - /// this build wrote and validated (magic + version), and daachorse checks - /// them again as it reads them. + /// Reverse of [`write_cache`]. The bytes come from a database file this + /// build wrote and validated (magic + version); the index checks its own + /// tables as it reads them. pub(crate) fn read_cache(mut r: R) -> std::io::Result { use crate::database::{alongside, dec, dec_bytes, dec_u32s}; let corrupt = |what: &str| { @@ -2906,34 +2478,30 @@ impl SigEngine { let (bodies, ldbs) = (dec_bytes(&mut r)?, dec_bytes(&mut r)?); let decode_bodies = || rmp_serde::from_slice::>(&bodies).map_err(std::io::Error::other); let decode_ldbs = || rmp_serde::from_slice::>(&ldbs).map_err(std::io::Error::other); - let read_parts = || -> std::io::Result> { + let read_parts = || -> std::io::Result<(Vec, Grams)> { let n_parts: u32 = dec(&mut r)?; - let mut parts = Vec::with_capacity(n_parts as usize); + // Not sized from `n_parts`: the checksum is checked after this + // is read, and a damaged count would ask for gigabytes. + let mut parts = Vec::new(); for _ in 0..n_parts { let target: u8 = dec(&mut r)?; let nocase: bool = dec(&mut r)?; - let ac = DoubleArrayAhoCorasick::deserialize(&dec_bytes(&mut r)?) - .map(|(ac, _)| ac) - .map_err(|e| corrupt(&format!("aho-corasick deserialize: {e:?}")))?; let groups = Groups { start: dec_u32s(&mut r)?, bodies: dec_u32s(&mut r)?, }; - let runs = dec_u32s(&mut r)?.try_into().map_err(|_| corrupt("anchor run table"))?; - parts.push(Partition { - target, - nocase, - ac, - groups, - runs, - }); - } - Ok(parts) + parts.push(Partition { target, nocase, groups }); + } + let values: Vec = parts.iter().map(|p| p.groups.start.len().saturating_sub(1)).collect(); + let grams = Grams::read(&mut r, &values)?; + Ok((parts, grams)) }; // Three threads where there are: the bodies and the logical signatures - // decode while this one reads the automatons. + // decode while this one reads the anchors. let ((bodies, ldbs), parts) = alongside(|| alongside(decode_bodies, decode_ldbs), read_parts); - let (bodies, ldbs, parts) = (bodies?, ldbs?, parts?); + let (bodies, ldbs, (parts, grams)) = (bodies?, ldbs?, parts?); + let pins: Pins = rmp_serde::from_slice(&dec_bytes(&mut r)?).map_err(std::io::Error::other)?; + let pins = pins.load(&bodies).ok_or_else(|| corrupt("pinned bodies"))?; let unsupported = dec(&mut r)?; let body_ldb = dec_u32s(&mut r)?; let fires_empty = dec_u32s(&mut r)?; @@ -2951,6 +2519,8 @@ impl SigEngine { let has_fuzzy = !fuzzy_ldbs.is_empty(); Ok(SigEngine { partitions, + grams, + pins, bodies, ldbs, unsupported, @@ -3126,17 +2696,17 @@ impl SigEngine { // tuple shape). let treject = 0u64; let (mut lit, mut tok, mut ok) = (0u64, 0u64, 0u64); - let mut budgets = Budgets::new(); + let mut budget = SIM_BUDGET; let mut occ = OccCache::new(); - for p in self.partitions.iter().filter(|p| target_ok(p.target, ft)) { - let hay: &[u8] = if p.nocase { &lower } else { buf }; - for m in p.ac.find_overlapping_iter(hay) { + self.each_hit(buf, ft, &mut |pidx, value, start, _, count| { + let p = &self.partitions[pidx]; + for s in start..start + count { if p.nocase { ci_hits += 1; } else { cs_hits += 1; } - let group = p.groups.get(m.value()); + let group = p.groups.get(value); fanout += group.len() as u64; for &bid in group { let body = &self.bodies[bid as usize]; @@ -3146,7 +2716,7 @@ impl SigEngine { tok += 1; } let t = std::time::Instant::now(); - if verify(body, buf, m.start(), layout, &lower[..], &mut budgets, &mut occ).is_some() { + if verify(body, buf, s, layout, &lower[..], &mut budget, &mut occ).is_some() { ok += 1; } let us = t.elapsed().as_micros() as u64; @@ -3155,7 +2725,8 @@ impl SigEngine { } } } - } + true + }); [cs_hits, ci_hits, fanout, treject, lit, tok, ok] } @@ -3163,18 +2734,14 @@ impl SigEngine { /// indexed by anchor byte length (0..=15, last bucket = >=15), a tuple of /// (anchor_hits, total_fanout). Lets us see which anchor lengths drive the /// verify explosion on dense content. - pub fn scan_diag_hist(&self, buf: &[u8], _ft: FileType) -> Vec<(u64, u64)> { + pub fn scan_diag_hist(&self, buf: &[u8], ft: FileType) -> Vec<(u64, u64)> { let mut hist = vec![(0u64, 0u64); 16]; - let lower = buf.to_ascii_lowercase(); - for p in &self.partitions { - let hay: &[u8] = if p.nocase { &lower } else { buf }; - for m in p.ac.find_overlapping_iter(hay) { - let len = (m.end() - m.start()).min(15); - let g = p.groups.get(m.value()).len() as u64; - hist[len].0 += 1; - hist[len].1 += g; - } - } + self.each_hit(buf, ft, &mut |pidx, value, _, len, count| { + let g = self.partitions[pidx].groups.get(value).len() as u64; + hist[len.min(15)].0 += count as u64; + hist[len.min(15)].1 += g * count as u64; + true + }); hist } @@ -3214,22 +2781,18 @@ impl SigEngine { pub fn scan_diag_groups( &self, buf: &[u8], - _ft: FileType, + ft: FileType, ) -> Vec<(usize, usize, u64, u64, i32, u32)> { // Count hits per group value (case-sensitive partitions only: they - // dominate). Keyed by `(partition index, value)` since sharded - // partitions reuse the same local value space. + // dominate), keyed by `(partition index, value)`. let mut hits: std::collections::HashMap<(usize, u32), (u64, usize)> = std::collections::HashMap::new(); - for (pidx, p) in self.partitions.iter().enumerate() { - if p.nocase { - continue; - } - for m in p.ac.find_overlapping_iter(buf) { - let len = m.end() - m.start(); - hits.entry((pidx, m.value())).or_insert((0, len)).0 += 1; + self.each_hit(buf, ft, &mut |pidx, value, _, len, count| { + if !self.partitions[pidx].nocase { + hits.entry((pidx, value)).or_insert((0, len)).0 += count as u64; } - } + true + }); let mut rows: Vec<(usize, usize, u64, u64, i32, u32)> = Vec::new(); let mut cons = Vec::new(); for ((pidx, val), (h, anchor_len)) in hits { @@ -3290,16 +2853,16 @@ impl SigEngine { /// would remove: the ceiling on what such a gate can save, before paying /// for whatever pass computes it. pub fn scan_diag_gate(&self, buf: &[u8], ft: FileType) -> (u64, u64, usize, usize) { - let lower = buf.to_ascii_lowercase(); - // Pass 1: which bodies have an anchor hit anywhere. + // Pass 1: which bodies have an anchor hit anywhere, and the hits. let mut hit = vec![false; self.bodies.len()]; - for (_pidx, ac, groups, hay) in self.active(ft, buf, &lower) { - for m in ac.find_overlapping_iter(hay) { - for &bid in groups.get(m.value()) { - hit[bid as usize] = true; - } + let mut hits = Vec::new(); + self.each_hit(buf, ft, &mut |pidx, value, _, _, count| { + for &bid in self.partitions[pidx].groups.get(value) { + hit[bid as usize] = true; } - } + hits.push((pidx, value, count)); + true + }); // Which logical sigs are still satisfiable given that. let mut alive: std::collections::HashMap = std::collections::HashMap::new(); for (bid, &h) in hit.iter().enumerate() { @@ -3324,14 +2887,12 @@ impl SigEngine { } // Pass 2: count the fan-out a gate would skip. let (mut fanout, mut prunable) = (0u64, 0u64); - for (_pidx, ac, groups, hay) in self.active(ft, buf, &lower) { - for m in ac.find_overlapping_iter(hay) { - for &bid in groups.get(m.value()) { - fanout += 1; - let li = self.body_ldb[bid as usize]; - if li != u32::MAX && !alive.get(&li).copied().unwrap_or(true) { - prunable += 1; - } + for (pidx, value, count) in hits { + for &bid in self.partitions[pidx].groups.get(value) { + fanout += count as u64; + let li = self.body_ldb[bid as usize]; + if li != u32::MAX && !alive.get(&li).copied().unwrap_or(true) { + prunable += count as u64; } } } @@ -3339,18 +2900,16 @@ impl SigEngine { (fanout, prunable, alive.len(), dead) } - /// Diagnostic: total anchor hits, running the automatons and nothing else. - /// Isolates the Aho-Corasick pass from the verification that follows it, - /// which is what says whether a scheme that trades an extra automaton pass - /// for fewer verifies can pay for itself. + /// Diagnostic: total anchor hits, running the anchor search and nothing + /// else. Isolates that pass from the verification that follows it, which is + /// what says whether a scheme that trades an extra pass for fewer verifies + /// can pay for itself. pub fn scan_diag_hits(&self, buf: &[u8], ft: FileType) -> u64 { - let lower = buf.to_ascii_lowercase(); let mut n = 0u64; - for (_pidx, ac, _groups, hay) in self.active(ft, buf, &lower) { - for _m in ac.find_overlapping_iter(hay) { - n += 1; - } - } + self.each_hit(buf, ft, &mut |_, _, _, _, count| { + n += count as u64; + true + }); n } @@ -3364,17 +2923,15 @@ impl SigEngine { buf: &[u8], ft: FileType, ) -> Vec<(Vec, u64, usize, Vec)> { - let lower = buf.to_ascii_lowercase(); let mut hits: std::collections::HashMap<(usize, u32), (u64, Vec)> = std::collections::HashMap::new(); - for (pidx, ac, _groups, hay) in self.active(ft, buf, &lower) { - for m in ac.find_overlapping_iter(hay) { - let e = hits - .entry((pidx, m.value())) - .or_insert_with(|| (0, hay[m.start()..m.end()].to_vec())); - e.0 += 1; - } - } + self.each_hit(buf, ft, &mut |pidx, value, start, len, count| { + let e = hits + .entry((pidx, value)) + .or_insert_with(|| (0, buf[start..start + len].to_vec())); + e.0 += count as u64; + true + }); let mut rows: Vec<(Vec, u64, usize, Vec)> = hits .into_iter() .map(|((pidx, val), (h, bytes))| { @@ -3437,16 +2994,6 @@ impl SigEngine { cand } - /// The automatons to run for file type `ft`, each with its partition. - fn lanes(&self, ft: FileType, is_image: bool) -> (Vec, Vec>) { - self.active_parts(ft, is_image) - .map(|(pidx, ac, _, nocase)| { - let runs = &self.partitions[pidx].runs; - (pidx, Lane { ac, fold: nocase, runs }) - }) - .unzip() - } - /// Which logical signatures can still fire, indexed by ldb id, given the /// anchor `hits` of a whole sweep, which stand for `total` hits. `None` /// when gating is not worth setting up. @@ -3467,7 +3014,7 @@ impl SigEngine { } let mut hit_body = vec![false; self.bodies.len()]; for h in hits { - for &bid in self.partitions[h.pidx as usize].groups.get(h.value) { + for &bid in self.group(h.pidx as usize, &h.value) { hit_body[bid as usize] = true; } } @@ -3529,18 +3076,16 @@ impl SigEngine { buf, layout, lower, - budgets: Budgets::new(), + budget: SIM_BUDGET, occ: OccCache::new(), - group_cost: GroupCost::new(max_group_steps()), + group_cost: GroupCost::new(MAX_GROUP_STEPS), on_match, }; - let (pidxs, lanes) = self.lanes(ft, is_image); let cap = max_buffered_hits(); // Nothing to gate without logical signatures. let mut buffered = (!self.ldbs.is_empty()).then(Vec::new); let mut total = 0u64; - let whole = sweep(&lanes, buf, &mut |lane, value, start, count| { - let pidx = pidxs[lane]; + let mut take = |pidx: usize, value: u32, start: usize, count: usize| { if let Some(hits) = buffered.as_mut().filter(|h| h.len() + count.div_ceil(SweepHit::MAX_COUNT) <= cap) { SweepHit::push(hits, pidx, value, start, count); total += count as u64; @@ -3552,7 +3097,17 @@ impl SigEngine { } } !v.hit(pidx, value, start, count, None) + }; + // The pinned bodies first, each a group of its own in the + // partition past the last: a few checks where their offsets put + // them. + let wanted = |b: &Body| target_ok(b.target, ft) || (b.target == 5 && is_image); + let pinned = self.partitions.len(); + let whole = self.pins.visit(&self.bodies, buf, lower, layout, &wanted, &mut |bid, at| { + take(pinned, bid as u32, at, 1) }); + let active = self.active(ft, is_image); + let whole = whole && sweep(&self.grams, &active, buf, &mut |pidx, value, start, _, count| take(pidx, value, start, count)); if let (true, Some(hits)) = (whole, buffered) { let alive = self.ldb_alive(&hits, total); v.replay(&hits, alive.as_deref()); @@ -3626,8 +3181,7 @@ impl SigEngine { } } - /// Verify every candidate body, lowercasing the object for the - /// case-insensitive partitions if any runs. + /// Verify every candidate body. /// `on_match(bid, start, n)` gets `n` matches of body `bid`, at `start`, /// `start + 1`, and so on. fn walk(&self, buf: &B, ft: FileType, is_image: bool, layout: Option<&PeLayout>, on_match: F) @@ -3635,20 +3189,16 @@ impl SigEngine { B: ByteSource + ?Sized, F: FnMut(usize, u64, u64) -> std::ops::ControlFlow<()>, { - // The case-insensitive automaton matches against a lowercased copy of - // the haystack; positions line up with the original (ASCII lowercasing - // preserves length), so verification still runs against `buf`. Only - // made if a case-insensitive partition runs, and for an object read - // through the cache it is a view, not a copy. + // Verifying a case-insensitive body reads the object lowercased, at + // the same positions (ASCII lowercasing preserves length). Held in + // memory, the object is lowercased the first time one is verified, + // which most scans never do; read through the cache, it is a view. if !self.needs_lower(ft, is_image) { let none: &[u8] = &[]; return self.walk_candidates(buf, ft, is_image, layout, none, on_match); } match buf.as_slice() { - Some(s) => { - let lower = s.to_ascii_lowercase(); - self.walk_candidates(buf, ft, is_image, layout, &lower[..], on_match) - } + Some(s) => self.walk_candidates(buf, ft, is_image, layout, &LazyLower::new(s), on_match), None => self.walk_candidates(buf, ft, is_image, layout, &Lower(buf), on_match), } } @@ -4054,13 +3604,13 @@ fn is_word_byte(b: u8) -> bool { } /// What verifying the candidates of one scan carries from one anchor hit to -/// the next: the step pools, the occurrence memo and the group costs. +/// the next: the step pool, the occurrence memo and the group costs. struct Verifier<'s, B: ?Sized, L: ?Sized, F> { eng: &'s SigEngine, buf: &'s B, layout: Option<&'s PeLayout>, lower: &'s L, - budgets: Budgets, + budget: u64, occ: OccCache, group_cost: GroupCost, on_match: F, @@ -4080,7 +3630,7 @@ where return false; } let eng = self.eng; - for &bid in eng.partitions[pidx].groups.get(value) { + for &bid in eng.group(pidx, &value) { let bid = bid as usize; if let Some(alive) = alive { let li = eng.body_ldb[bid]; @@ -4088,26 +3638,32 @@ where continue; } } - let body = &eng.bodies[bid]; - let pool_before = self.budgets.sim + self.budgets.legacy; - let on_match = &mut self.on_match; - let (budgets, occ) = (&mut self.budgets, &mut self.occ); - let stop = if count == 1 { - verify(body, self.buf, start, self.layout, self.lower, budgets, occ) - .is_some_and(|s| on_match(bid, s, 1).is_break()) - } else { - let mut report = |s, n| on_match(bid, s, n).is_break(); - verify_run(body, self.buf, start, count, self.layout, self.lower, budgets, occ, &mut report) - }; - let drawn = pool_before - (self.budgets.sim + self.budgets.legacy); - self.group_cost.charge(pidx, value, drawn); - if stop { + if self.body(bid, start, count, (pidx, value)) { return true; } } false } + /// Verify body `bid` at `count` anchor hits from `start`, charging the + /// steps to `group`; `true` when `on_match` asks to stop. + fn body(&mut self, bid: usize, start: usize, count: usize, group: (usize, u32)) -> bool { + let body = &self.eng.bodies[bid]; + let pool_before = self.budget; + let on_match = &mut self.on_match; + let (budget, occ) = (&mut self.budget, &mut self.occ); + let stop = if count == 1 { + verify(body, self.buf, start, self.layout, self.lower, budget, occ) + .is_some_and(|s| on_match(bid, s, 1).is_break()) + } else { + let mut report = |s, n| on_match(bid, s, n).is_break(); + verify_run(body, self.buf, start, count, self.layout, self.lower, budget, occ, &mut report) + }; + let drawn = pool_before - self.budget; + self.group_cost.charge(group.0, group.1, drawn); + stop + } + /// [`Self::hit`] for each of the buffered `hits`, in order; `true` when /// `on_match` asks to stop. fn replay(&mut self, hits: &[SweepHit], alive: Option<&[bool]>) -> bool { @@ -4116,10 +3672,11 @@ where let (value, start) = (h.value, h.start()); // A run longer than one entry holds was pushed as consecutive // entries: verified as one, its interior is not cut into pieces - // with edges of their own. + // with edges of their own. A pinned body's candidates are no run. let mut count = h.count(); next += 1; - while let Some(g) = hits.get(next) { + let pinned = h.pidx as usize == self.eng.partitions.len(); + while let Some(g) = hits.get(next).filter(|_| !pinned) { if (g.pidx, g.value, g.start()) != (h.pidx, value, start + count) { break; } @@ -4140,10 +3697,10 @@ fn verify( anchor_start: usize, layout: Option<&PeLayout>, lower: &L, - budgets: &mut Budgets, + budget: &mut u64, occ: &mut OccCache, ) -> Option { - let start = verify_inner(body, buf, anchor_start, layout, lower, budgets, occ)?; + let start = verify_inner(body, buf, anchor_start, layout, lower, budget, occ)?; let Some(len) = body.fullword_len else { return Some(start); }; @@ -4207,14 +3764,14 @@ fn verify_run( count: usize, layout: Option<&PeLayout>, lower: &L, - budgets: &mut Budgets, + budget: &mut u64, occ: &mut OccCache, on_match: &mut dyn FnMut(u64, u64) -> bool, ) -> bool { let end = start + count; // Verify the hit at `s`, and report a match as `n` of them. let mut one = |s: usize, n: usize| { - verify(body, buf, s, layout, lower, budgets, occ).is_some_and(|m| on_match(m, n as u64)) + verify(body, buf, s, layout, lower, budget, occ).is_some_and(|m| on_match(m, n as u64)) }; // The hits the offset allows, as ranges of anchor positions. let mut allowed: Vec<(usize, usize)> = Vec::new(); @@ -4271,11 +3828,11 @@ fn verify_inner( anchor_start: usize, layout: Option<&PeLayout>, lower: &L, - budgets: &mut Budgets, + budget: &mut u64, occ: &mut OccCache, ) -> Option { - // Pure literal: the Aho-Corasick hit is already the full match; only the - // offset constraint remains (no backtracking, so no budget is drawn). + // Pure literal: the anchor hit is already the full match; only the + // offset constraint remains, so no budget is drawn. let elems = match &body.elems { None => { return offset_ok(&body.offset, anchor_start as u64, buf.len() as u64, layout) @@ -4299,130 +3856,59 @@ fn verify_inner( // `gap_split_match_backward`, which reconstructs the byte-for-byte same // start the backtracking walk returned. // - // Everything draws from the `sim` pool; the `legacy` pool stays untouched. - if split_match_enabled() { - macro_rules! sim { - ($toks:expr, $at:expr) => {{ - if budgets.sim == 0 { - SCAN_TRUNCATED.with(|c| c.set(true)); - return None; - } - gap_split_match($toks, buf, $at, body.nocase, lower, &mut budgets.sim, occ) - }}; - } - return match body.prefix { - Prefix::Fixed { len, .. } => { - if anchor_start < len as usize { - return None; - } - let start = anchor_start - len as usize; - if !off_at(start) { - return None; - } - sim!(elems, start).then_some(start as u64) - } - Prefix::Floating { anchor_idx } => { - if !off_at(anchor_start) { - return None; - } - sim!(&elems[anchor_idx as usize..], anchor_start).then_some(anchor_start as u64) - } - Prefix::Internal { anchor_idx } => { - if !sim!(&elems[anchor_idx as usize..], anchor_start) { - return None; - } - if budgets.sim == 0 { - SCAN_TRUNCATED.with(|c| c.set(true)); - return None; - } - // The start comes from the backward simulator, which returns the - // same start the legacy walk did but without backtracking, so - // this path draws from the polynomial `sim` pool too, and the - // exponential `legacy` pool is never touched. - gap_split_match_backward( - &elems[..anchor_idx as usize], - buf, - anchor_start, - body.nocase, - lower, - &mut budgets.sim, - occ, - ) - .filter(|&start| off_at(start)) - .map(|start| start as u64) + // The offset constraint is checked before the simulator wherever the start + // is known up front: an offset-pinned body whose only anchor is a common + // run hits thousands of times per file, and all but the pinned hit are + // rejected without running it. + macro_rules! sim { + ($toks:expr, $at:expr) => {{ + if *budget == 0 { + SCAN_TRUNCATED.with(|c| c.set(true)); + return None; } - }; - } - - // Legacy backtracking path (split matcher disabled). - if budgets.legacy == 0 { - SCAN_TRUNCATED.with(|c| c.set(true)); - return None; + gap_split_match($toks, buf, $at, body.nocase, lower, budget, occ) + }}; } - let call_cap = VERIFY_BUDGET.min(budgets.legacy); - let mut budget = call_cap; - let result = match body.prefix { + match body.prefix { Prefix::Fixed { len, .. } => { if anchor_start < len as usize { - None - } else { - let start = anchor_start - len as usize; - // The pattern start is known up front, so apply the (cheap) - // offset constraint BEFORE the expensive backtracking match. - // Offset-pinned sigs (EP+0/Abs/Sec) whose only viable anchor is a - // common run (e.g. a zero triple) hit thousands of times per file; - // this rejects all but the ~one hit at the pinned position without - // ever running `match_forward`. For `Offset::Any` the gate is a - // constant `true`, so nothing is lost on the common case. - if !off_at(start) { - None - } else { - match_forward(elems, buf, start, body.nocase, &mut budget).then_some(start) - } + return None; } + let start = anchor_start - len as usize; + if !off_at(start) { + return None; + } + sim!(elems, start).then_some(start as u64) } Prefix::Floating { anchor_idx } => { if !off_at(anchor_start) { - None - } else { - match_forward( - &elems[anchor_idx as usize..], - buf, - anchor_start, - body.nocase, - &mut budget, - ) - .then_some(anchor_start) + return None; } + sim!(&elems[anchor_idx as usize..], anchor_start).then_some(anchor_start as u64) } - // Anchor sits past variable gaps: confirm the suffix forward from the - // anchor, then the prefix backward across the gaps to the pattern start. - // The start is only known after backward matching, so the offset gate - // runs last (these are `Offset::Any`-only, so it is a no-op anyway). + // The start is only known once the backward simulator has found it, + // so the offset check runs last. Prefix::Internal { anchor_idx } => { - if match_forward( - &elems[anchor_idx as usize..], + if !sim!(&elems[anchor_idx as usize..], anchor_start) { + return None; + } + if *budget == 0 { + SCAN_TRUNCATED.with(|c| c.set(true)); + return None; + } + gap_split_match_backward( + &elems[..anchor_idx as usize], buf, anchor_start, body.nocase, - &mut budget, - ) { - match_backward( - &elems[..anchor_idx as usize], - buf, - anchor_start, - body.nocase, - &mut budget, - ) - .filter(|&start| off_at(start)) - } else { - None - } + lower, + budget, + occ, + ) + .filter(|&start| off_at(start)) + .map(|start| start as u64) } - }; - // Charge the steps this verify actually spent back to the legacy pool. - budgets.legacy = budgets.legacy.saturating_sub(call_cap - budget); - result.map(|start| start as u64) + } } fn bytes_eq(a: &[u8], b: &[u8], nocase: bool) -> bool { @@ -4501,7 +3987,9 @@ fn add_delta(base: u64, delta: i64) -> Option { /// Match the token program against `buf` at `pos`. Backtracks over gaps and /// alternations; `budget` bounds total work. `nocase` folds ASCII case on -/// literal comparisons (the `i` subsig modifier). +/// literal comparisons (the `i` subsig modifier). The reference the tests hold +/// [`gap_split_match`] to. +#[cfg(test)] fn match_forward( toks: &[Elem], buf: &B, @@ -4630,9 +4118,9 @@ fn match_forward( /// Match a token program *backward* so it ends exactly at `end`, returning the /// start position (where the first token begins). The mirror of [`match_forward`] /// for verifying an [`Prefix::Internal`] anchor's preceding context across -/// variable gaps. `budget` bounds backtracking work. Only reached on the rare, -/// selective internal anchors, so it keeps the simple scalar gap loop (no SIMD -/// fast path): clarity over a micro-optimization that never runs hot. +/// variable gaps. `budget` bounds backtracking work. The reference the tests +/// hold [`gap_split_match_backward`] to. +#[cfg(test)] fn match_backward( toks: &[Elem], buf: &B, @@ -4733,7 +4221,7 @@ fn match_backward( // --- Gap-split anchored matching ------------------------------------------- // -// The legacy `match_forward` walks a wildcard body by recursive backtracking: +// The backtracking `match_forward` walks a wildcard body recursively: // at every variable gap it tries each gap length (or every occurrence of the // following literal) and recurses. On dense content a body with several large // or unbounded gaps fans out multiplicatively, so a per-buffer step budget has @@ -4752,20 +4240,6 @@ fn match_backward( // present/absent decision the backtracking walk computes, but without the // combinatorial blow-up, so the verify budget is (essentially) never spent. -/// Whether the non-backtracking gap-split matcher is used. `EXAV_SPLIT_MATCH=0` -/// (also `false`/`no`/`off`) forces the legacy backtracking `verify` path, so a -/// scan can be run both ways and the detections compared. Read once. -fn split_match_enabled() -> bool { - use std::sync::OnceLock; - static E: OnceLock = OnceLock::new(); - *E.get_or_init(|| { - !matches!( - std::env::var("EXAV_SPLIT_MATCH").ok().as_deref(), - Some("0") | Some("false") | Some("no") | Some("off") - ) - }) -} - /// Coalesce a position-interval set into sorted, disjoint form. Two intervals /// are merged when they touch or overlap (`next.lo <= cur.hi + 1`), since the /// positions are integers and `[a,b] ∪ [b+1,c] = [a,c]`. Keeping the set minimal @@ -5674,6 +5148,219 @@ enum Cmp { #[cfg(test)] mod tests { use super::*; + use crate::grams::{anchor_score, MIN_SKIP_RUN, NO_RUN_COLLAPSE, SWEEP_WINDOW}; + + thread_local! { + /// Builds on this thread pin no body: the reference the pinned + /// bodies' checks are held to. + pub(super) static NO_PINS: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + + /// A body its offset pins is found where it matches, and only there, as + /// the sweep found it: at a fixed offset, in a window stepped through or + /// searched, from the end, the entry point or a section, in either case, + /// and as a logical signature's subsignature. + #[test] + fn pinned_bodies_match_as_swept_ones_did() { + let mut rng = Rng(0x5EED_0F0F_F5E7); + let layout = PeLayout { + entry: Some(700), + section_rawptrs: vec![512, 1500], + section_rawsizes: vec![988, 2000], + version_info: Vec::new(), + }; + let lits: [&[u8]; 6] = [b"\x00\x00\x00", b"\x00\x00", b"MZ\x90\x00", b"abc", b"\xff\xfe\x00\x00\x00\x00\x00\x01", b"Ab"]; + let mut ndb = String::new(); + let mut ldb = String::new(); + for i in 0..300 { + let lit = lits[rng.below(lits.len())]; + let hex: String = lit.iter().map(|b| format!("{b:02x}")).collect(); + let off = match rng.below(9) { + 0 => "0".to_string(), + 1 => format!("{}", rng.below(3000)), + 2 => format!("{},{}", rng.below(3000), 1 + rng.below(60)), + 3 => format!("{},{}", rng.below(3000), 100 + rng.below(3000)), + 4 => format!("EOF-{}", 1 + rng.below(3000)), + 5 => format!("EP+{}", rng.below(40)), + 6 => format!("EP-{},{}", rng.below(40), rng.below(80)), + 7 => format!("S{}+{}", rng.below(2), rng.below(100)), + _ => format!("SL+{},{}", rng.below(100), rng.below(200)), + }; + let body = match rng.below(3) { + 0 => hex.clone(), + 1 => format!("{hex}??{hex}"), + _ => format!("{hex}{{1-4}}{hex}"), + }; + match rng.below(3) { + 0 => { + let nocase = if rng.below(2) == 0 { "::i" } else { "" }; + ldb.push_str(&format!("L.{i};Engine:51-255,Target:1;0&1;{off}:{body}{nocase};6d61726b\n")); + } + _ => ndb.push_str(&format!("N.{i}:1:{off}:{body}\n")), + } + } + let build = |no_pins: bool| { + NO_PINS.set(no_pins); + let mut b = EngineBuilder::new(); + b.add_ndb(&ndb, false); + b.add_ldb(&ldb, false); + let e = b.build(); + NO_PINS.set(false); + e + }; + let (with, without) = (build(false), build(true)); + assert!(with.pins.bodies().count() > 150 && without.pins.bodies().count() == 0); + let mut matched = 0; + for round in 0..40 { + let mut hay = vec![b'.'; 3500]; + for _ in 0..60 { + let lit = lits[rng.below(lits.len())]; + let at = rng.below(hay.len() - 2 * lit.len() - 4); + hay[at..at + lit.len()].copy_from_slice(lit); + if rng.below(2) == 0 { + hay[at + lit.len() + 1..at + 2 * lit.len() + 1].copy_from_slice(lit); + } + } + hay.extend_from_slice(b"mark"); + for ly in [Some(&layout), None] { + let all = |e: &SigEngine| { + let mut out = Vec::new(); + e.scan_all_with_layout(&hay, FileType::Pe, ly, None, &mut out); + out.sort(); + out + }; + let want = all(&without); + matched += want.len(); + assert_eq!(all(&with), want, "round {round}, layout {}", ly.is_some()); + } + } + assert!(matched > 500, "{matched}"); + } + + /// Every `(body, start)` a scan's sweep of `hay` as `ft` hands on, and in + /// how many reports. + fn body_hits(e: &SigEngine, hay: &H, ft: FileType) -> (Vec<(u32, usize)>, usize) { + let active = e.active(ft, false); + let (mut out, mut reports) = (Vec::new(), 0); + sweep(&e.grams, &active, hay, &mut |pidx, value, start, _, count| { + reports += 1; + for &b in e.partitions[pidx].groups.get(value) { + out.extend((start..start + count).map(|s| (b, s))); + } + true + }); + out.sort_unstable(); + (out, reports) + } + + /// Every `(body, start)` of the anchors `partition` chose, searched for + /// one by one: what the index has to find. + fn searched_hits(e: &SigEngine, anchors: &[(u32, bool, u32, Vec)], hay: &[u8], ft: FileType) -> Vec<(u32, usize)> { + let active = e.active(ft, false); + let mut out = Vec::new(); + for (part, nocase, value, a) in anchors { + if !active[*part as usize] || a.len() > hay.len() { + continue; + } + for s in 0..=hay.len() - a.len() { + let w = &hay[s..s + a.len()]; + if (*nocase && w.eq_ignore_ascii_case(a)) || (!*nocase && w == &a[..]) { + out.extend(e.partitions[*part as usize].groups.get(*value).iter().map(|&b| (b, s))); + } + } + } + out.sort_unstable(); + out + } + + /// The anchor index finds every anchor at every place it occurs, in its + /// case and for its file types, and nowhere else: in memory or read + /// through a cache, across windows, at the edges and inside runs, where + /// an anchor of the run's byte comes as a few reports. + #[test] + fn indexed_anchors_are_found_wherever_they_occur() { + let mut rng = Rng(0x0DDB_A11C_AFE0_1234); + let alpha = b"ab\x00Zz"; + let mut ndb = String::new(); + let mut ldb = String::new(); + let mut anchors = Vec::new(); + let mut any_type = Vec::new(); + for i in 0..400 { + let len = 2 + rng.below(14); + let lit: Vec = (0..len).map(|_| alpha[rng.below(alpha.len())]).collect(); + let hex: String = lit.iter().map(|b| format!("{b:02x}")).collect(); + let target = [0, 1, 3, 7][rng.below(4)]; + match rng.below(4) { + 0 => ldb.push_str(&format!("L.{i};Engine:51-255,Target:{target};0;{hex}::i\n")), + 1 => ndb.push_str(&format!("W.{i}:{target}:*:{hex}??{hex}\n")), + _ => ndb.push_str(&format!("N.{i}:{target}:*:{hex}\n")), + } + if target == 0 && lit.len() >= 6 && lit.iter().any(|&c| c != lit[0]) { + any_type.push(lit.clone()); + } + anchors.push(lit); + } + // Anchors that start inside a run of zeros, found at its last + // positions, and anchors of one repeated byte, of every width. + ndb.push_str("E.1:0:*:0000000000ab\nE.2:0:*:00000000abcd\nE.3:0:*:000000abcdef\n"); + ndb.push_str("U.1:0:*:0000\nU.2:0:*:000000\nU.3:0:*:00000000\nU.4:0:*:0000000000000000000000\n"); + ldb.push_str("U.5;Engine:51-255,Target:0;0;6161::i\nU.6;Engine:51-255,Target:0;0;4141414141414141\n"); + let builder = || { + let mut b = EngineBuilder::new(); + b.add_ndb(&ndb, false); + b.add_ldb(&ldb, false); + b + }; + let e = builder().build(); + let (_, chosen, _, _) = builder().partition(); + // Anchors placed at every offset around window seams, and runs that + // end in anchors, one of them past a seam. + let longs: Vec<&Vec> = any_type.iter().take(20).collect(); + let mut seams = vec![b'.'; 40 * SWEEP_WINDOW]; + for (k, d) in (1..40).map(|k| (k, k % 20)) { + let a = longs[k % longs.len()]; + let at = k * SWEEP_WINDOW - d; + seams[at..at + a.len()].copy_from_slice(a); + } + seams.extend(std::iter::repeat_n(0u8, 2 * SWEEP_WINDOW)); + seams.extend_from_slice(longs[0]); + for run in [300, 2 * SWEEP_WINDOW + 7] { + seams.extend(std::iter::repeat_n(0u8, run)); + seams.extend_from_slice(b"\xab\xcd\xef."); + } + seams.extend(std::iter::repeat_n(b'A', 3 * SWEEP_WINDOW + 5)); + seams.extend(std::iter::repeat_n(b'a', 200)); + let cache = byte_source::BlockCache::with_sizes(std::io::Cursor::new(seams.clone()), 97, 8 * 97).unwrap(); + for ft in [FileType::Unknown, FileType::Pe] { + let want = searched_hits(&e, &chosen, &seams, ft); + assert!(want.len() > 20, "{}", want.len()); + let (got, reports) = body_hits(&e, &seams[..], ft); + assert_eq!(got, want, "seams {ft:?}"); + assert!(reports < want.len() / 50, "{reports} reports for {} hits", want.len()); + assert_eq!(body_hits(&e, &cache, ft).0, want, "seams {ft:?}, read through a cache"); + } + for round in 0..6 { + let mut hay = Vec::new(); + while hay.len() < 5000 { + match rng.below(8) { + 0 => hay.extend(std::iter::repeat_n(0u8, 50 + rng.below(3000))), + 1 => { + let a = &anchors[rng.below(anchors.len())]; + hay.extend(a.iter().map(|&c| if rng.below(2) == 0 { c.to_ascii_uppercase() } else { c })); + } + 2 => hay.extend_from_slice(&anchors[rng.below(anchors.len())]), + _ => hay.push(alpha[rng.below(alpha.len())]), + } + } + let cache = byte_source::BlockCache::with_sizes(std::io::Cursor::new(hay.clone()), 511, 8 * 511).unwrap(); + for ft in [FileType::Unknown, FileType::Pe, FileType::Html, FileType::Text] { + let want = searched_hits(&e, &chosen, &hay, ft); + assert!(!want.is_empty()); + assert_eq!(body_hits(&e, &hay[..], ft).0, want, "round {round} {ft:?}"); + assert_eq!(body_hits(&e, &cache, ft).0, want, "round {round} {ft:?}, read through a cache"); + } + } + } #[test] fn fuzzy_img_ldb_fires_despite_target5_gate() { @@ -5745,6 +5432,41 @@ mod tests { b.build() } + /// Past [`MAX_BUFFERED_HITS`] the sweep stops buffering its hits and + /// verifies them as they come, ungated: the same signatures match, first + /// or all, wherever the buffer overflows. + #[test] + fn a_full_hit_buffer_changes_no_answer() { + let mut b = EngineBuilder::new(); + b.add_ndb("N.Run:0:*:41414141??41", false); + b.add_ndb("N.Tail:0:*:5a5a5a5a", false); + b.add_ldb("L.Two;Engine:51-255,Target:0;0&1;42424242;43434343", false); + b.add_ldb("L.None;Engine:51-255,Target:0;0&1;42424242;44444444", false); + let e = b.build(); + let mut buf = Vec::new(); + for k in 0..500 { + buf.extend_from_slice(b"AAAA"); + buf.push(b'0' + (k % 10) as u8); + buf.extend_from_slice(b"A.."); + } + buf.extend_from_slice(b"..BBBB..CCCC..ZZZZ"); + let answers = |cap: usize| { + BUFFERED_HITS_CAP.set(cap); + let first = e.scan(&buf, FileType::Unknown).map(|(n, o, _)| (n, o)); + let mut all = Vec::new(); + e.scan_all_with_layout(&buf, FileType::Unknown, None, None, &mut all); + all.sort(); + BUFFERED_HITS_CAP.set(MAX_BUFFERED_HITS); + (first, all) + }; + let want = answers(MAX_BUFFERED_HITS); + let names: Vec<_> = want.1.iter().map(|(n, _, _)| n.as_str()).collect(); + assert_eq!(names, ["L.Two", "N.Run", "N.Tail"]); + for cap in [0, 1, 7, 250] { + assert_eq!(answers(cap), want, "cap {cap}"); + } + } + /// Decode hex (whitespace ignored) so test buffers hold the real bytes a /// signature matches, not their ASCII spelling. fn hx(s: &str) -> Vec { @@ -6463,7 +6185,7 @@ mod tests { /// any reference corpus. #[test] fn a_rare_literal_outranks_a_popular_one() { - use super::parse::{anchor_score, pick_anchor, GramStats}; + use super::parse::pick_anchor; // Each call is one body. The proportions are what the real set looks // like: a literal hundreds of signatures are built around, against one // that appears in a single signature. @@ -6511,7 +6233,6 @@ mod tests { /// must not change any score, since `log2(1)` is already zero. #[test] fn single_body_grams_cost_nothing() { - use super::parse::{anchor_score, GramStats}; let stats = GramStats::build(2, |f| { f(b"ONLYONCE"); f(b"ALSOONCE"); @@ -6532,7 +6253,6 @@ mod tests { /// spreadsheet part, with `"://"` and `"htt"` becoming hot anchors. #[test] fn a_short_literal_is_charged_for_every_body_containing_it() { - use super::parse::{anchor_score, GramStats}; // No body IS "//" or "://": they only ever appear inside a longer one. let stats = GramStats::build(500, |f| { for _ in 0..500 { @@ -6556,7 +6276,7 @@ mod tests { /// identically. #[test] fn the_repeat_filter_changes_no_score() { - use super::parse::{anchor_score, GramStats, FILTER_MIN_BODIES}; + use crate::grams::FILTER_MIN_BODIES; let feed = |f: &mut dyn FnMut(&[u8])| { for _ in 0..300 { f(b"POPULAR!"); @@ -6727,9 +6447,8 @@ mod tests { /// every token program, buffer, start, and case-fold flag. This correctness /// is what makes the split path a detection-preserving drop-in. Checked /// against the exhaustive `bt_match` oracle over a large random sample. - /// Additionally asserts the split path never *misses* what the legacy - /// `match_forward` finds (`match_forward ⟹ gap_split`): the switch can only - /// ever preserve or add a true detection, never drop one. + /// Additionally asserts the split path never *misses* what the backtracking + /// `match_forward` finds (`match_forward ⟹ gap_split`). #[test] fn gap_split_matches_backtracker() { let alpha = b"AaB"; // includes an upper/lower pair so nocase matters @@ -6760,12 +6479,12 @@ mod tests { let mut b2 = u64::MAX; let mut occ = OccCache::new(); let got = gap_split_match(&toks, &buf, pos, nocase, &lower, &mut b2, &mut occ); - // The legacy walk must never find a match the split path misses. + // The backtracking walk must never find a match the split path misses. let mut b1 = u64::MAX; if match_forward(&toks, &buf, pos, nocase, &mut b1) { assert!( got, - "gap-split dropped a legacy match: toks={toks:?} buf={buf:?} pos={pos} nocase={nocase}" + "gap-split dropped a backtracking match: toks={toks:?} buf={buf:?} pos={pos} nocase={nocase}" ); } } @@ -6862,29 +6581,24 @@ mod tests { Elem::Bytes(b"a".to_vec()), ]; let lower = buf.to_ascii_lowercase(); + // The per-call cap the backtracking walk ran under. + const CAP: u64 = 200_000; - // Legacy walk: cut off by its per-call cap, having decided nothing. - let mut legacy = VERIFY_BUDGET; - assert_eq!( - match_backward(&toks, &buf, buf.len(), false, &mut legacy), - None - ); - assert_eq!(legacy, 0, "the backtracker should have drained its budget"); + // Backtracking walk: cut off by its cap, having decided nothing. + let mut steps = CAP; + assert_eq!(match_backward(&toks, &buf, buf.len(), false, &mut steps), None); + assert_eq!(steps, 0, "the backtracker should have drained its budget"); // Simulator: a real "no match", with the budget barely touched. reset_scan_truncated(); - let mut sim = VERIFY_BUDGET; + let mut sim = CAP; let mut occ = OccCache::new(); assert_eq!( gap_split_match_backward(&toks, &buf, buf.len(), false, &lower, &mut sim, &mut occ), None ); assert!(!scan_was_truncated(), "must not report an incomplete search"); - assert!( - sim > VERIFY_BUDGET / 2, - "should finish well inside the budget, spent {}", - VERIFY_BUDGET - sim - ); + assert!(sim > CAP / 2, "should finish well inside the budget, spent {}", CAP - sim); } #[test] @@ -6960,10 +6674,10 @@ mod tests { } /// The trickier anchor/element classes (negated alternation and an Internal - /// anchor whose start comes from the backward walk) must detect correctly - /// under BOTH the split path and the legacy path, and agree with each other. + /// anchor whose start comes from the backward walk) detect correctly, at + /// the right start. #[test] - fn split_and_legacy_agree_on_tricky_constructs() { + fn tricky_constructs_are_detected_at_their_start() { let run = |line: &str, buf: &[u8]| -> bool { let e = ndb(line); e.scan(buf, FileType::Unknown).is_some() @@ -6977,13 +6691,10 @@ mod tests { assert!(run("Intern:0:*:00000000*cafebabe", &hx("00000000 99 cafebabe"))); assert!(!run("Intern:0:*:00000000*cafebabe", &hx("00000000 99 deadbeef"))); - // The Internal-anchor detection must report the SAME start offset whether - // the forward suffix is decided by the simulator or the legacy walk (the - // start always comes from the backward walk, so it must not drift). + // The Internal-anchor detection reports the start the backward walk + // finds. let internal = ndb("IOff:0:*:00000000*cafebabe"); let buf = hx("77 00000000 99 cafebabe"); - // (env is process-global; this test documents the invariant that the - // backward-derived start is path-independent: start = 1 here.) assert_eq!( internal.scan(&buf, FileType::Unknown).map(|(_, o, _)| o), Some(1) @@ -7270,18 +6981,7 @@ mod tests { assert_eq!(fast.5, fast.2, "streamed every match, {}", ctx()); assert_eq!(fast.6, fast.3, "streamed logical offsets, {}", ctx()); matched += slow.0.is_some() as usize; - let longest = |c: u8| { - buf.chunk_by(|a, b| a == b) - .filter(|r| r[0] == c) - .map(<[u8]>::len) - .max() - .unwrap_or(0) - }; - collapsed += e - .partitions - .iter() - .any(|p| (0..=255u8).any(|c| longest(c) > p.runs[c as usize] as usize + 1)) - as usize; + collapsed += buf.chunk_by(|a, b| a == b).any(|r| r.len() >= MIN_SKIP_RUN) as usize; } } reset_scan_truncated(); @@ -7289,150 +6989,6 @@ mod tests { assert!(collapsed > 1000, "too few haystacks collapsed ({collapsed})"); } - /// A long run of one byte comes out of the sweep as a few entries whose - /// expansion is exactly the automaton's hit-by-hit output, in memory and - /// read through the block cache across its seams. - #[test] - fn a_long_run_is_swept_as_a_few_entries() { - use crate::byte_source::BlockCache; - let mut b = EngineBuilder::new(); - for (i, body) in ["0000", "000000", "00000000ff", "ff00", "6161"].iter().enumerate() { - b.add_ndb(&format!("S{i}:0:*:{body}"), false); - } - b.add_ldb("L;Engine:81-255,Target:0;0;61616161::i", false); - let e = b.build(); - let mut buf = vec![b'a'; 5]; - buf.extend(std::iter::repeat_n(0u8, 1 << 20)); - buf.extend(b"\xff\x00\x00aaa"); - buf.extend(std::iter::repeat_n(b'A', 3000)); - let lanes = all_lanes(&e); - assert!(lanes.iter().any(|l| l.fold) && lanes.iter().any(|l| !l.fold)); - let want = one_by_one(&lanes, &buf); - let src = BlockCache::with_sizes(std::io::Cursor::new(buf.clone()), 4096, 3).unwrap(); - let slice: &[u8] = &buf; - for hay in [&slice as &dyn ByteSource, &src] { - let (got, entries) = swept(&lanes, hay); - assert_eq!(got, want); - assert!(entries < 100, "{entries} entries for two runs"); - } - } - - /// Every automaton of `e`, as [`sweep`] takes them. - fn all_lanes(e: &SigEngine) -> Vec> { - e.partitions - .iter() - .map(|p| Lane { - ac: &p.ac, - fold: p.nocase, - runs: &p.runs, - }) - .collect() - } - - /// Each automaton's own matches on `buf` (lowercased for a folding one), - /// as `(lane, value, start)`, sorted: what [`sweep`] has to report. - fn one_by_one(lanes: &[Lane], buf: &[u8]) -> Vec<(usize, u32, usize)> { - let lower = buf.to_ascii_lowercase(); - let mut want = Vec::new(); - for (k, l) in lanes.iter().enumerate() { - let hay = if l.fold { &lower[..] } else { buf }; - want.extend(l.ac.find_overlapping_iter(hay).map(|m| (k, m.value(), m.start()))); - } - want.sort_unstable(); - want - } - - /// [`sweep`]'s matches over `hay`, expanded and sorted as [`one_by_one`] - /// gives them, and how many entries they came as. - fn swept(lanes: &[Lane], hay: &dyn ByteSource) -> (Vec<(usize, u32, usize)>, usize) { - let (mut got, mut entries) = (Vec::new(), 0); - assert!(sweep(lanes, hay, &mut |k, v, s, n| { - entries += 1; - got.extend((s..s + n).map(|s| (k, v, s))); - true - })); - got.sort_unstable(); - (got, entries) - } - - /// Automatons swept together report exactly what each reports on its own, - /// however many there are (the budget shards them into up to a dozen), - /// case-folded or not, over haystacks full of runs, in memory and read - /// through small cache blocks and windows. - #[test] - fn automatons_swept_together_match_one_by_one() { - use crate::byte_source::BlockCache; - let mut rng = Rng(0x10c5_7e95); - let mut lane_counts = std::collections::BTreeSet::new(); - for round in 0..150 { - let mut b = EngineBuilder::new(); - for i in 0..3 + rng.below(12) { - let (logical, line) = runny_sig(&mut rng, i); - if logical { - b.add_ldb(&line, false); - } else { - b.add_ndb(&line, false); - } - } - // A budget of one anchor per shard at most, and none at all. - let budget = [Some(3072), Some(3 * 3072), None][rng.below(3)]; - let e = b.build_with_budget(budget); - let lanes = all_lanes(&e); - lane_counts.insert(lanes.len()); - for _ in 0..4 { - let mut buf = runny_haystack(&mut rng); - // Long enough to cross windows. - for _ in 0..rng.below(3) { - buf.extend_from_within(..); - } - let want = one_by_one(&lanes, &buf); - let src = BlockCache::with_sizes(std::io::Cursor::new(buf.clone()), 64, 256).unwrap(); - let slice: &[u8] = &buf; - for hay in [&slice as &dyn ByteSource, &src] { - assert_eq!(swept(&lanes, hay).0, want, "round {round}, {} lanes", lanes.len()); - } - } - } - assert!(lane_counts.iter().any(|&n| n > 2 * LANES), "{lane_counts:?}"); - assert!(lane_counts.iter().any(|&n| n % LANES != 0), "{lane_counts:?}"); - } - - /// `next_long_run` finds exactly the first run of at least `MIN_SKIP_RUN` - /// bytes, whatever its alignment and length, next to other runs, and at - /// the buffer's edges. - #[test] - fn long_runs_are_found_at_any_alignment() { - let naive = |s: &[u8], from: usize| { - let mut i = from; - for r in s[from..].chunk_by(|a, b| a == b) { - if r.len() >= MIN_SKIP_RUN { - return Some((i, r.len())); - } - i += r.len(); - } - None - }; - for pre in 0..20 { - for len in [1, 7, 8, 15, 56, 57, 62, 63, 64, 65, 71, 72, 200] { - for (other, other_len) in [(0u8, 0), (1, 30), (0, 63), (2, 64)] { - for post in [0, 1, 9] { - let mut s = vec![7u8; pre]; - s.extend(std::iter::repeat_n(other, other_len)); - s.extend(std::iter::repeat_n(0u8, len)); - s.extend(std::iter::repeat_n(5u8, post)); - for from in [0, pre / 2, pre] { - assert_eq!( - next_long_run(&s, from), - naive(&s, from), - "pre {pre} len {len} other {other}x{other_len} post {post} from {from}" - ); - } - } - } - } - } - } - /// Verifying a body at every hit inside a long run gives the matches a hit /// by hit verify gives, and draws work only near the run's edges and at /// the positions an offset allows. @@ -7450,7 +7006,7 @@ mod tests { let e = ndb(line); let body = &e.bodies[0]; let (mut slow, mut fast) = (Vec::new(), Vec::new()); - let (mut b1, mut b2) = (Budgets::new(), Budgets::new()); + let (mut b1, mut b2) = (SIM_BUDGET, SIM_BUDGET); let (mut o1, mut o2) = (OccCache::new(), OccCache::new()); for s in start..start + count { if let Some(m) = verify(body, &buf[..], s, None, &buf[..], &mut b1, &mut o1) { @@ -7463,7 +7019,7 @@ mod tests { }); assert!(!stopped); assert_eq!(fast, slow, "{line}"); - let (d1, d2) = (Budgets::new().sim - b1.sim, Budgets::new().sim - b2.sim); + let (d1, d2) = (SIM_BUDGET - b1, SIM_BUDGET - b2); assert!(d2 <= 10_000, "{line}: drew {d2} (hit by hit: {d1})"); } } @@ -7566,7 +7122,7 @@ mod tests { assert!(OccRun::push(&mut v, 10, 7, 7 + 3 * m)); let got: Vec<(usize, usize)> = v.iter().map(|r| (r.first(), r.last())).collect(); assert_eq!(got, [(5, 5), (7, 6 + m), (7 + m, 6 + 2 * m), (7 + 2 * m, 6 + 3 * m), (7 + 3 * m, 7 + 3 * m)]); - assert!(!OccRun::push(&mut v, 6, 1 << 40, (1 << 40) + m)); + assert!(!OccRun::push(&mut v, 6, 1 << 30, (1 << 30) + m)); assert_eq!(v.len(), 5); } @@ -7599,7 +7155,7 @@ mod tests { // Every position the four-byte anchor fits at, and nothing more. let (start, count) = (1, 997); let (mut slow, mut fast) = (Vec::new(), Vec::new()); - let (mut b1, mut b2) = (Budgets::new(), Budgets::new()); + let (mut b1, mut b2) = (SIM_BUDGET, SIM_BUDGET); let (mut o1, mut o2) = (OccCache::new(), OccCache::new()); for s in start..start + count { slow.extend(verify(body, &buf[..], s, None, &buf[..], &mut b1, &mut o1)); diff --git a/crates/exav-core/src/engine/parse.rs b/crates/exav-core/src/engine/parse.rs index 35dd120b..9c620430 100644 --- a/crates/exav-core/src/engine/parse.rs +++ b/crates/exav-core/src/engine/parse.rs @@ -4,6 +4,7 @@ use super::logic::parse_expr; use super::*; +use crate::grams::anchor_score; /// Parse an `.ldb` subsignature: `[Offset:]HexBody[::Modifiers]`. Expands to /// one or more compiled variants (ASCII and/or wide). `None` if unsupported @@ -556,230 +557,24 @@ fn decode_masked_hex(s: &str) -> Option> { Some(out) } -/// How many signature bodies contain each short byte sequence, counted over the -/// database being built. -/// -/// This is the selectivity signal [`anchor_score`] needs, derived from the -/// signature set itself rather than shipped alongside it. A literal that -/// hundreds of signatures are written around — `"http://"`, `"target=\""`, -/// `", -} - -/// Bits per presence filter. 2^28 bits is 32 MiB, ~8% occupancy against the -/// ~22M distinct grams of a full set, so few singletons are wrongly promoted to -/// the counting map — and a promotion costs only the memory, never a wrong -/// count, since the second pass counts what it actually sees. -const SEEN_BITS: usize = 1 << 28; - -/// Body count from which the repeat filter is worth its fixed cost. Below it -/// the two bitsets would be 64 MiB to filter a handful of grams — which is what -/// every in-process engine build does, from a unit test to a small sig dir. -pub(super) const FILTER_MIN_BODIES: usize = 100_000; - -/// A gram's key: its bytes, tagged with the length so `"ab"` and the prefix of -/// `"abcd"` cannot collide. Lengths are 2..=4, so this is lossless. -fn gram_key(g: &[u8]) -> u64 { - let mut v = 0u64; - for &b in g { - v = (v << 8) | b as u64; - } - v | ((g.len() as u64) << 56) -} - -/// Every gram of `lit` to CREDIT while counting: all windows of 2, 3 and 4 -/// bytes. -/// -/// Counting and lookup have to agree on what a gram's count means, and the -/// obvious shortcut — credit only the window size the lookup will use — is -/// wrong. Lookup asks a 2-byte literal about a 2-gram; if only 2-byte literals -/// ever credited 2-grams, `"//"` would be counted from the handful of two-byte -/// signatures rather than from the tens of thousands of literals that contain -/// it. Short ubiquitous literals then look rare, score well, and win the -/// anchor — the exact inversion this is built to prevent. Measured: doing it -/// that way moved anchors onto `"://"`, `"//"` and `"htt"` and raised token -/// verifies on one spreadsheet part by 84%. -fn count_grams_of(lit: &[u8], out: &mut Vec) { - for n in 2..=4usize { - if lit.len() < n { - break; - } - for w in lit.windows(n) { - out.push(gram_key(w)); - } - } -} - -/// The grams of `lit` to QUERY: the widest window it supports, up to 4. A -/// literal occurs in at most as many bodies as its rarest gram, and the widest -/// window gives the tightest such bound. -fn query_grams_of(lit: &[u8]) -> impl Iterator + '_ { - let n = lit.len().min(4); - lit.windows(n).map(gram_key) -} - -impl GramStats { - /// Count grams over every candidate literal `each` yields. - /// - /// `bodies` sizes the work. Past [`FILTER_MIN_BODIES`] the repeat filter - /// pays for itself and `each` is called twice — once to find which grams - /// repeat, once to count only those. Below it the filter's fixed 64 MiB - /// would dwarf the data it is filtering, so the counts are taken exactly in - /// a single pass. Both give identical scores: a gram the filter drops - /// appears in one body, and `log2(1)` is the zero an absent gram scores. - pub(super) fn build(bodies: usize, mut each: impl FnMut(&mut dyn FnMut(&[u8]))) -> Self { - let mut buf: Vec = Vec::new(); - if bodies < FILTER_MIN_BODIES { - let mut counts: rustc_hash::FxHashMap = rustc_hash::FxHashMap::default(); - each(&mut |lit| { - buf.clear(); - count_grams_of(lit, &mut buf); - buf.sort_unstable(); - buf.dedup(); - for &k in &buf { - *counts.entry(k).or_insert(0) += 1; - } - }); - return GramStats { counts }; - } - let mut once = vec![0u64; SEEN_BITS / 64]; - let mut twice = vec![0u64; SEEN_BITS / 64]; - let mark = |bits: &mut [u64], i: usize| bits[i >> 6] |= 1 << (i & 63); - let test = |bits: &[u64], i: usize| bits[i >> 6] >> (i & 63) & 1 == 1; - let slot = |k: u64| { - // Multiply-shift: the key is already a good integer, it only needs - // its high bits spread into the index range. - (k.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> (64 - SEEN_BITS.trailing_zeros())) as usize - }; - // Pass 1: which grams appear in more than one body. - each(&mut |lit| { - buf.clear(); - count_grams_of(lit, &mut buf); - buf.sort_unstable(); - buf.dedup(); - for &k in &buf { - let i = slot(k); - if test(&once, i) { - mark(&mut twice, i); - } else { - mark(&mut once, i); - } - } - }); - drop(once); - // Pass 2: count the repeats. Per body, not per occurrence, so a literal - // repeated inside one signature does not look popular. - let mut counts: rustc_hash::FxHashMap = rustc_hash::FxHashMap::default(); - each(&mut |lit| { - buf.clear(); - count_grams_of(lit, &mut buf); - buf.sort_unstable(); - buf.dedup(); - for &k in &buf { - if test(&twice, slot(k)) { - *counts.entry(k).or_insert(0) += 1; - } - } - }); - GramStats { counts } - } - - /// How many bodies contain `lit`, bounded above: a literal occurs in at most - /// as many bodies as its rarest gram does. Erring high is the safe - /// direction — it can only make a literal look like a worse anchor. - fn popularity(&self, lit: &[u8]) -> u32 { - if lit.len() < 2 { - return 0; - } - query_grams_of(lit) - .map(|k| self.counts.get(&k).copied().unwrap_or(0)) - .min() - .unwrap_or(0) - } -} - -/// `floor(log2(c))`, and 0 for the "nothing known" counts 0 and 1 — so an -/// unseen literal is never penalised, and the penalty grows with an order of -/// magnitude rather than with a raw count that spans 1..10^6. -fn log2_floor(c: u32) -> isize { - if c <= 1 { - 0 - } else { - (31 - c.leading_zeros()) as isize - } -} - -/// Selectivity score of a candidate anchor run. A low-entropy run (a constant -/// byte like a zero/0xFF pad, or a 2-symbol repeat) matches repetitive content -/// — PE padding, BSS — millions of times, so it is a terrible prefilter even -/// when long. Down-rank such runs sharply so a shorter but varied run wins; a -/// varied run scores its length (longer = rarer = better). -/// -/// Length alone is not selectivity, though: `"http://"` is 7 varied bytes and -/// scored best by that rule, yet it is the single worst anchor on web/office -/// content — one 12 MB spreadsheet relationships part hit it 61,489 times, -/// while the same body's `":7878"` occurs essentially never. So a literal is -/// also penalised by how many signature bodies share it ([`GramStats`]), which -/// lets a shorter but rarer run win. `stats` is `None` while a body is first -/// parsed, because the count is not known until the whole set has been read — -/// the choice is then revisited at build time. -pub(super) fn anchor_score(b: &[u8], stats: Option<&GramStats>) -> isize { - let mut seen = [false; 256]; - let mut distinct = 0usize; - for &x in b { - if !seen[x as usize] { - seen[x as usize] = true; - distinct += 1; - } - } - let structural = match distinct { - 0 | 1 => 1, // constant run: near-useless anchor - 2 => 3.min(b.len()), // 2-symbol repeat (e.g. ababab): weak - _ => b.len(), // varied: length is the selectivity - } as isize; - // Both terms are log-scaled — length stands in for how rare a run should be, - // the popularity count for how common it actually is — so the difference - // reads as a selectivity estimate. Deliberately signed and unsaturated: - // clamping at zero puts every ubiquitous literal in a tie that the length - // tie-break then resolves backwards, handing the anchor to the *longest* - // common run (`"http://"` at 7 bytes beating `"umber:"` at 6, which is - // exactly the pairing that costs a 12 MB relationships part 61,489 hits). - structural - stats.map_or(0, |s| log2_floor(s.popularity(b))) -} - pub(super) fn pick_anchor( elems: &[Elem], allow_internal: bool, stats: Option<&GramStats>, ) -> Option { - // Pick the most *selective* literal run as the Aho-Corasick anchor (highest + // Pick the most *selective* literal run as the anchor (highest // [`anchor_score`], tie-break longer), not merely the longest: a long // constant run is a far worse prefilter than a shorter varied one. Fewer - // spurious AC hits → fewer verifies, the dominant scan cost. + // spurious anchor hits, fewer verifies, the dominant scan cost. // // The anchor in the *fixed-width prefix* (before the first variable gap) is - // free to verify — its distance back to the pattern start (`len`) is exact. + // free to verify: its distance back to the pattern start (`len`) is exact. // But some patterns have only a useless literal there (a zero-run) while a // rare literal sits past a gap. For those, anchoring past the gap (`Internal`) - // and verifying backward across it slashes AC hits by orders of magnitude. + // and verifying backward across it cuts anchor hits by orders of magnitude. // We only take an internal anchor when it is *strictly more selective* than // the best fixed-prefix one and the pattern offset is `*` (so the floating - // start need not satisfy a fixed offset) — passed via `allow_internal`. + // start need not satisfy a fixed offset), passed via `allow_internal`. let mut fixed = 0usize; let mut in_fixed_prefix = true; // (score, len, prefix) for the best fixed-prefix candidate and the best diff --git a/crates/exav-core/src/engine/pins.rs b/crates/exav-core/src/engine/pins.rs new file mode 100644 index 00000000..5cac8679 --- /dev/null +++ b/crates/exav-core/src/engine/pins.rs @@ -0,0 +1,272 @@ +//! The bodies an offset pins to a few places in an object, checked there +//! rather than looked for everywhere. +//! +//! A subsignature written `0:000000` or `EP+0:0000...` can only match where +//! its offset says, but its anchor, a few zero bytes, occurs throughout any +//! executable: looked for by the sweep, it was most of the anchor hits of a +//! PE, each rejected by the offset check. Such a body is instead checked at +//! the starts its offset allows once the object's length and layout are +//! known: a table lookup of the bytes where its anchor would be, per group of +//! bodies sharing an offset, or for the few whose window is wide, a search of +//! that window for the anchor. + +use crate::byte_source::ByteSource; +use crate::pe::PeLayout; + +use super::{start_set, Body, Elem, Offset, OffsetKind, Prefix, StartSet, FOLD}; + +/// Widest window of starts a pinned body may have: past it the sweep's one +/// pass over the object costs less than the searches. +const MAX_WINDOW: u64 = 4096; + +/// Widest window checked a start at a time; wider ones are searched. +const MAX_STEPPED: u64 = 64; + +/// Bodies sharing one offset, keyed by where their anchor sits past a start +/// and its first two bytes, folded. +#[derive(serde::Serialize, serde::Deserialize)] +struct Group { + offset: Offset, + /// Every distinct distance from a start to an anchor, ascending. Derived, + /// not stored. + #[serde(skip)] + dists: Vec, + /// `(distance, key, body)`, sorted. + entries: Vec<(u32, u16, u32)>, +} + +/// The pinned bodies of an engine. +#[derive(Default, serde::Serialize, serde::Deserialize)] +pub(super) struct Pins { + groups: Vec, + /// Bodies whose window is too wide to step through: each searched for + /// its anchor in it. + wide: Vec, + /// Whether any is case-insensitive, so verifying it reads the object + /// lowercased. Derived, not stored. + #[serde(skip)] + nocase: bool, +} + +/// Whether `body` is pinned, and where its anchor sits past its start. +fn pinned(body: &Body) -> Option<(u64, u32)> { + let Offset::Constrained(k) = &body.offset else { + return None; + }; + let shift = match **k { + OffsetKind::Abs { shift, .. } + | OffsetKind::Eof { shift, .. } + | OffsetKind::Ep { shift, .. } + | OffsetKind::Sec { shift, .. } + | OffsetKind::SecLast { shift, .. } => shift, + OffsetKind::SecIn { .. } | OffsetKind::VersionInfo => return None, + }; + let dist = match body.prefix { + Prefix::Fixed { len, .. } => len, + Prefix::Floating { .. } => 0, + Prefix::Internal { .. } => return None, + }; + (shift <= MAX_WINDOW).then_some((shift, dist)) +} + +/// The anchor of `body`, which [`pinned`] took. +fn anchor(body: &Body) -> &[u8] { + match &body.elems.as_deref().unwrap_or_default()[body.prefix.anchor_idx() as usize] { + Elem::Bytes(b) => b, + _ => unreachable!("an anchor is a literal"), + } +} + +fn key(a: &[u8]) -> u16 { + u16::from_le_bytes([FOLD[a[0] as usize], FOLD[a[1] as usize]]) +} + +impl Pins { + /// Take the pinned bodies out of `bodies`' anchoring: `true` for each one + /// taken, which the anchor index then leaves out. A + /// pure literal gets its anchor back as its pattern, since nothing else + /// will hold it. + pub(super) fn take(bodies: &mut [Body], anchors: impl Fn(usize) -> Vec) -> (Pins, Vec) { + let mut pins = Pins::default(); + let mut taken = vec![false; bodies.len()]; + let mut by_offset: std::collections::BTreeMap = Default::default(); + #[cfg(test)] + if super::tests::NO_PINS.get() { + return (pins, taken); + } + for (id, body) in bodies.iter_mut().enumerate() { + let Some((shift, dist)) = pinned(body) else { + continue; + }; + if body.elems.is_none() { + body.elems = Some(vec![Elem::Bytes(anchors(id))]); + body.prefix = Prefix::Fixed { anchor_idx: 0, len: 0 }; + } + taken[id] = true; + if shift > MAX_STEPPED { + pins.wide.push(id as u32); + continue; + } + let g = *by_offset.entry(body.offset.clone()).or_insert_with(|| { + pins.groups.push(Group { + offset: body.offset.clone(), + dists: Vec::new(), + entries: Vec::new(), + }); + pins.groups.len() - 1 + }); + pins.groups[g].entries.push((dist, key(anchor(body)), id as u32)); + } + pins.derive(bodies); + (pins, taken) + } + + /// What is derived from the stored pins and `bodies`. + fn derive(&mut self, bodies: &[Body]) { + for g in &mut self.groups { + g.entries.sort_unstable(); + g.dists = g.entries.iter().map(|e| e.0).collect(); + g.dists.dedup(); + } + let nocase = self.bodies().any(|b| bodies[b as usize].nocase); + self.nocase = nocase; + } + + /// Every pinned body. + pub(super) fn bodies(&self) -> impl Iterator + '_ { + self.groups.iter().flat_map(|g| g.entries.iter().map(|e| e.2)).chain(self.wide.iter().copied()) + } + + pub(super) fn any_nocase(&self) -> bool { + self.nocase + } + + /// Pins as read from a database, checked against `bodies`: every stored + /// body is one, pinned, with a literal anchor of two bytes or more, and + /// stored where [`Self::take`] puts it. + pub(super) fn load(mut self, bodies: &[Body]) -> Option { + let at = |b: u32| { + let body = bodies.get(b as usize)?; + let (shift, dist) = pinned(body)?; + let a = body.elems.as_deref()?.get(body.prefix.anchor_idx() as usize)?; + matches!(a, Elem::Bytes(a) if a.len() >= 2).then_some((body, shift, dist)) + }; + let grouped = self.groups.iter().all(|g| { + g.entries.iter().all(|&(d, k, b)| { + at(b).is_some_and(|(body, shift, dist)| { + shift <= MAX_STEPPED && dist == d && body.offset == g.offset && key(anchor(body)) == k + }) + }) + }); + if !grouped || !self.wide.iter().all(|&b| at(b).is_some_and(|(_, shift, _)| shift > MAX_STEPPED)) { + return None; + } + self.derive(bodies); + Some(self) + } + + /// Hand `f` every place a pinned body of `bodies` could match in `buf`, + /// as `(body, anchor start)`, for the bodies `wanted` keeps: its anchor's + /// first two bytes are there, or for a wide window, the whole anchor. + /// `lower` is `buf` lowercased. `false` when `f` asks to stop. + pub(super) fn visit( + &self, + bodies: &[Body], + buf: &B, + lower: &L, + layout: Option<&PeLayout>, + wanted: &dyn Fn(&Body) -> bool, + f: &mut dyn FnMut(usize, usize) -> bool, + ) -> bool { + let len = buf.len() as u64; + for g in &self.groups { + let Some(StartSet::Range(lo, hi)) = start_set(&g.offset, len, layout) else { + continue; + }; + for s in lo..=hi.min(len) { + for &d in &g.dists { + let at = s + d as u64; + let w = buf.window(at as usize, 2); + if w.len() < 2 { + continue; + } + let k = key(&w); + let from = g.entries.partition_point(|e| (e.0, e.1) < (d, k)); + for &(_, _, b) in g.entries[from..].iter().take_while(|e| (e.0, e.1) == (d, k)) { + if wanted(&bodies[b as usize]) && !f(b as usize, at as usize) { + return false; + } + } + } + } + } + for &b in &self.wide { + let body = &bodies[b as usize]; + if !wanted(body) { + continue; + } + let Some(StartSet::Range(lo, hi)) = start_set(&body.offset, len, layout) else { + continue; + }; + let dist = pinned(body).map_or(0, |(_, d)| d) as u64; + let a = anchor(body); + let from = lo.saturating_add(dist); + let to = hi.saturating_add(dist).saturating_add(a.len() as u64).min(len); + if from >= to { + continue; + } + let (from_u, n) = (from as usize, (to - from) as usize); + let (w, finder) = match body.nocase { + true => (lower.window(from_u, n), memchr::memmem::Finder::new(&a.to_ascii_lowercase()).into_owned()), + false => (buf.window(from_u, n), memchr::memmem::Finder::new(a).into_owned()), + }; + // Every occurrence, overlapping ones included: in a run, each + // start is a match of its own. + let mut p = 0; + while let Some(q) = finder.find(&w[p..]) { + if !f(b as usize, from as usize + p + q) { + return false; + } + p += q + 1; + } + } + true + } +} + +#[cfg(test)] +mod tests { + use super::super::EngineBuilder; + use super::*; + + /// Pins read back are refused unless every entry is a pinned body, stored + /// where [`Pins::take`] puts it. + #[test] + fn a_misplaced_pin_is_refused() { + let mut b = EngineBuilder::new(); + b.add_ndb( + "P.Abs:0:0:4d5a9000\nP.Ep:1:EP+2:e8000000\nP.Wide:0:10,500:70696e6e6564\nP.Any:0:*:616e79776865726521", + false, + ); + let e = b.build(); + let copy = || rmp_serde::from_slice::(&rmp_serde::to_vec(&e.pins).unwrap()).unwrap(); + assert_eq!((copy().groups.len(), copy().wide.len()), (2, 1)); + assert!(copy().load(&e.bodies).is_some()); + let misplaced: [&dyn Fn(&mut Pins); 6] = [ + &|p: &mut Pins| p.groups[0].entries[0].0 += 1, + &|p: &mut Pins| p.groups[0].entries[0].1 ^= 1, + &|p: &mut Pins| p.groups[0].offset = p.groups[1].offset.clone(), + &|p: &mut Pins| p.groups[0].entries[0].2 = 3, + &|p: &mut Pins| p.wide.push(p.groups[0].entries[0].2), + &|p: &mut Pins| { + let w = p.wide.pop().unwrap(); + p.groups[0].entries.push((0, 0, w)); + }, + ]; + for (i, m) in misplaced.iter().enumerate() { + let mut p = copy(); + m(&mut p); + assert!(p.load(&e.bodies).is_none(), "misplacement {i}"); + } + } +} diff --git a/crates/exav-core/src/grams/mod.rs b/crates/exav-core/src/grams/mod.rs new file mode 100644 index 00000000..86d742f7 --- /dev/null +++ b/crates/exav-core/src/grams/mod.rs @@ -0,0 +1,976 @@ +//! Literal anchors found in an object, through a few filters that stay in cache +//! instead of an automaton. +//! +//! An automaton is stepped at every byte, and one over a million anchors is +//! tens of megabytes of states, each step a cache miss. Here each anchor is +//! indexed by windows of its bytes instead: the anchor itself when it is 1 to +//! 3 bytes long, 4 bytes of it when it is 4 or 5, and when it is longer, which +//! is nearly all of them, [`STRIDE`] consecutive 6-byte windows (4 bytes again +//! if it has no such windows that are not one repeated byte). A scan reads +//! the bytes at every position of the object once, folded to lowercase. The +//! short windows are looked up at every position, first in one table of the +//! first two bytes of every short window; the long ones only at every +//! [`STRIDE`]th position, which one of an anchor's consecutive windows always +//! starts at, in a filter of a few megabytes. Only a position that passes is +//! looked up in the width's table of windows, and every anchor behind that +//! window compared whole. +//! +//! An anchor's windows are the ones the signature set shares least, since +//! every anchor behind a window is compared wherever the window occurs. +//! Windows are case-folded for the case-sensitive anchors as for the others, +//! so one lookup serves both; a case-sensitive anchor is then compared exactly. +//! +//! An anchor made of one repeated byte is indexed by its first bytes. Inside +//! a long run of its byte it matches at every position, and there it is +//! reported as one run of starts rather than looked up position by position: +//! a run of zeros costs what its edges do ([`sweep`]). +//! +//! The index is stored as its anchors and, per width, the windows they are +//! found by; the tables a scan looks up are derived from those at load. + +mod rarity; +mod sweep; + +#[cfg(test)] +pub(crate) use rarity::FILTER_MIN_BODIES; +pub(crate) use rarity::{anchor_score, GramStats}; +#[cfg(feature = "yara")] +pub(crate) use rarity::structural; +pub(crate) use sweep::sweep; +#[cfg(test)] +pub(crate) use sweep::{MIN_SKIP_RUN, NO_RUN_COLLAPSE, SWEEP_WINDOW}; + +use crate::byte_source::ByteSource; + +/// ASCII lowercasing, as a table. +pub(crate) static FOLD: [u8; 256] = { + let mut t = [0u8; 256]; + let mut i = 0; + while i < 256 { + t[i] = (i as u8).to_ascii_lowercase(); + i += 1; + } + t +}; + +/// Bytes of each width of window, narrowest first. +const WIDTH: [usize; 5] = [1, 2, 3, 4, 6]; + +/// The classes of window the two-byte table answers for: exactly for the +/// first two, as a first filter for the others. +const SHORT: usize = 4; + +/// The class of the widest windows, which index the long anchors. +const LONG: usize = 4; + +/// The widest window: positions whose window lies inside a skipped run hold +/// no anchor but one of its byte. +pub(crate) const GRAM: usize = WIDTH[LONG]; + +/// One position in this many is looked up for the long windows; a long anchor +/// is indexed by as many consecutive windows, so one of them starts there. +/// Fewer lookups against more keys in the filter. +const STRIDE: usize = 2; + +/// Positions read and looked up together: their filter words are loaded +/// before any is tested, so the processor waits on them at once. +const BATCH: usize = 8; + +/// An empty slot of a window table. +const EMPTY: u32 = u32::MAX; + +/// Bits of filter per distinct window. +const FILTER_BITS_PER_KEY: usize = 64; + +/// Largest filter, in words: 8 MiB. +const MAX_FILTER_WORDS: usize = 1 << 20; + +/// One anchor behind a window: its bytes in the pool, lowercased when it is +/// case-insensitive, and where the window starts in it. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Entry { + off: u32, + len: u32, + back: u32, + /// The partition, with [`NOCASE`] set when the anchor is case-insensitive. + part: u32, + value: u32, + /// For an anchor of one repeated byte, its index in [`Grams::uniform`] + /// plus one; zero for any other. Derived, not stored. + run: u32, +} + +/// [`Entry::part`]'s flag for a case-insensitive anchor. +const NOCASE: u32 = 1 << 31; + +/// The windows of one width. +#[derive(Default)] +struct Class { + /// Two bits per window, both in one word. Empty for the classes the + /// two-byte table answers for exactly. + filter: Vec, + /// The distinct windows, open-addressed: an index into `keys`, or [`EMPTY`]. + slots: Vec, + keys: Vec, + /// Window `i`'s anchors are `entries[start[i]..start[i + 1]]`. + start: Vec, + /// Sorted by window. + entries: Vec, +} + +/// An anchor of one byte repeated, reported by runs of it. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Uniform { + /// The byte, lowercased when case-insensitive. + byte: u8, + len: u32, + part: u32, + value: u32, +} + +/// The anchors of every partition. +#[derive(Default)] +pub(crate) struct Grams { + classes: [Class; 5], + pool: Vec, + uniform: Vec, + /// Per first two bytes of a window, folded: bit `c` when a window of + /// class `c < SHORT` starts with them. + first: Vec, +} + +/// One object's scan: the partitions to report, the object, and per anchor of +/// one repeated byte, the start below which its run has been reported. +pub(crate) struct Scan<'a, H: ?Sized> { + active: &'a [bool], + hay: &'a H, + marks: Vec, +} + +/// An anchor to index: its partition, case, value in that partition, and +/// bytes, which the index lowercases when case-insensitive. +pub(crate) struct IndexAnchor<'a> { + pub(crate) part: u32, + pub(crate) nocase: bool, + pub(crate) value: u32, + pub(crate) bytes: &'a [u8], +} + +/// Spreads every bit of a window into the high half and the low bits. +#[inline(always)] +fn mix(k: u64) -> u64 { + let h = k.wrapping_mul(0x9E37_79B9_7F4A_7C15); + h ^ (h >> 32) +} + +/// ASCII-lowercases each byte of `x`. +#[inline(always)] +fn fold8(x: u64) -> u64 { + const L: u64 = 0x0101_0101_0101_0101; + let ascii = !x & (0x80 * L); + let low = x & (0x7f * L); + let from_a = low + (0x80 - b'A' as u64) * L; + let past_z = low + (0x80 - b'Z' as u64 - 1) * L; + let upper = from_a & !past_z & ascii; + x | (upper >> 2) +} + +/// The bits of a little-endian word that hold a window of `w` bytes. +const fn mask(w: usize) -> u64 { + if w >= 8 { + u64::MAX + } else { + (1u64 << (8 * w)) - 1 + } +} + +/// The first bytes of `s`, up to eight, folded, as a little-endian word. +fn word_of(s: &[u8]) -> u64 { + let mut b = [0u8; 8]; + for (d, &c) in b.iter_mut().zip(s) { + *d = FOLD[c as usize]; + } + u64::from_le_bytes(b) +} + +fn is_uniform(a: &[u8]) -> bool { + a.iter().all(|&b| b == a[0]) +} + +/// Where an anchor is indexed: its class, and where in it each of its windows +/// starts, the ones `stats` says the set shares least. +fn place(a: &[u8], stats: &GramStats) -> (usize, Vec) { + let cost = |w: &[u8]| rarity::window_cost(w, stats); + if a.len() >= GRAM && !is_uniform(a) { + // `STRIDE` consecutive windows, none of one repeated byte, since a + // position inside a run is not looked up: the run whose dearest + // window costs least. + let costs: Vec<_> = a + .windows(GRAM) + .map(|w| (!is_uniform(w)).then(|| cost(w))) + .collect(); + let best = (0..costs.len().saturating_sub(STRIDE - 1)) + .filter_map(|b| { + costs[b..b + STRIDE] + .iter() + .copied() + .collect::>>() + .map(|c| (c, b)) + }) + .min_by_key(|(c, b)| (c.iter().max().copied(), *b)); + if let Some((_, b)) = best { + return (LONG, (b..b + STRIDE).collect()); + } + } + let c = match a.len() { + 1 => 0, + 2 => 1, + 3 => 2, + _ => 3, + }; + let w = WIDTH[c]; + if is_uniform(a) { + return (c, vec![0]); + } + let back = a + .windows(w) + .enumerate() + .filter(|(_, x)| !is_uniform(x)) + .min_by_key(|&(b, x)| (cost(x), b)) + .map(|(b, _)| b) + .expect("an anchor of more than one byte value has a varied window"); + (c, vec![back]) +} + +impl Class { + /// The class of width `w` over `entries`, sorted by window, whose bytes + /// are in `pool`. + fn build(w: usize, pool: &[u8], entries: Vec) -> Class { + let mut c = Class { + entries, + ..Class::default() + }; + if c.entries.is_empty() { + return c; + } + for (i, e) in c.entries.iter().enumerate() { + let k = window(pool, e, w); + if c.keys.last() != Some(&k) { + c.keys.push(k); + c.start.push(i as u32); + } + } + c.start.push(c.entries.len() as u32); + if w > 2 { + let words = (c.keys.len() * FILTER_BITS_PER_KEY / 64) + .next_power_of_two() + .clamp(64, MAX_FILTER_WORDS); + c.filter = vec![0; words]; + } + c.slots = vec![EMPTY; (c.keys.len() * 2).next_power_of_two()]; + for (i, &k) in c.keys.iter().enumerate() { + let h = mix(k); + if !c.filter.is_empty() { + let w = c.word(h); + c.filter[w] |= Self::bits(h); + } + let mut s = c.slot(h); + while c.slots[s] != EMPTY { + s = (s + 1) & (c.slots.len() - 1); + } + c.slots[s] = i as u32; + } + c + } + + #[inline(always)] + fn word(&self, h: u64) -> usize { + (h >> (64 - self.filter.len().trailing_zeros())) as usize + } + + #[inline(always)] + fn bits(h: u64) -> u64 { + 1 << (h & 63) | 1 << ((h >> 6) & 63) + } + + #[inline(always)] + fn slot(&self, h: u64) -> usize { + (h >> 12) as usize & (self.slots.len() - 1) + } + + /// Whether window `k` may be indexed. + #[inline(always)] + fn maybe(&self, k: u64) -> bool { + let h = mix(k); + let b = Self::bits(h); + self.filter[self.word(h)] & b == b + } + + /// The anchors behind window `k`. + fn entries(&self, k: u64) -> &[Entry] { + if self.slots.is_empty() { + return &[]; + } + let mut s = self.slot(mix(k)); + loop { + let i = self.slots[s]; + if i == EMPTY { + return &[]; + } + let i = i as usize; + if self.keys[i] == k { + return &self.entries[self.start[i] as usize..self.start[i + 1] as usize]; + } + s = (s + 1) & (self.slots.len() - 1); + } + } +} + +/// The window of `w` bytes `e` is indexed by, folded. +fn window(pool: &[u8], e: &Entry, w: usize) -> u64 { + let at = (e.off + e.back) as usize; + word_of(&pool[at..at + w]) & mask(w) +} + +impl Grams { + /// Index `anchors`, each by the windows `stats` says the set shares least. + pub(crate) fn build(anchors: &[IndexAnchor], stats: &GramStats) -> Grams { + let mut pool = Vec::new(); + let mut lists: [Vec; 5] = Default::default(); + for a in anchors { + let off = pool.len() as u32; + pool.extend( + a.bytes + .iter() + .map(|&b| if a.nocase { FOLD[b as usize] } else { b }), + ); + let (c, backs) = place(&pool[off as usize..], stats); + lists[c].extend(backs.into_iter().map(|back| Entry { + off, + len: a.bytes.len() as u32, + back: back as u32, + part: a.part | if a.nocase { NOCASE } else { 0 }, + value: a.value, + run: 0, + })); + } + for (c, list) in lists.iter_mut().enumerate() { + list.sort_by_cached_key(|e| (window(&pool, e, WIDTH[c]), e.part, e.value, e.back)); + } + Grams::index(pool, lists) + } + + /// The tables a scan looks up, derived from the anchors in `pool` and + /// each class's entries, sorted by window. + fn index(pool: Vec, lists: [Vec; 5]) -> Grams { + let mut g = Grams { + pool, + ..Grams::default() + }; + for (c, mut list) in lists.into_iter().enumerate() { + for e in &mut list { + let a = &g.pool[e.off as usize..(e.off + e.len) as usize]; + if e.back == 0 && is_uniform(a) { + g.uniform.push(Uniform { + byte: a[0], + len: e.len, + part: e.part, + value: e.value, + }); + e.run = g.uniform.len() as u32; + } + } + g.classes[c] = Class::build(WIDTH[c], &g.pool, list); + } + let mut t = vec![0u8; 1 << 16]; + for (c, class) in g.classes[..SHORT].iter().enumerate() { + for &k in &class.keys { + match WIDTH[c] { + 1 => (0..256).for_each(|second| t[k as usize | second << 8] |= 1 << c), + _ => t[(k & 0xffff) as usize] |= 1 << c, + } + } + } + g.first = t; + g + } + + pub(crate) fn is_empty(&self) -> bool { + self.classes.iter().all(|c| c.keys.is_empty()) + } + + /// The state of one object's scan: [`Self::scan`] and the rest take it. + pub(crate) fn start<'a, H: ByteSource + ?Sized>( + &self, + active: &'a [bool], + hay: &'a H, + ) -> Scan<'a, H> { + Scan { + active, + hay, + marks: vec![0; self.uniform.len()], + } + } + + /// Hand `f` every anchor whose window starts at a position of + /// `buf[from..to]` where eight bytes can be read. `buf` is the window of + /// the object at `at`. `false` when `f` asks to stop. + pub(crate) fn scan( + &self, + s: &mut Scan, + buf: &[u8], + at: usize, + from: usize, + to: usize, + f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool, + ) -> bool { + let to = to.min(buf.len().saturating_sub(7)); + let upto = at + to; + let mut i = from; + while i + BATCH <= to { + let block: &[u8; BATCH + 7] = buf[i..i + BATCH + 7].try_into().unwrap(); + let x: [u64; BATCH] = std::array::from_fn(|j| { + fold8(u64::from_le_bytes(block[j..j + 8].try_into().unwrap())) + }); + if !self.probe_block(s, &x, at + i, buf, at, upto, f) { + return false; + } + i += BATCH; + } + for p in i..to { + let x = fold8(u64::from_le_bytes(buf[p..p + 8].try_into().unwrap())); + let long = (at + p).is_multiple_of(STRIDE) && self.long_maybe(x); + if !self.probe(s, x, long, at + p, buf, at, upto, f) { + return false; + } + } + true + } + + /// [`Self::scan`] at the positions `from..to` of the object, reading them + /// from it: those whose eight bytes run past a window held in memory. + pub(crate) fn scan_source( + &self, + s: &mut Scan, + from: usize, + to: usize, + f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool, + ) -> bool { + let to = to.min(s.hay.len()); + if from >= to { + return true; + } + // Past the end of the object the bytes are zeros: an anchor that + // would need them is refused by its length when compared. + let mut buf = s.hay.window(from, to - from + 7).into_owned(); + buf.resize(to - from + 7, 0); + self.scan(s, &buf, from, 0, to - from, f) + } + + /// Whether the long window of `x` may be indexed. + #[inline(always)] + fn long_maybe(&self, x: u64) -> bool { + let long = &self.classes[LONG]; + !long.filter.is_empty() && long.maybe(x & mask(GRAM)) + } + + /// [`Self::probe`] at the [`BATCH`] positions from `pos`, whose first eight + /// bytes, folded, are `x`: every long window's filter word is read before + /// any is tested. + #[inline(always)] + #[allow(clippy::too_many_arguments)] + fn probe_block( + &self, + s: &mut Scan, + x: &[u64; BATCH], + pos: usize, + buf: &[u8], + at: usize, + upto: usize, + f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool, + ) -> bool { + let long = &self.classes[LONG]; + let mut hit = [false; BATCH]; + if !long.filter.is_empty() { + let first = (STRIDE - pos % STRIDE) % STRIDE; + let mut h = [0u64; BATCH]; + let mut words = [0u64; BATCH]; + for j in (first..BATCH).step_by(STRIDE) { + h[j] = mix(x[j] & mask(GRAM)); + words[j] = long.filter[long.word(h[j])]; + } + for j in (first..BATCH).step_by(STRIDE) { + let b = Class::bits(h[j]); + hit[j] = words[j] & b == b; + } + } + (0..BATCH).all(|j| self.probe(s, x[j], hit[j], pos + j, buf, at, upto, f)) + } + + /// Look up the short windows at `pos`, whose first eight bytes, folded, + /// are `x`, and resolve those that pass, then the long one when `long` + /// says its filter passed. + #[inline(always)] + #[allow(clippy::too_many_arguments)] + fn probe( + &self, + s: &mut Scan, + x: u64, + long: bool, + pos: usize, + buf: &[u8], + at: usize, + upto: usize, + f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool, + ) -> bool { + let first = self.first[(x & 0xffff) as usize]; + if first != 0 { + for (c, class) in self.classes[..SHORT].iter().enumerate() { + let k = x & mask(WIDTH[c]); + // A window of two bytes or fewer is answered by the table. + if first & (1 << c) != 0 + && (WIDTH[c] <= 2 || class.maybe(k)) + && !self.resolve(s, class, k, pos, buf, at, upto, f) + { + return false; + } + } + } + !long + || self.resolve( + s, + &self.classes[LONG], + x & mask(GRAM), + pos, + buf, + at, + upto, + f, + ) + } + + /// Report the anchors of one repeated byte that match at the positions + /// `from..to`, inside a run of byte `b` that ends at `end`: each as one + /// run of starts. + pub(crate) fn run( + &self, + s: &Scan, + b: u8, + from: usize, + to: usize, + end: usize, + f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool, + ) -> bool { + for &u in &self.uniform { + let nocase = u.part & NOCASE != 0; + let byte_matches = match nocase { + true => FOLD[b as usize] == u.byte, + false => b == u.byte, + }; + let part = (u.part & !NOCASE) as usize; + if !byte_matches || !s.active[part] { + continue; + } + // A case-insensitive anchor goes on matching past the run, into + // the other case of its letter. + let len = u.len as usize; + let reach = match nocase { + true => { + end + s + .hay + .window(end, len) + .iter() + .take_while(|&&c| FOLD[c as usize] == u.byte) + .count() + } + false => end, + }; + let last = to.min((reach + 1).saturating_sub(len)); + if last > from && !f(part, u.value, from, len, last - from) { + return false; + } + } + true + } + + /// Compare every active anchor behind window `k` of `class`, found at + /// `pos`, and report those that match. An anchor of one repeated byte is + /// reported for every start of the run it is in, up to `upto`, at once, + /// and not again at those starts. Out of line: the loop that calls it + /// keeps its state in registers. + #[allow(clippy::too_many_arguments)] + #[inline(never)] + fn resolve( + &self, + sc: &mut Scan, + class: &Class, + k: u64, + pos: usize, + buf: &[u8], + at: usize, + upto: usize, + f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool, + ) -> bool { + let hay = sc.hay; + let byte = |q: usize| match q.checked_sub(at) { + Some(i) if i < buf.len() => buf[i], + _ => hay.window(q, 1).first().copied().unwrap_or(0), + }; + for e in class.entries(k) { + let part = (e.part & !NOCASE) as usize; + if !sc.active[part] { + continue; + } + let Some(s) = pos.checked_sub(e.back as usize) else { + continue; + }; + if e.run != 0 && s < sc.marks[e.run as usize - 1] { + continue; + } + let len = e.len as usize; + if s + len > hay.len() { + continue; + } + let want = &self.pool[e.off as usize..e.off as usize + len]; + let nocase = e.part & NOCASE != 0; + let held; + let got: &[u8] = if s >= at && s + len <= at + buf.len() { + &buf[s - at..s - at + len] + } else { + held = hay.window(s, len); + &held + }; + let same = match nocase { + true => { + got.len() == len && got.iter().zip(want).all(|(&g, &w)| FOLD[g as usize] == w) + } + false => got == want, + }; + if !same { + continue; + } + let mut count = 1; + if e.run != 0 { + // The rest of the run, as far as a start before `upto` can + // reach. + let (b, reach) = (want[0], (upto + len - 1).min(hay.len())); + let mut q = s + len; + while q < reach + && (if nocase { + FOLD[byte(q) as usize] + } else { + byte(q) + }) == b + { + q += 1; + } + let last = (q + 1 - len).min(upto).max(s + 1); + sc.marks[e.run as usize - 1] = last; + count = last - s; + } + if !f(part, e.value, s, len, count) { + return false; + } + } + true + } + + /// Write the index: its anchors, and per class, the entries that index + /// them. + pub(crate) fn write(&self, w: &mut W) -> std::io::Result<()> { + use crate::database::{enc_bytes, enc_u32s}; + enc_bytes(&self.pool, w)?; + for c in &self.classes { + let entries: Vec = c + .entries + .iter() + .flat_map(|e| [e.off, e.len, e.back, e.part, e.value]) + .collect(); + enc_u32s(&entries, w)?; + } + Ok(()) + } + + /// Read what [`Self::write`] wrote, checked against `values[p]`, the number + /// of values of partition `p`, so a scan can index with what it holds. + pub(crate) fn read( + r: &mut R, + values: &[usize], + ) -> std::io::Result { + use crate::database::{dec_bytes, dec_u32s}; + let bad = || { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "corrupt signature database (anchor index)", + ) + }; + let pool = dec_bytes(r)?; + let mut lists: [Vec; 5] = Default::default(); + for (c, list) in lists.iter_mut().enumerate() { + let flat = dec_u32s(r)?; + let (chunks, rest) = flat.as_chunks::<5>(); + if !rest.is_empty() { + return Err(bad()); + } + let w = WIDTH[c]; + *list = chunks + .iter() + .map(|&[off, len, back, part, value]| Entry { + off, + len, + back, + part, + value, + run: 0, + }) + .collect(); + let fits = |e: &Entry| { + (e.back as usize) + .checked_add(w) + .is_some_and(|end| end <= e.len as usize) + && (e.off as usize) + .checked_add(e.len as usize) + .is_some_and(|end| end <= pool.len()) + && values + .get((e.part & !NOCASE) as usize) + .is_some_and(|&n| (e.value as usize) < n) + }; + if !list.iter().all(fits) + || !list + .windows(2) + .all(|p| window(&pool, &p[0], w) <= window(&pool, &p[1], w)) + { + return Err(bad()); + } + } + Ok(Grams::index(pool, lists)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::byte_source::BlockCache; + + #[test] + fn fold8_is_ascii_lowercasing() { + for b in 0..=255u8 { + let x = u64::from_le_bytes([b, b'A', b, b'z', 0x80 | b, b'@', b'[', b]); + let want = u64::from_le_bytes(x.to_le_bytes().map(|c| c.to_ascii_lowercase())); + assert_eq!(fold8(x), want, "{b:#x}"); + } + } + + /// `anchors` as `(partition, nocase, bytes)`, indexed with statistics over + /// themselves. + fn index(anchors: &[(u32, bool, Vec)]) -> Grams { + let stats = GramStats::build(anchors.len(), |f| anchors.iter().for_each(|(.., a)| f(a))); + let index: Vec = anchors + .iter() + .enumerate() + .map(|(v, (part, nocase, bytes))| IndexAnchor { + part: *part, + nocase: *nocase, + value: v as u32, + bytes, + }) + .collect(); + Grams::build(&index, &stats) + } + + /// Every place each anchor occurs in `hay`, in its case, for the active + /// partitions: what a sweep has to find. + fn searched( + anchors: &[(u32, bool, Vec)], + hay: &[u8], + active: &[bool], + ) -> Vec<(usize, u32, usize, usize)> { + let mut want = Vec::new(); + for (v, (part, nocase, a)) in anchors.iter().enumerate() { + if !active[*part as usize] || a.len() > hay.len() { + continue; + } + for s in 0..=hay.len() - a.len() { + let w = &hay[s..s + a.len()]; + if (*nocase && w.eq_ignore_ascii_case(a)) || (!*nocase && w == &a[..]) { + want.push((*part as usize, v as u32, s, a.len())); + } + } + } + want.sort_unstable(); + want + } + + /// A sweep's hits, expanded and sorted as [`searched`] gives them, and how + /// many reports they came as. + fn swept( + g: &Grams, + hay: &H, + active: &[bool], + ) -> (Vec<(usize, u32, usize, usize)>, usize) { + let (mut got, mut reports) = (Vec::new(), 0); + assert!(sweep(g, active, hay, &mut |p, v, s, l, n| { + reports += 1; + got.extend((s..s + n).map(|s| (p, v, s, l))); + true + })); + got.sort_unstable(); + (got, reports) + } + + /// Every anchor, of every width, is found at every place it occurs and + /// nowhere else, in its case, in memory or read through a cache; a run of + /// one byte comes as a few reports; the index survives a round trip and + /// refuses a value its partition lacks. + #[test] + fn every_occurrence_is_found_once() { + let anchors: Vec<(u32, bool, Vec)> = [ + (0, false, &b"MZ\x90\x00\x03\x00"[..]), + (0, false, b"\x00\x00\x00\x00\x00\x01\x02"), + (1, true, b"http://evil"), + (1, false, b"Http://Evil"), + (0, false, b"abcdefgh"), + (1, false, b"xxabcdefghyy"), + (0, false, b"PK"), + (1, true, b"ab"), + (0, false, b"\x00\xe8"), + (1, false, b"xyz"), + (0, true, b"evil"), + (1, false, b"\x00\x00\x00\x00\x01"), + (0, false, b"\x00\x00\x00\x00\x00\x00\x00\x00"), + (1, false, b"--"), + (0, false, b"q"), + (1, true, b"Z"), + (0, false, b"\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00"), + ] + .iter() + .map(|&(p, n, a)| (p, n, a.to_vec())) + .collect(); + let g = index(&anchors); + let mut hay = + b"..MZ\x90\x00\x03\x00..HTTP://EVIL..Http://Evil..PK.Ab.aB.\x00\xe8xyzXYZq.z".to_vec(); + hay.extend_from_slice( + b"\x00\x00\x00\x00\x00\x00\x00\x00\x01\x02xxabcdefghyy..abcdefg----------", + ); + hay.extend(std::iter::repeat_n(0u8, 300)); + hay.extend_from_slice(b"\x01"); + hay.extend(std::iter::repeat_n(0u8, 20)); + hay.extend(std::iter::repeat_n(b'Z', 100)); + let src = BlockCache::with_sizes(std::io::Cursor::new(hay.clone()), 13, 8 * 13).unwrap(); + for active in [[true, true], [true, false], [false, true]] { + let want = searched(&anchors, &hay, &active); + assert_eq!(swept(&g, &hay[..], &active).0, want, "{active:?}"); + assert_eq!( + swept(&g, &src, &active).0, + want, + "{active:?}, read through a cache" + ); + } + let want = searched(&anchors, &hay, &[true, true]); + let (_, reports) = swept(&g, &hay[..], &[true, true]); + assert!( + reports < want.len() / 4, + "{reports} reports for {} matches", + want.len() + ); + let mut blob = Vec::new(); + g.write(&mut blob).unwrap(); + let back = Grams::read(&mut &blob[..], &[17, 17]).unwrap(); + for (a, b) in back.classes.iter().zip(&g.classes) { + assert_eq!( + (&a.keys, &a.start, &a.entries, &a.slots, &a.filter), + (&b.keys, &b.start, &b.entries, &b.slots, &b.filter) + ); + } + assert_eq!( + (back.pool, back.uniform, back.first), + (g.pool, g.uniform, g.first) + ); + assert!( + Grams::read(&mut &blob[..], &[17, 3]).is_err(), + "a value past its partition" + ); + } + + /// A case-insensitive anchor of one repeated letter matches across the end + /// of a skipped run of that letter into its other case, at every start + /// that reaches there. + #[test] + fn a_nocase_run_anchor_runs_on_into_the_other_case() { + let anchors: Vec<(u32, bool, Vec)> = vec![ + (0, true, b"aaaaaaa".to_vec()), + (0, false, b"aaaaaaa".to_vec()), + ]; + let g = index(&anchors); + for tail in [&b"Ax"[..], b"AAAAAAAAx", b"AaAax", b"A"] { + let mut hay = vec![b'a'; 100]; + hay.extend_from_slice(tail); + assert_eq!( + swept(&g, &hay[..], &[true]).0, + searched(&anchors, &hay, &[true]), + "{tail:?}" + ); + } + } + + /// Random anchors over a small alphabet, partitions and cases, swept over + /// haystacks full of runs and copies of them: exactly what a search for + /// each finds, in memory and through small cache blocks, across windows. + #[test] + fn random_anchors_are_found_as_searched() { + let mut state = 0x9a3f_e107_b2c4_d5e6_u64; + let mut next = |n: usize| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state % n as u64) as usize + }; + const ALPHA: &[u8] = b"aAb\x00\xff"; + for round in 0..120 { + let anchors: Vec<(u32, bool, Vec)> = (0..1 + next(30)) + .map(|_| { + let len = [1, 2, 3, 4, 5, 6, 7, 9, 14, 40][next(10)]; + let nocase = next(3) == 0; + let a: Vec = (0..len).map(|_| ALPHA[next(ALPHA.len())]).collect(); + (next(3) as u32, nocase, a) + }) + .collect(); + let g = index(&anchors); + let mut hay = Vec::new(); + while hay.len() < 3000 + next(3000) { + match next(5) { + 0 => hay.extend(std::iter::repeat_n( + ALPHA[next(ALPHA.len())], + [3, 70, 900, 2500][next(4)], + )), + 1 => { + let a = &anchors[next(anchors.len())].2; + hay.extend(a.iter().map(|&c| { + if next(2) == 0 { + c.to_ascii_uppercase() + } else { + c + } + })); + } + _ => hay.push(ALPHA[next(ALPHA.len())]), + } + } + let src = + BlockCache::with_sizes(std::io::Cursor::new(hay.clone()), 61, 4 * 61).unwrap(); + for active in [[true, true, true], [false, true, false]] { + let want = searched(&anchors, &hay, &active); + assert_eq!( + swept(&g, &hay[..], &active).0, + want, + "round {round} {active:?}" + ); + assert_eq!( + swept(&g, &src, &active).0, + want, + "round {round} {active:?}, through a cache" + ); + } + } + } +} diff --git a/crates/exav-core/src/grams/rarity.rs b/crates/exav-core/src/grams/rarity.rs new file mode 100644 index 00000000..2b9157e9 --- /dev/null +++ b/crates/exav-core/src/grams/rarity.rs @@ -0,0 +1,210 @@ +//! How common a literal is likely to be in a scanned object: what picks the +//! literal a signature is found by, and the window of it the index looks up. + +use super::FOLD; + +/// How many signature literals contain each short byte sequence, case-folded, +/// counted over the set being built. +/// +/// This is the selectivity signal, derived from the signature set itself rather +/// than shipped alongside it. A literal that hundreds of signatures are written +/// around (`"http://"`, `"target=\""`, `", +} + +/// Bits per presence filter. 2^28 bits is 32 MiB, ~8% occupancy against the +/// ~22M distinct grams of a full set, so few singletons are wrongly promoted to +/// the counting map, and a promotion costs only the memory, never a wrong +/// count, since the second pass counts what it actually sees. +const SEEN_BITS: usize = 1 << 28; + +/// Literal count from which the repeat filter is worth its fixed cost. Below it +/// the two bitsets would be 64 MiB to filter a handful of grams, which is what +/// every in-process engine build does, from a unit test to a small sig dir. +pub(crate) const FILTER_MIN_BODIES: usize = 100_000; + +/// A gram's key: its bytes folded, tagged with the length so `"ab"` and the +/// prefix of `"abcd"` cannot collide. Lengths are 2..=4, so this is lossless. +fn gram_key(g: &[u8]) -> u64 { + let mut v = 0u64; + for &b in g { + v = (v << 8) | FOLD[b as usize] as u64; + } + v | ((g.len() as u64) << 56) +} + +/// Every gram of `lit` to CREDIT while counting: all windows of 2, 3 and 4 +/// bytes. +/// +/// Counting and lookup have to agree on what a gram's count means, and the +/// obvious shortcut, crediting only the window size the lookup will use, is +/// wrong. Lookup asks a 2-byte literal about a 2-gram; if only 2-byte literals +/// ever credited 2-grams, `"//"` would be counted from the handful of two-byte +/// signatures rather than from the tens of thousands of literals that contain +/// it. Short ubiquitous literals then look rare, score well, and win the +/// anchor: the exact inversion this is built to prevent. +fn count_grams_of(lit: &[u8], out: &mut Vec) { + out.clear(); + for n in 2..=4usize { + if lit.len() < n { + break; + } + out.extend(lit.windows(n).map(gram_key)); + } + out.sort_unstable(); + out.dedup(); +} + +impl GramStats { + /// Count grams over every literal `each` yields; `literals` is about how + /// many it yields, which sizes the work. + /// + /// Past [`FILTER_MIN_BODIES`] the repeat filter pays for itself and `each` + /// is called twice: once to find which grams repeat, once to count only + /// those. Below it the filter's fixed 64 MiB would dwarf the data it is + /// filtering, so the counts are taken exactly in a single pass. Both give + /// identical scores: a gram the filter drops appears in one literal, and + /// `log2(1)` is the zero an absent gram scores. + pub(crate) fn build(literals: usize, mut each: impl FnMut(&mut dyn FnMut(&[u8]))) -> Self { + let mut buf: Vec = Vec::new(); + let mut counts: rustc_hash::FxHashMap = rustc_hash::FxHashMap::default(); + if literals < FILTER_MIN_BODIES { + each(&mut |lit| { + count_grams_of(lit, &mut buf); + for &k in &buf { + *counts.entry(k).or_insert(0) += 1; + } + }); + return GramStats { counts }; + } + let mut once = vec![0u64; SEEN_BITS / 64]; + let mut twice = vec![0u64; SEEN_BITS / 64]; + let mark = |bits: &mut [u64], i: usize| bits[i >> 6] |= 1 << (i & 63); + let test = |bits: &[u64], i: usize| bits[i >> 6] >> (i & 63) & 1 == 1; + let slot = |k: u64| { + (k.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> (64 - SEEN_BITS.trailing_zeros())) as usize + }; + // Pass 1: which grams appear in more than one literal. + each(&mut |lit| { + count_grams_of(lit, &mut buf); + for &k in &buf { + let i = slot(k); + if test(&once, i) { + mark(&mut twice, i); + } else { + mark(&mut once, i); + } + } + }); + drop(once); + // Pass 2: count the repeats. Per literal, not per occurrence, so a + // sequence repeated inside one literal does not look popular. + each(&mut |lit| { + count_grams_of(lit, &mut buf); + for &k in &buf { + if test(&twice, slot(k)) { + *counts.entry(k).or_insert(0) += 1; + } + } + }); + GramStats { counts } + } + + /// How many literals contain `lit`, bounded above: a literal occurs in at + /// most as many literals as its rarest gram does, and the widest gram it + /// holds gives the tightest such bound. Erring high is the safe direction: + /// it can only make a literal look like a worse anchor. + pub(crate) fn popularity(&self, lit: &[u8]) -> u32 { + if lit.len() < 2 { + return 0; + } + lit.windows(lit.len().min(4)) + .map(|g| self.counts.get(&gram_key(g)).copied().unwrap_or(0)) + .min() + .unwrap_or(0) + } +} + +/// `floor(log2(c))`, and 0 for the "nothing known" counts 0 and 1: an unseen +/// literal is never penalised, and the penalty grows with an order of +/// magnitude rather than with a raw count that spans 1..10^6. +fn log2_floor(c: u32) -> isize { + if c <= 1 { + 0 + } else { + c.ilog2() as isize + } +} + +/// How rare `b` should be from its bytes alone. A low-entropy run (a constant +/// byte like a zero or 0xFF pad, or a 2-symbol repeat) matches repetitive +/// content, PE padding and BSS, millions of times, so it is a terrible +/// prefilter even when long; a varied run scores its length. +pub(crate) fn structural(b: &[u8]) -> isize { + let mut seen = [false; 256]; + let mut distinct = 0usize; + for &x in b { + if !seen[x as usize] { + seen[x as usize] = true; + distinct += 1; + } + } + match distinct { + 0 | 1 => 1, + 2 => 3.min(b.len()) as isize, + _ => b.len() as isize, + } +} + +/// Selectivity of a candidate anchor, higher is rarer: [`structural`], less +/// how many signature literals share it. +/// +/// Length alone is not selectivity: `"http://"` is 7 varied bytes, yet it is +/// the single worst anchor on web and office content, while a shorter literal +/// of the same body (`":7878"`) occurs essentially never. Both terms are +/// log-scaled, length standing in for how rare a run should be and the count +/// for how common it actually is, so the difference reads as a selectivity +/// estimate. Deliberately signed and unsaturated: clamping at zero puts every +/// ubiquitous literal in a tie that the length tie-break then resolves +/// backwards, handing the anchor to the *longest* common run. `stats` is `None` +/// while a body is first parsed, before the set is known. +pub(crate) fn anchor_score(b: &[u8], stats: Option<&GramStats>) -> isize { + structural(b) - stats.map_or(0, |s| log2_floor(s.popularity(b))) +} + +/// What looking a window of anchor bytes up at every position is expected to +/// cost, lower is better: how many literals share it, then how plain its bytes +/// are (padding and blanks first, then few distinct bytes). +pub(crate) fn window_cost(w: &[u8], stats: &GramStats) -> (isize, usize) { + let mut seen = [false; 256]; + let mut distinct = 0; + let mut weight = 0; + for &b in w { + weight += match b { + 0x00 => 3, + 0xff => 2, + b' ' | b'\n' | b'\r' | b'\t' | 0x90 | 0xcc => 1, + _ => 0, + }; + if !seen[b as usize] { + seen[b as usize] = true; + distinct += 1; + } + } + (log2_floor(stats.popularity(w)), weight + w.len() - distinct) +} diff --git a/crates/exav-core/src/grams/sweep.rs b/crates/exav-core/src/grams/sweep.rs new file mode 100644 index 00000000..46211b59 --- /dev/null +++ b/crates/exav-core/src/grams/sweep.rs @@ -0,0 +1,219 @@ +//! One pass over an object, a window at a time, looking up the anchors of a +//! [`Grams`] at every position but inside long runs of one byte. + +use super::{Grams, Scan, GRAM}; +use crate::byte_source::ByteSource; + +/// Bytes of the object [`sweep`] reads at a time; in memory, a slice of it. +/// Small in tests, so their haystacks cross windows. +pub(crate) const SWEEP_WINDOW: usize = if cfg!(test) { 1000 } else { 1 << 20 }; + +/// Shortest run of one byte [`sweep`] skips any of. A shorter one would save a +/// few dozen lookups at most. +pub(crate) const MIN_SKIP_RUN: usize = 64; + +/// Whole aligned words inside any run of [`MIN_SKIP_RUN`] bytes: seven of its +/// bytes at most can fall before the first one. +const RUN_WORDS: usize = (MIN_SKIP_RUN - 7) / 8; + +/// Whether [`sweep`] collapses runs of one repeated byte: always, except on a +/// test thread that compares both ways. +fn run_collapse_enabled() -> bool { + #[cfg(test)] + return !NO_RUN_COLLAPSE.with(|c| c.get()); + #[cfg(not(test))] + true +} + +#[cfg(test)] +thread_local! { + /// Turns [`run_collapse_enabled`] off on this thread, for the tests that + /// compare both ways. + pub(crate) static NO_RUN_COLLAPSE: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Look up the anchors of `g` at every position of `hay`, for the partitions +/// `active` keeps, handing `f` each one found as `(partition, value, start, +/// len, count)`: the anchor of `len` bytes matched at `start`, `start + 1`, ... +/// `count` times over. Stops when `f` returns `false`; returns whether the +/// sweep reached the end. +/// +/// Inside a run of one repeated byte, a position whose widest window lies in +/// the run can only start an anchor of that byte, so past its first position +/// a run of at least [`MIN_SKIP_RUN`] bytes is not looked up position by +/// position: each anchor of its byte is reported for the whole stretch at once. +/// +/// Within a window, anchors come in the order their windows start. +pub(crate) fn sweep( + g: &Grams, + active: &[bool], + hay: &H, + f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool, +) -> bool { + if g.is_empty() { + return true; + } + let mut sc = g.start(active, hay); + let collapse = run_collapse_enabled(); + let mut skips = Vec::new(); + let mut at = 0; + while at < hay.len() { + let buf = hay.window(at, (hay.len() - at).min(SWEEP_WINDOW)); + if buf.is_empty() { + break; + } + skips.clear(); + let next = match collapse { + true => plan_skips(&buf, at, hay, &mut skips), + false => at + buf.len(), + }; + if !sweep_window(g, &mut sc, &buf, at, next, &skips, f) { + return false; + } + at = next; + } + true +} + +/// [`sweep`] over the window `buf` at `at`, the next one starting at `next`, +/// past the runs `skips`. +fn sweep_window( + g: &Grams, + sc: &mut Scan, + buf: &[u8], + at: usize, + next: usize, + skips: &[Skip], + f: &mut impl FnMut(usize, u32, usize, usize, usize) -> bool, +) -> bool { + // The positions whose eight bytes are in `buf`, and past them, those + // read from the object: across into the next window, or past a run that + // goes on beyond this one, just before the next window starts after it. + let fits = buf.len().saturating_sub(7); + let (mut from, mut across) = (0, at + fits); + for s in skips { + let (lo, hi) = (at + s.start + 1, s.end.saturating_sub(GRAM - 1)); + if hi <= lo { + continue; + } + if !g.scan(sc, buf, at, from, (lo - at).min(fits), f) + || !g.run(sc, buf[s.start], lo, hi, s.end, f) + { + return false; + } + from = from.max(hi - at); + across = across.max(hi); + } + g.scan(sc, buf, at, from, fits, f) && g.scan_source(sc, across, next, f) +} + +/// A run of one byte [`sweep`] skips: it starts at `start` in its window and +/// ends at `end` in the object. +struct Skip { + start: usize, + end: usize, +} + +/// The runs to skip in `buf`, at `at` in `hay`; returns where in `hay` the +/// next window starts, past the end of a run that goes on beyond this one. +#[inline(never)] +fn plan_skips( + buf: &[u8], + at: usize, + hay: &H, + skips: &mut Vec, +) -> usize { + let mut from = 0; + while let Some((r, n)) = next_long_run(buf, from) { + let b = buf[r]; + let mut end = at + r + n; + if r + n == buf.len() { + hay.chunks(end, hay.len(), &mut |_, c| { + let same = c.iter().take_while(|&&x| x == b).count(); + end += same; + same == c.len() + }); + } + skips.push(Skip { start: r, end }); + if end >= at + buf.len() { + return end; + } + from = end - at; + } + at + buf.len() +} + +/// The first run of at least [`MIN_SKIP_RUN`] equal bytes in `s[from..]`, as +/// `(start, len)`. Such a run holds [`RUN_WORDS`] whole eight-byte-aligned words +/// of its byte, so the search reads words, and bytes only at a found run's +/// edges. Executables are full of short runs of zeros, which this passes over +/// a word at a time. +fn next_long_run(s: &[u8], from: usize) -> Option<(usize, usize)> { + let word = |w: usize| u64::from_le_bytes(s[w..w + 8].try_into().unwrap()); + let mut w = from.next_multiple_of(8); + while w + 8 <= s.len() { + let x = word(w); + if x != (x & 0xff).wrapping_mul(0x0101_0101_0101_0101) { + w += 8; + continue; + } + // The aligned words equal to this one, from it on. + let mut e = w + 8; + while e + 8 <= s.len() && word(e) == x { + e += 8; + } + if e - w < RUN_WORDS * 8 { + w = e; + continue; + } + let c = s[w]; + let start = w - s[from..w].iter().rev().take_while(|&&b| b == c).count(); + let end = e + s[e..].iter().take_while(|&&b| b == c).count(); + if end - start >= MIN_SKIP_RUN { + return Some((start, end - start)); + } + w = end.next_multiple_of(8); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `next_long_run` finds exactly the first run of at least `MIN_SKIP_RUN` + /// bytes, whatever its alignment and length, next to other runs, and at + /// the buffer's edges. + #[test] + fn long_runs_are_found_at_any_alignment() { + let naive = |s: &[u8], from: usize| { + let mut i = from; + for r in s[from..].chunk_by(|a, b| a == b) { + if r.len() >= MIN_SKIP_RUN { + return Some((i, r.len())); + } + i += r.len(); + } + None + }; + for pre in 0..20 { + for len in [1, 7, 8, 15, 56, 57, 62, 63, 64, 65, 71, 72, 200] { + for (other, other_len) in [(0u8, 0), (1, 30), (0, 63), (2, 64)] { + for post in [0, 1, 9] { + let mut s = vec![7u8; pre]; + s.extend(std::iter::repeat_n(other, other_len)); + s.extend(std::iter::repeat_n(0u8, len)); + s.extend(std::iter::repeat_n(5u8, post)); + for from in [0, pre / 2, pre] { + assert_eq!( + next_long_run(&s, from), + naive(&s, from), + "pre {pre} len {len} other {other}x{other_len} post {post} from {from}" + ); + } + } + } + } + } + } +} diff --git a/crates/exav-core/src/hashes.rs b/crates/exav-core/src/hashes.rs index 11f0c8b4..67877b29 100644 --- a/crates/exav-core/src/hashes.rs +++ b/crates/exav-core/src/hashes.rs @@ -24,7 +24,10 @@ fn hex_array(hex: &str) -> Option<[u8; N]> { /// allocations). /// /// [`finalize`]: SortedTable::finalize -#[derive(serde::Serialize, serde::Deserialize)] +/// +/// Stored as a few blobs rather than millions of entries decoded one element +/// at a time: the keys as fixed-width records, the spans as words, the names +/// as one string and the provenance as bits. struct SortedTable { keys: Vec, /// (offset, len) into `names`, parallel to `keys`. @@ -33,12 +36,98 @@ struct SortedTable { /// Per-entry provenance (parallel to `keys`): whether the signature came from /// an unofficial (non-`.cvd`) database. The clean name is stored in `names`; /// the `.UNOFFICIAL` suffix is applied by the report layer (compat mode). - #[serde(default)] unofficial: Vec, - #[serde(skip)] pending: Vec<(K, String, bool)>, } +/// A key a [`SortedTable`] stores as a fixed-width record. +trait Record: Sized { + const WIDTH: usize; + fn put(&self, out: &mut Vec); + fn get(b: &[u8]) -> Self; +} + +impl Record for [u8; N] { + const WIDTH: usize = N; + fn put(&self, out: &mut Vec) { + out.extend_from_slice(self); + } + fn get(b: &[u8]) -> Self { + b.try_into().expect("a record of its width") + } +} + +impl Record for (u64, [u8; N]) { + const WIDTH: usize = 8 + N; + fn put(&self, out: &mut Vec) { + out.extend_from_slice(&self.0.to_le_bytes()); + out.extend_from_slice(&self.1); + } + fn get(b: &[u8]) -> Self { + ( + u64::from_le_bytes(b[..8].try_into().expect("eight bytes")), + b[8..].try_into().expect("a record of its width"), + ) + } +} + +impl Serialize for SortedTable { + fn serialize(&self, s: S) -> Result { + use crate::database::Blob; + let mut keys = Vec::with_capacity(self.keys.len() * K::WIDTH); + for k in &self.keys { + k.put(&mut keys); + } + let spans: Vec = self.spans.iter().flat_map(|&(o, l)| [o.to_le_bytes(), l.to_le_bytes()]).flatten().collect(); + let mut bits = vec![0u8; self.unofficial.len().div_ceil(8)]; + for (i, _) in self.unofficial.iter().enumerate().filter(|(_, &u)| u) { + bits[i / 8] |= 1 << (i % 8); + } + (Blob(&keys), Blob(&spans), &self.names, Blob(&bits)).serialize(s) + } +} + +impl<'de, K: Record + Ord> Deserialize<'de> for SortedTable { + fn deserialize>(d: D) -> Result { + use crate::database::BlobBuf; + use serde::de::Error; + let (keys, spans, names, bits): (BlobBuf, BlobBuf, String, BlobBuf) = Deserialize::deserialize(d)?; + let (keys, spans, bits) = (keys.0, spans.0, bits.0); + let n = keys.len() / K::WIDTH; + if keys.len() % K::WIDTH != 0 || spans.len() != n * 8 || bits.len() != n.div_ceil(8) { + return Err(D::Error::custom("hash table blobs of unequal lengths")); + } + // `as_chunks` cannot take a generic's associated const as its width. + #[allow(clippy::chunks_exact_to_as_chunks)] + let keys: Vec = keys.chunks_exact(K::WIDTH).map(K::get).collect(); + let spans: Vec<(u32, u32)> = spans + .as_chunks::<8>() + .0 + .iter() + .map(|w| { + let (o, l) = w.split_at(4); + (u32::from_le_bytes(o.try_into().unwrap()), u32::from_le_bytes(l.try_into().unwrap())) + }) + .collect(); + // In u64, so a corrupt span cannot overflow a 32-bit `usize`. + let in_names = |&(o, l): &(u32, u32)| { + let e = o as u64 + l as u64; + e <= names.len() as u64 && names.is_char_boundary(o as usize) && names.is_char_boundary(e as usize) + }; + if !keys.is_sorted() || !spans.iter().all(in_names) { + return Err(D::Error::custom("hash table out of order or out of bounds")); + } + let unofficial = (0..n).map(|i| bits[i / 8] >> (i % 8) & 1 != 0).collect(); + Ok(SortedTable { + keys, + spans, + names, + unofficial, + pending: Vec::new(), + }) + } +} + impl Default for SortedTable { fn default() -> Self { Self { @@ -91,11 +180,8 @@ impl SortedTable { /// Return `(clean_name, unofficial)` for `key`. fn get(&self, key: &K) -> Option<(&str, bool)> { let i = self.keys.binary_search(key).ok()?; - let (off, len) = self.spans[i]; - // `unofficial` may be empty when loading an older database (serde default); - // treat a missing entry as official. - let unofficial = self.unofficial.get(i).copied().unwrap_or(false); - Some((&self.names[off as usize..(off + len) as usize], unofficial)) + let (off, len) = (self.spans[i].0 as usize, self.spans[i].1 as usize); + Some((&self.names[off..off + len], self.unofficial[i])) } } @@ -267,22 +353,7 @@ impl HashDb { /// signature's `(clean_name, unofficial)`. A sized signature matches only at /// that exact length; a `*` signature matches any length. pub fn lookup(&self, digests: &Digests, size: u64) -> Option<(String, bool)> { - if let Some(k) = hex_array::<16>(&digests.md5) { - if let Some((n, u)) = self.md5.get(&k, size) { - return Some((n.to_string(), u)); - } - } - if let Some(k) = hex_array::<20>(&digests.sha1) { - if let Some((n, u)) = self.sha1.get(&k, size) { - return Some((n.to_string(), u)); - } - } - if let Some(k) = hex_array::<32>(&digests.sha256) { - if let Some((n, u)) = self.sha256.get(&k, size) { - return Some((n.to_string(), u)); - } - } - None + lookup_in(&self.md5, &self.sha1, &self.sha256, digests, size) } } @@ -316,10 +387,14 @@ impl SectionHashDb { self.sha256.finalize(); } - /// True if any SHA (not just MD5) section signatures are loaded, so the - /// caller knows whether computing per-section SHA digests is worthwhile. - pub fn wants_sha(&self) -> bool { - self.sha1.len() > 0 || self.sha256.len() > 0 + /// The digests [`Self::lookup`] can match for a section of `size` bytes: + /// most sections' sizes are no signature's, and need none. + pub(crate) fn wants(&self, size: u64) -> Want { + Want { + md5: self.md5.wants(size), + sha1: self.sha1.wants(size), + sha256: self.sha256.wants(size), + } } /// Parse `.mdb`/`.mdu`/`.msb` lines (`SectionSize:HASH:Name`). The first @@ -379,46 +454,54 @@ impl SectionHashDb { /// Look up a section by its raw size and computed digests; returns the /// matching signature's `(clean_name, unofficial)`. pub fn lookup(&self, size: u64, digests: &Digests) -> Option<(String, bool)> { - if let Some(k) = hex_array::<16>(&digests.md5) { - if let Some((n, u)) = self.md5.get(&k, size) { - return Some((n.to_string(), u)); - } - } - if let Some(k) = hex_array::<20>(&digests.sha1) { - if let Some((n, u)) = self.sha1.get(&k, size) { - return Some((n.to_string(), u)); - } - } - if let Some(k) = hex_array::<32>(&digests.sha256) { - if let Some((n, u)) = self.sha256.get(&k, size) { - return Some((n.to_string(), u)); - } - } - None + lookup_in(&self.md5, &self.sha1, &self.sha256, digests, size) } } -/// Hex digests computed over a file. -#[derive(Debug, Clone)] +/// The first signature of the three tables matching `digests` at `size`. +fn lookup_in( + md5: &Md5Table, + sha1: &Sha1Table, + sha256: &Sha256Table, + d: &Digests, + size: u64, +) -> Option<(String, bool)> { + d.md5 + .and_then(|k| md5.get(&k, size)) + .or_else(|| d.sha1.and_then(|k| sha1.get(&k, size))) + .or_else(|| d.sha256.and_then(|k| sha256.get(&k, size))) + .map(|(n, u)| (n.to_string(), u)) +} + +/// Digests computed over a file, those asked for. +#[derive(Debug, Clone, Default, PartialEq)] pub struct Digests { - pub md5: String, - pub sha1: String, - pub sha256: String, + pub md5: Option<[u8; 16]>, + pub sha1: Option<[u8; 20]>, + pub sha256: Option<[u8; 32]>, } -/// Digests for section-hash matching: always MD5; SHA1/SHA256 only when -/// `want_sha` (i.e. `.msb` signatures are loaded), since they are otherwise -/// unused. Skipped fields are left empty and their lookups simply miss. -pub fn section_digests(data: &[u8], want_sha: bool) -> Digests { - if want_sha { - digests_of(data) - } else { - Digests { - md5: encode_hex(&Md5::digest(data)), - sha1: String::new(), - sha256: String::new(), - } +impl Digests { + /// The MD5, in hex; empty when it was not computed. + pub fn md5_hex(&self) -> String { + self.md5.map(|d| encode_hex(&d)).unwrap_or_default() + } + + /// The SHA-1, in hex; empty when it was not computed. + pub fn sha1_hex(&self) -> String { + self.sha1.map(|d| encode_hex(&d)).unwrap_or_default() } + + /// The SHA-256, in hex; empty when it was not computed. + pub fn sha256_hex(&self) -> String { + self.sha256.map(|d| encode_hex(&d)).unwrap_or_default() + } +} + +/// The digests of section `data`, of `size` bytes, that section signatures of +/// that size could match; none, without reading it, for most sections. +pub(crate) fn section_digests(db: &SectionHashDb, size: u64, data: &[u8]) -> Digests { + digests_wanted(&data, db.wants(size)) } /// Compute digests over a byte slice (used for already-buffered content @@ -479,9 +562,9 @@ impl Hashing { pub(crate) fn finish(self) -> Digests { Digests { - md5: self.md5.map_or_else(String::new, |h| encode_hex(&h.finalize())), - sha1: self.sha1.map_or_else(String::new, |h| encode_hex(&h.finalize())), - sha256: self.sha256.map_or_else(String::new, |h| encode_hex(&h.finalize())), + md5: self.md5.map(|h| h.finalize().into()), + sha1: self.sha1.map(|h| h.finalize().into()), + sha256: self.sha256.map(|h| h.finalize().into()), } } } @@ -494,9 +577,34 @@ mod tests { fn known_md5() { // md5("") = d41d8cd98f00b204e9800998ecf8427e let d = digests_of(b""); - assert_eq!(d.md5, "d41d8cd98f00b204e9800998ecf8427e"); + assert_eq!(d.md5_hex(), "d41d8cd98f00b204e9800998ecf8427e"); // sha256("") = e3b0c442... - assert!(d.sha256.starts_with("e3b0c44298fc1c14")); + assert!(d.sha256_hex().starts_with("e3b0c44298fc1c14")); + } + + /// A table stored and read back answers every lookup as before, and one + /// whose span runs out of its names is refused rather than read. + #[test] + fn hash_tables_round_trip_as_blobs() { + let (a, b, c) = (digests_of(b"one"), digests_of(b"two"), digests_of(b"three")); + let mut db = HashDb::new(); + db.extend_from_text(&format!("{}:3:Sized.Md5\n{}:*:Any.Sha1\n", a.md5_hex(), b.sha1_hex())); + db.extend_from_text_prov(&format!("{}:5:Sized.Sha256.é\n", c.sha256_hex()), true); + db.finalize(); + let blob = rmp_serde::to_vec(&db).unwrap(); + let back: HashDb = rmp_serde::from_slice(&blob).unwrap(); + for (d, size) in [(&a, 3), (&b, 99), (&c, 5), (&a, 4), (&c, 6)] { + assert_eq!(back.lookup(d, size), db.lookup(d, size), "{size}"); + } + assert_eq!(back.lookup(&c, 5), Some(("Sized.Sha256.é".to_string(), true))); + assert_eq!(back.lookup(&b, 99), Some(("Any.Sha1".to_string(), false))); + // Break the last span so it runs past the names: refused. + let mut t = SortedTable::<[u8; 4]>::default(); + t.insert(*b"abcd", "Name", false); + t.finalize(); + t.spans[0].1 = 99; + let blob = rmp_serde::to_vec(&t).unwrap(); + assert!(rmp_serde::from_slice::>(&blob).is_err()); } #[test] @@ -504,7 +612,7 @@ mod tests { let mut db = HashDb::new(); let d = digests_of(b"malware"); let n = "malware".len() as u64; - db.extend_from_text(&format!("{}:*:Test.Malware\n", d.md5)); + db.extend_from_text(&format!("{}:*:Test.Malware\n", d.md5_hex())); db.finalize(); assert_eq!( db.lookup(&d, n).map(|(n, _)| n).as_deref(), @@ -518,7 +626,7 @@ mod tests { // A sized signature matches only at that exact file length. let mut db = HashDb::new(); let d = digests_of(b"malware"); - db.extend_from_text(&format!("{}:7:Test.Sized\n", d.md5)); + db.extend_from_text(&format!("{}:7:Test.Sized\n", d.md5_hex())); db.finalize(); assert_eq!( db.lookup(&d, 7).map(|(n, _)| n).as_deref(), @@ -531,7 +639,7 @@ mod tests { fn hashdb_sha_whole_file() { let mut db = HashDb::new(); let d = digests_of(b"payload"); - db.extend_from_text(&format!("{}:*:Test.BySha256\n", d.sha256)); + db.extend_from_text(&format!("{}:*:Test.BySha256\n", d.sha256_hex())); db.finalize(); assert_eq!( db.lookup(&d, 7).map(|(n, _)| n).as_deref(), @@ -548,10 +656,20 @@ mod tests { // .mdb (MD5) sized + wildcard, plus an .msb (SHA256) wildcard entry. db.extend_from_text(&format!( "4096:{}:Sig.Sized\n*:{}:Sig.AnySize\n*:{}:Sig.BySha\n", - s.md5, other.md5, other.sha256 + s.md5_hex(), + other.md5_hex(), + other.sha256_hex() )); db.finalize(); - assert!(db.wants_sha()); + // Only what a signature of the size could match. + assert_eq!(db.wants(4096), Want { md5: true, sha1: false, sha256: true }); + assert_eq!(db.wants(512), Want { md5: true, sha1: false, sha256: true }); + let mut sized = SectionHashDb::new(); + sized.extend_from_text(&format!("4096:{}:Sig.Sized\n", s.md5_hex())); + sized.finalize(); + assert_eq!(sized.wants(512), Want::default()); + assert_eq!(section_digests(&sized, 512, b"section-bytes"), Digests::default()); + assert_eq!(section_digests(&sized, 4096, b"section-bytes").md5, s.md5); // exact size+md5 assert_eq!( db.lookup(4096, &s).map(|(n, _)| n).as_deref(), diff --git a/crates/exav-core/src/lib.rs b/crates/exav-core/src/lib.rs index 04cebd61..d03a28e8 100644 --- a/crates/exav-core/src/lib.rs +++ b/crates/exav-core/src/lib.rs @@ -92,6 +92,7 @@ engine_internals!( ); mod byte_source; +mod grams; mod stream_regex; // YARA rule support. The native engine (compiler/scanner/modules) lives in-tree @@ -1355,8 +1356,8 @@ fn allowlisted(db: &Scanner, facts: &Facts) -> bool { .is_some() } -/// Build every structure a scan initialises lazily (engine automata, compiled -/// YARA rules), so a process that forks workers after loading shares them +/// Build every structure a scan initialises lazily (compiled YARA rules and +/// their automata), so a process that forks workers after loading shares them /// copy-on-write instead of each worker building its own. pub fn warm_up(db: &Scanner) { const PROBE: &[u8] = b"MZ\x90\x00\x00\x00\x00\x00"; @@ -1611,16 +1612,24 @@ fn limits_alert_name(kind: unpack::LimitKind) -> Option<&'static str> { /// ceiling come through here, so the two cannot name one condition two ways, /// and the kind is passed as a type so the name is looked up and never guessed. fn max_file_size_report(size: u64, max: u64, opts: &ScanOptions) -> ScanReport { + match max_file_size(size, max, opts) { + Ok(name) => ScanReport::infected(name.to_string(), 0, Method::Heuristic, Vec::new()), + Err(reason) => ScanReport::limits(reason, Vec::new()), + } +} + +/// The alert a top-level file larger than `max_scan_size` is reported as +/// under `--alert-exceeds-max`, or else the reason it is over the limit. +fn max_file_size(size: u64, max: u64, opts: &ScanOptions) -> Result<&'static str, String> { if opts.alert_exceeds_max { if let Some(name) = limits_alert_name(unpack::LimitKind::MaxFileSize) { match_loc_record(); - return ScanReport::infected(name.to_string(), 0, Method::Heuristic, Vec::new()); + return Ok(name); } } - ScanReport::limits( - format!("file size {size} exceeds max-input-bytes {max}; scanned first {max} bytes only"), - Vec::new(), - ) + Err(format!( + "file size {size} exceeds max-input-bytes {max}; scanned first {max} bytes only" + )) } /// Turn a budget stop into an outcome, honouring `--alert-exceeds-max`. @@ -2150,11 +2159,37 @@ pub fn analyze_all_with_outcome( /// As [`analyze_all_with_outcome`], over a seekable input of `size` bytes. /// One too large to hold is read through a block cache of bounded size, so /// every detection is listed whatever the input's size. +/// +/// Past `--max-input-bytes`, what the first `max_scan_size` bytes hold is +/// listed and the input is reported over the limit, as [`scan_seekable`] +/// reports it. pub fn analyze_all_seekable( db: &Scanner, mut reader: R, size: u64, opts: &ScanOptions, +) -> io::Result<(Vec<(String, Method)>, AllMatchOutcome)> { + if let Some(max) = opts.max_scan_size { + if size > max { + let (mut names, outcome) = all_within(db, Window::new(&mut reader, max)?, max, opts)?; + return Ok(match max_file_size(size, max, opts) { + Ok(name) => { + names.push((name.to_string(), Method::Heuristic)); + (names, outcome) + } + Err(reason) => (names, AllMatchOutcome::LimitsExceeded(reason)), + }); + } + } + all_within(db, reader, size, opts) +} + +/// [`analyze_all_seekable`] for an input within `--max-input-bytes`. +fn all_within( + db: &Scanner, + mut reader: R, + size: u64, + opts: &ScanOptions, ) -> io::Result<(Vec<(String, Method)>, AllMatchOutcome)> { if size <= opts.deep_analysis_max { let mut data = Vec::new(); @@ -2854,7 +2889,16 @@ fn scan_archive( return members; } let mut outcome = members; - let rest = match core(db, facts, obj.core_type.unwrap_or(ft), obj, opts, sink) { + let writes = write_limits(budget); + let rest = match core( + db, + facts, + obj.core_type.unwrap_or(ft), + obj, + opts, + writes, + sink, + ) { Ok(rest) => rest, Err(stop) => return stop, }; @@ -3044,7 +3088,16 @@ fn scan_file( let (depth, container) = (obj.depth, obj.container); let mut unpacked = Vec::new(); let (ft, fmt) = if obj.core { - let rest = match core(db, facts, obj.core_type.unwrap_or(ft), obj, opts, sink) { + let mut writes = write_limits(budget); + let rest = match core( + db, + facts, + obj.core_type.unwrap_or(ft), + obj, + opts, + writes, + sink, + ) { Ok(rest) => rest, Err(stop) => return stop, }; @@ -3059,7 +3112,9 @@ fn scan_file( Some(retyped) => { // Worth something only if the object is matched AS the new // type, and unpacked as it. - match core(db, facts, retyped, obj, opts, sink) { + let written: u64 = unpacked.iter().map(|b| b.len() as u64).sum(); + writes.total = writes.total.saturating_sub(written); + match core(db, facts, retyped, obj, opts, writes, sink) { Ok(rest) => unpacked.extend(rest.unpacked), Err(stop) => return stop, } @@ -3274,7 +3329,7 @@ fn scan_file( if let Some(info) = pe::analyze(data) { // imphash (`.imp`) matching is a ClamAV default (matched whenever the // loaded DB carries `.imp` sigs), so it runs under `clamav_heuristics` - // (i.e. under `--clamav-compat` too). The exav-exclusive ML scorer, + // (i.e. under `--clamav-compat` too). The exav-exclusive static scorer, // packed-injection heuristic, and the `-v` diagnostic findings below // stay behind the full `--detect exav-heuristics` flag. if let Some(hit) = db @@ -3674,6 +3729,16 @@ struct CoreRest { retype: Option, } +/// What bytecode programs may write for one object: no output larger than a +/// buffer the scan may hold, and all of it within the bytes the scan has left. +/// The output is charged to the budget when it is scanned. +fn write_limits(budget: &Budget) -> bytecode::exec::WriteLimits { + bytecode::exec::WriteLimits { + file: budget.limits().max_buffer_bytes, + total: budget.remaining_scan(), + } +} + /// The matching core over an object's own bytes, `ft` being the type it is /// matched as. Every detection goes to `sink`; `Err` means the walk must stop. fn core( @@ -3682,10 +3747,11 @@ fn core( ft: FileType, obj: Obj<'_>, opts: &ScanOptions, + writes: bytecode::exec::WriteLimits, sink: &mut Sink, ) -> Result { let data = facts.data; - let unpacked = core_matches(db, facts, ft, obj, opts, sink); + let unpacked = core_matches(db, facts, ft, obj, opts, writes, sink); // Taken whatever the outcome, so it cannot outlive this object and retype // a later one at the same address. let retype = engine::take_retype_source(data); @@ -3704,6 +3770,7 @@ fn core_matches( ft: FileType, obj: Obj<'_>, opts: &ScanOptions, + writes: bytecode::exec::WriteLimits, sink: &mut Sink, ) -> Result>, DeepOutcome> { let data = facts.data; @@ -3804,7 +3871,8 @@ fn core_matches( let mut unpacked = Vec::new(); if !db.bytecode.is_empty() { let (det, extracted) = profile::timed("bytecode", data.len() as u64, || { - db.bytecode.scan_source(data, ft, materialize, &fired) + db.bytecode + .scan_source(data, ft, materialize, &fired, writes) }); if let Some((name, _)) = det { hit(&name, false, 0, Method::Bytecode)?; @@ -3813,13 +3881,12 @@ fn core_matches( } if !db.sections.is_empty() { if let Some(image) = &image { - let want_sha = db.sections.wants_sha(); let found: Vec<_> = profile::timed("sections", data.len() as u64, || { pe::section_slices(image) .into_iter() .filter_map(|(size, slice)| { db.sections - .lookup(size, &hashes::section_digests(slice, want_sha)) + .lookup(size, &hashes::section_digests(&db.sections, size, slice)) }) .collect() }); @@ -3999,7 +4066,7 @@ mod tests { #[cfg(feature = "all-formats")] assert_eq!(want_fmt, Some(unpack::Format::Autoit)); assert!(!want_embedded.pe.is_empty() && !want_embedded.archives.is_empty()); - assert!(!want_digests.md5.is_empty() && !want_digests.sha256.is_empty()); + assert!(want_digests.md5.is_some() && want_digests.sha256.is_some()); for first in 0..3 { let cache = byte_source::BlockCache::with_sizes(Cursor::new(data.clone()), 4093, 16 * 4093) @@ -4350,11 +4417,18 @@ mod tests { /// it does when nested in another archive. The wildcard signature matches /// only a member NAME, i.e. the raw container; the hash is the container's. #[test] + #[cfg_attr( + target_family = "wasm", + ignore = "host filesystem/tempdir unavailable under WASI" + )] fn a_top_level_container_gets_the_full_engine_over_its_own_bytes() { let zip = build_zip(&[("payload_marker.txt", b"nothing to see here")]); let wild = db_from(&[("w.ndb", "Test.ZipName:0:*:7061796c6f6164??6d61726b6572\n")]); let d = digests_of(&zip); - let hash = db_from(&[("h.hdb", &format!("{}:{}:Test.ZipHash\n", d.md5, zip.len()))]); + let hash = db_from(&[( + "h.hdb", + &format!("{}:{}:Test.ZipHash\n", d.md5_hex(), zip.len()), + )]); let opts = ScanOptions::default(); assert_eq!( @@ -4448,6 +4522,10 @@ mod tests { /// text file is clean only when its normalised views could be made, which /// takes a spill. #[test] + #[cfg_attr( + target_family = "wasm", + ignore = "host filesystem/tempdir unavailable under WASI" + )] fn a_file_over_the_deep_analysis_limit_gets_the_full_engine() { let db = db_from(&[ ("w.ndb", "Test.Wild:0:*:6576696c??7061796c6f6164\n"), @@ -4504,12 +4582,66 @@ mod tests { } } + /// Past `--max-input-bytes` an all-match scan lists what the first bytes + /// hold and reports the rest unscanned, as a single-match scan does: it + /// neither reads past the limit nor calls the input complete. + #[test] + fn an_all_match_scan_keeps_to_the_input_limit() { + let db = db_from(&[( + "t.ndb", + "Test.Head:0:*:4558415654455354\nTest.Tail:0:*:5441494c5441494c\n", + )]); + let mut data = b"EXAVTEST".to_vec(); + data.extend(std::iter::repeat_n(b'.', 10_000)); + data.extend_from_slice(b"TAILTAIL"); + let opts = ScanOptions { + max_scan_size: Some(4096), + ..ScanOptions::default() + }; + let all = |data: &[u8], opts: &ScanOptions| { + analyze_all_seekable(&db, Cursor::new(data), data.len() as u64, opts).unwrap() + }; + let (names, outcome) = all(&data, &opts); + let names: Vec<_> = names.into_iter().map(|(n, _)| n).collect(); + assert_eq!(names, ["Test.Head"]); + match outcome { + AllMatchOutcome::LimitsExceeded(reason) => { + assert!(reason.contains("max-input-bytes"), "{reason}") + } + other => panic!("expected a limit, got {other:?}"), + } + // Nothing in the first bytes: the limit alone, as a single-match scan + // reports it. + let (names, outcome) = all(&data[8..], &opts); + assert!(names.is_empty(), "{names:?}"); + assert!( + matches!(outcome, AllMatchOutcome::LimitsExceeded(_)), + "{outcome:?}" + ); + // Under ClamAV's alert, by ClamAV's name for it. + let alerting = ScanOptions { + alert_exceeds_max: true, + ..opts.clone() + }; + let want = seekable(&db, &data[8..], &alerting); + let (names, _) = all(&data[8..], &alerting); + let names: Vec<_> = names.into_iter().map(|(n, _)| n).collect(); + assert_eq!( + Some(names.as_slice()), + infected_as(&want).map(|n| vec![n.to_string()]).as_deref() + ); + // Within the limit, the whole input. + let (names, outcome) = all(&data, &ScanOptions::default()); + assert_eq!(names.len(), 2, "{names:?}"); + assert!(matches!(outcome, AllMatchOutcome::Complete), "{outcome:?}"); + } + /// The `.fp` allowlist covers a top-level archive, not only its members. #[test] fn an_allowlisted_top_level_archive_is_clean() { let zip = build_zip(&[("b.txt", eicar())]); let d = digests_of(&zip); - let db = db_from(&[("x.fp", &format!("{}:{}:Allowed\n", d.md5, zip.len()))]); + let db = db_from(&[("x.fp", &format!("{}:{}:Allowed\n", d.md5_hex(), zip.len()))]); assert_eq!(seekable(&db, &zip, &ScanOptions::default()), Verdict::Clean); } @@ -4713,7 +4845,7 @@ mod tests { let mut db = Scanner::builtin(); let d = digests_of(b"some clean-looking content"); db.hashes - .extend_from_text(&format!("{}:*:Test.ByHash\n", d.sha256)); + .extend_from_text(&format!("{}:*:Test.ByHash\n", d.sha256_hex())); db.hashes.finalize(); let r = analyze(&db, b"some clean-looking content", &ScanOptions::default()); match r.verdict { @@ -4739,7 +4871,7 @@ mod tests { let mut db = Scanner::builtin(); let d = digests_of(&data); db.hashes - .extend_from_text(&format!("{}:{}:Test.Tiny{}\n", d.md5, n, n)); + .extend_from_text(&format!("{}:{}:Test.Tiny{}\n", d.md5_hex(), n, n)); db.hashes.finalize(); let hit = matches!( analyze(&db, &data, &ScanOptions::default()).verdict, @@ -4772,7 +4904,7 @@ mod tests { ); let d = digests_of(b"abcd"); db.allow - .extend_from_text(&format!("{}:4:Test.Allow\n", d.md5)); + .extend_from_text(&format!("{}:4:Test.Allow\n", d.md5_hex())); db.allow.finalize(); let r = analyze(&db, b"abcd", &ScanOptions::default()); assert!( @@ -4809,7 +4941,11 @@ mod tests { // `.fp` allowlisting this file's hash clears the detection. let d = digests_of(&data); - std::fs::write(dir.path().join("b.fp"), format!("{}:*:Allowed\n", d.md5)).unwrap(); + std::fs::write( + dir.path().join("b.fp"), + format!("{}:*:Allowed\n", d.md5_hex()), + ) + .unwrap(); assert_eq!( analyze(&loader::load(dir.path()).unwrap(), &data, &opts).verdict, Verdict::Clean @@ -5054,7 +5190,7 @@ mod tests { let mut db = Scanner::builtin(); let d = digests_of(&blob); db.hashes - .extend_from_text(&format!("{}:*:Test.TarWholeHash\n", d.sha256)); + .extend_from_text(&format!("{}:*:Test.TarWholeHash\n", d.sha256_hex())); db.hashes.finalize(); let f = write_temp(&blob); let r = scan_path(&db, f.path(), &ScanOptions::default()).unwrap(); @@ -5477,20 +5613,21 @@ mod parity { } /// The same, over a real database and a directory of samples: - /// `EXAV_PARITY_DB=<.exavdb> EXAV_PARITY_CORPUS=

cargo test -p exav-core - /// --release --lib parity::corpus -- --ignored --nocapture`. + /// `EXAV_DEBUG_PARITY_DB=<.exavdb> EXAV_DEBUG_PARITY_CORPUS= cargo test + /// -p exav-core --release --lib parity::corpus -- --ignored --nocapture`, + /// and `EXAV_DEBUG_PARITY_LIMIT=` for the first `n` samples. #[test] #[ignore = "needs a database and a corpus"] fn corpus() { let (Ok(db), Ok(dir)) = ( - std::env::var("EXAV_PARITY_DB"), - std::env::var("EXAV_PARITY_CORPUS"), + std::env::var("EXAV_DEBUG_PARITY_DB"), + std::env::var("EXAV_DEBUG_PARITY_CORPUS"), ) else { - eprintln!("skip: EXAV_PARITY_DB / EXAV_PARITY_CORPUS unset"); + eprintln!("skip: EXAV_DEBUG_PARITY_DB / EXAV_DEBUG_PARITY_CORPUS unset"); return; }; let db = database::load(std::path::Path::new(&db)).expect("load the database"); - let limit: usize = std::env::var("EXAV_PARITY_LIMIT") + let limit: usize = std::env::var("EXAV_DEBUG_PARITY_LIMIT") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(usize::MAX); diff --git a/crates/exav-core/src/loader.rs b/crates/exav-core/src/loader.rs index bdf901de..5e4cf98f 100644 --- a/crates/exav-core/src/loader.rs +++ b/crates/exav-core/src/loader.rs @@ -100,7 +100,7 @@ use crate::Scanner; pub struct Builder { /// The full in-memory engine, fed signature text file-by-file so the raw /// text is dropped as it's compiled (rather than concatenated and held - /// alive through the memory-heavy automaton build). + /// alive through the index build). engine: EngineBuilder, hashes: HashDb, sections: SectionHashDb, @@ -134,11 +134,6 @@ pub struct Builder { /// Passwords parsed from `.pwdb` files, used to decrypt encrypted archive /// members. User-supplied (the official CVDs ship no `.pwdb`). passwords: Vec, - /// Optional per-shard memory budget for the signature-automaton build - /// (`--build-shard-bytes`). `None` = build each partition as a single - /// automaton (fastest). A budget shards large partitions so a huge set - /// (`main+daily`) can build on a small host. Affects `--build-db` builds. - max_build_mem: Option, /// Version + build-time of the newest loaded `.cvd`/`.cld` container, for the /// clamd-compatible daemon `VERSION` reply. See [`Scanner::db_version`]. db_version: Option<(u32, String)>, @@ -165,12 +160,6 @@ impl Builder { self.engine.set_detect_pua(on); } - /// Set the per-shard automaton-build memory budget (`--build-shard-bytes`). - /// `None` (default) builds each partition as one automaton. - pub fn set_max_build_mem(&mut self, bytes: Option) { - self.max_build_mem = bytes; - } - /// Retained for API/CLI compatibility (`--clamav-compat`). The unofficial /// suffix is applied at scan/report time (gated on `ScanOptions::unofficial_suffix`), /// using per-signature provenance recorded at load regardless of this flag, so @@ -300,13 +289,14 @@ impl Builder { "ndb" | "ndu" => self.engine.add_ndb(text, unofficial), // Literal `Name=HEX` signatures. "db" => { - if let Ok(pats) = crate::patterns::parse_simple(text) { - for (name, bytes) in pats { - if name.starts_with("PUA.") && !self.detect_pua { - continue; - } - self.engine.add_literal_prov(&name, &bytes, unofficial); + let (pats, skipped) = crate::patterns::parse_simple(text); + self.engine + .skip_unloadable("db line not Name=plain hex", skipped); + for (name, bytes) in pats { + if name.starts_with("PUA.") && !self.detect_pua { + continue; } + self.engine.add_literal_prov(&name, &bytes, unofficial); } } "hdb" | "hsb" | "hdu" | "hsu" => self.hashes.extend_from_text_prov(text, unofficial), @@ -359,11 +349,14 @@ impl Builder { // signature-name / target-description scoping is not yet applied // (every loaded password is tried against every encrypted member). "pwdb" => parse_pwdb_into(text, &mut self.passwords), - // Bytecode programs: keep the raw text; parsing + gating happens at - // build() (the runtime drops any that fail to parse). + // Bytecode programs: keep the raw text of those that parse, for + // build(); one that does not is counted unsupported. "cbc" if crate::bytecode::parse(text).is_ok() => { self.bytecode_sources.push(text.to_string()); } + "cbc" => self + .engine + .skip_unloadable("bytecode program does not parse", 1), _ => {} } } @@ -373,9 +366,9 @@ impl Builder { pub fn build(mut self) -> Result { self.engine.add_literal("Eicar-Test-Signature", eicar()); - // Sort + intern the hash tables BEFORE building the (memory-heavy) - // automaton: finalize collapses millions of per-entry strings into one - // buffer, so that transient doesn't coexist with the build's peak. + // Sort + intern the hash tables BEFORE building the signature index: + // finalize collapses millions of per-entry strings into one buffer, so + // that transient doesn't coexist with the build's peak. let mut hashes = self.hashes; let mut sections = self.sections; let mut allow = self.allow; @@ -391,7 +384,7 @@ impl Builder { self.bytecode_sources, &mut self.engine, ); - let engine = self.engine.build_with_budget(self.max_build_mem); + let engine = self.engine.build(); Ok(Scanner { engine, @@ -478,7 +471,7 @@ fn hex_decode(s: &str) -> Option> { /// Load a database from a file/dir path (convenience). A prebuilt exav database /// file (see [`crate::database`]) is detected by its magic tag and loaded -/// directly, skipping signature parsing and automaton construction. +/// directly, skipping signature parsing and the index build. pub fn load(path: &Path) -> Result { load_with_pua(path, false) } @@ -499,17 +492,6 @@ pub fn load_with_options( path: &Path, detect_pua: bool, unofficial_suffix: bool, -) -> Result { - load_with_options_mem(path, detect_pua, unofficial_suffix, None) -} - -/// As [`load_with_options`], with a per-shard automaton-build memory budget -/// (`--build-shard-bytes`). Ignored when `path` is a prebuilt database file. -pub fn load_with_options_mem( - path: &Path, - detect_pua: bool, - unofficial_suffix: bool, - max_build_mem: Option, ) -> Result { if path.is_file() && crate::database::is_database_file(path) { return crate::database::load(path).map_err(|e| LoadError::Database { @@ -528,7 +510,6 @@ pub fn load_with_options_mem( let mut builder = Builder::new(); builder.set_detect_pua(detect_pua); builder.set_unofficial_suffix(unofficial_suffix); - builder.set_max_build_mem(max_build_mem); builder.add_path(path)?; let prebuilt = builder.prebuilt_skipped.take(); let scanner = builder.build()?; @@ -593,6 +574,10 @@ mod tests { /// typo, or a directory whose only database is a prebuilt one (which loads /// by its own path). #[test] + #[cfg_attr( + target_family = "wasm", + ignore = "host filesystem/tempdir unavailable under WASI" + )] fn a_path_that_loads_nothing_is_an_error() { let dir = std::env::temp_dir().join(format!("exav-loader-empty-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -658,6 +643,28 @@ Sig.C;td;0;cleartext-pw\n"; // duplicate password de-duped assert_eq!(db.passwords, vec!["hunter2".to_string()]); } + /// A `.db` line that does not parse is skipped and counted on its own, and + /// the file's other signatures still load. A `.cbc` program that does not + /// parse is counted too. + #[test] + fn an_unloadable_db_line_or_cbc_program_is_counted() { + let mut b = Builder::new(); + b.add_named_bytes( + "x.db", + b"Test.Good=6d616c776172652d676f6f64\nTest.Wild=6d61??6c\nno separator\n", + false, + ); + b.add_named_bytes("y.cbc", b"not a bytecode program\n", false); + let db = b.build().unwrap(); + assert_eq!(db.unsupported_count(), 3); + let report = crate::analyze(&db, b"xx malware-good xx", &crate::ScanOptions::default()); + assert!( + matches!(&report.verdict, crate::Verdict::Infected { signature, .. } if signature == "Test.Good"), + "{:?}", + report.verdict + ); + } + #[test] #[cfg_attr( target_family = "wasm", @@ -671,7 +678,11 @@ Sig.C;td;0;cleartext-pw\n"; // duplicate password de-duped ) .unwrap(); let d = crate::hashes::digests_of(b"x"); - std::fs::write(dir.path().join("b.hdb"), format!("{}:*:Sig.H\n", d.md5)).unwrap(); + std::fs::write( + dir.path().join("b.hdb"), + format!("{}:*:Sig.H\n", d.md5_hex()), + ) + .unwrap(); let mut f = std::fs::File::create(dir.path().join("c.fdb")).unwrap(); writeln!(f, "imphash:abcd:Sig.F").unwrap(); diff --git a/crates/exav-core/src/patterns.rs b/crates/exav-core/src/patterns.rs index e4d248bc..32ea6274 100644 --- a/crates/exav-core/src/patterns.rs +++ b/crates/exav-core/src/patterns.rs @@ -3,25 +3,30 @@ /// The EICAR anti-virus test string, assembled at runtime. /// /// Re-exported rather than redefined so the sequence has exactly one home in the -/// workspace — see [`exav_unpack::eicar`] for why it is never stored as a +/// workspace: see [`exav_unpack::eicar`] for why it is never stored as a /// literal. pub use exav_unpack::eicar; -/// Parse a `Name=HEX` database into `(name, bytes)` pairs. -pub fn parse_simple(text: &str) -> Result)>, String> { +/// Parse a `Name=HEX` database into `(name, bytes)` pairs, and the number of +/// lines skipped because they are not a name and plain hex: no `=`, or a body +/// that is not an even number of hex digits (one with a wildcard, say). +pub fn parse_simple(text: &str) -> (Vec<(String, Vec)>, usize) { let mut patterns = Vec::new(); - for (i, line) in text.lines().enumerate() { + let mut skipped = 0; + for line in text.lines() { let line = line.trim(); if line.is_empty() || line.starts_with('#') { continue; } - let (name, hex) = line + let parsed = line .split_once('=') - .ok_or_else(|| format!("line {}: expected Name=HEX", i + 1))?; - let bytes = crate::hexsig::decode_hex(hex).map_err(|e| format!("line {}: {e}", i + 1))?; - patterns.push((name.trim().to_string(), bytes)); + .and_then(|(name, hex)| Some((name.trim().to_string(), crate::hexsig::decode_hex(hex).ok()?))); + match parsed { + Some(p) => patterns.push(p), + None => skipped += 1, + } } - Ok(patterns) + (patterns, skipped) } #[cfg(test)] @@ -30,14 +35,16 @@ mod tests { #[test] fn simple_db_lines_parse() { - let got = parse_simple("# comment\nSig.A =4142\n\nSig.B=ff\n").unwrap(); + let got = parse_simple("# comment\nSig.A =4142\n\nno separator\nSig.W=41??\nSig.B=ff\n"); assert_eq!( got, - vec![ - ("Sig.A".to_string(), b"AB".to_vec()), - ("Sig.B".to_string(), vec![0xff]) - ] + ( + vec![ + ("Sig.A".to_string(), b"AB".to_vec()), + ("Sig.B".to_string(), vec![0xff]) + ], + 2 + ) ); - assert!(parse_simple("no separator").is_err()); } } diff --git a/crates/exav-core/src/source.rs b/crates/exav-core/src/source.rs index 7a991d65..82cdf6a7 100644 --- a/crates/exav-core/src/source.rs +++ b/crates/exav-core/src/source.rs @@ -44,16 +44,19 @@ mod http { /// Open a URL, probing its length and range support with a single /// `bytes=0-0` request. pub fn open(url: &str) -> io::Result { - let agent = ureq::AgentBuilder::new() + let agent: ureq::Agent = ureq::Agent::config_builder() .user_agent(concat!("exav/", env!("CARGO_PKG_VERSION"))) - .build(); + .build() + .into(); let resp = agent .get(url) - .set("Range", "bytes=0-0") + .header("Range", "bytes=0-0") .call() .map_err(|e| io::Error::other(e.to_string()))?; let total = resp - .header("Content-Range") + .headers() + .get("Content-Range") + .and_then(|cr| cr.to_str().ok()) .and_then(|cr| cr.rsplit('/').next()) .and_then(|n| n.trim().parse::().ok()) .ok_or_else(|| { @@ -102,11 +105,22 @@ mod http { let resp = self .agent .get(&self.url) - .set("Range", &format!("bytes={start}-{last}")) + .header("Range", format!("bytes={start}-{last}")) .call() .map_err(|e| io::Error::other(e.to_string()))?; + // Anything but a partial response is not the range asked for: a + // `200` is the object from its first byte. + if resp.status() != 206 { + return Err(io::Error::other(format!( + "range request answered with status {}", + resp.status() + ))); + } let mut buf = Vec::new(); - resp.into_reader().take(size).read_to_end(&mut buf)?; + resp.into_body() + .into_reader() + .take(size) + .read_to_end(&mut buf)?; self.bytes_fetched += buf.len() as u64; self.requests += 1; self.block = buf; @@ -172,3 +186,81 @@ mod http { #[cfg(feature = "http")] pub use http::HttpRangeReader; + +#[cfg(all(test, feature = "http"))] +mod tests { + use super::HttpRangeReader; + use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; + use std::net::TcpListener; + + /// Serve `body` on a local port, one request per connection, answering a + /// range request with `206` and that range, except that past the first + /// request a `full` server answers `200` and the whole body. + fn serve(body: Vec, full: bool) -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/obj", listener.local_addr().unwrap()); + std::thread::spawn(move || { + for (n, conn) in listener.incoming().enumerate() { + let Ok(mut conn) = conn else { return }; + let mut range = None; + let mut reader = BufReader::new(conn.try_clone().unwrap()); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" { + break; + } + if let Some(r) = line.to_ascii_lowercase().strip_prefix("range: bytes=") { + let (a, b) = r.trim().split_once('-').unwrap(); + range = Some((a.parse::().unwrap(), b.parse::().unwrap())); + } + } + let head = match range { + Some((a, b)) if !(full && n > 0) => { + let part = &body[a..=b]; + let h = format!( + "HTTP/1.1 206 Partial Content\r\nContent-Range: bytes {a}-{b}/{}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len(), + part.len() + ); + [h.as_bytes(), part].concat() + } + _ => { + let h = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + [h.as_bytes(), &body].concat() + } + }; + let _ = conn.write_all(&head); + } + }); + url + } + + #[test] + fn ranges_are_read_where_they_are() { + let body: Vec = (0..200_000u32).map(|i| (i % 251) as u8).collect(); + let mut r = HttpRangeReader::open(&serve(body.clone(), false)).unwrap(); + assert_eq!(r.len(), body.len() as u64); + let mut buf = vec![0; 1000]; + r.seek(SeekFrom::Start(150_000)).unwrap(); + r.read_exact(&mut buf).unwrap(); + assert_eq!(buf, body[150_000..151_000]); + let mut all = Vec::new(); + r.seek(SeekFrom::Start(0)).unwrap(); + r.read_to_end(&mut all).unwrap(); + assert_eq!(all, body); + } + + /// A server that answers a range with the whole object is not read as + /// though the object started at the range. + #[test] + fn a_range_answered_whole_is_an_error() { + let body: Vec = (0..200_000u32).map(|i| (i % 251) as u8).collect(); + let mut r = HttpRangeReader::open(&serve(body, true)).unwrap(); + r.seek(SeekFrom::Start(150_000)).unwrap(); + let mut buf = vec![0; 1000]; + assert!(r.read_exact(&mut buf).is_err()); + } +} diff --git a/crates/exav-core/src/yara/atoms.rs b/crates/exav-core/src/yara/atoms.rs index 37394b64..42d092a5 100644 --- a/crates/exav-core/src/yara/atoms.rs +++ b/crates/exav-core/src/yara/atoms.rs @@ -1,11 +1,12 @@ -//! Atom-based Aho-Corasick prefilter for the YARA string matcher. +//! Atom prefilter for the YARA string matcher. //! //! Scanning every compiled pattern over the whole buffer (one `find_all` per //! pattern) is O(patterns × bytes) and dominates scan time on many-rule sets. //! This module builds a *required-literal gate*: for each pattern we extract one //! or more **atoms**: literal byte substrings that MUST appear in EVERY match of -//! the pattern. A single shared Aho-Corasick automaton (daachorse) is run over -//! the buffer ONCE; a pattern's `find_all` is then executed ONLY if one of its +//! the pattern. One anchor index over every atom ([`crate::grams`], the one the +//! signature engine uses) is swept over the buffer ONCE; a pattern's `find_all` +//! is then executed ONLY if one of its //! atoms was found (or if the pattern is *ungated*, i.e. has no provable required //! literal, e.g. `xor`, `/a+/`, alternations with an atomless branch). //! @@ -27,16 +28,16 @@ use std::collections::HashMap; -use daachorse::{DoubleArrayAhoCorasick, DoubleArrayAhoCorasickBuilder}; use regex_syntax::hir::literal::{ExtractKind, Extractor}; use serde::{Deserialize, Serialize}; use crate::byte_source::ByteSource; +use crate::grams::{structural, sweep, GramStats, Grams, IndexAnchor}; use crate::yara::matcher::{parse_hir, widen_hir, Base64Sub, Needle}; /// Maximum atom window length taken from a literal / base64 needle. A longer -/// window is a strictly more selective (rarer) gate; the cap bounds the shared -/// automaton's size for pathologically long literals. +/// window is a strictly more selective (rarer) gate; the cap bounds what the +/// index holds for pathologically long literals. const MAX_ATOM: usize = 16; /// The gate classification of one compiled pattern. @@ -53,31 +54,10 @@ pub(crate) enum Gate { // Per-pattern atom extraction (compile time) // --------------------------------------------------------------------------- -/// Selectivity score of a candidate atom window, reused verbatim from the -/// AV-engine heuristic (`engine::parse::anchor_score`): a low-entropy run (a -/// constant byte, or a 2-symbol repeat) matches repetitive content millions of -/// times and is a terrible prefilter even when long, so it is down-ranked; a -/// varied run scores its length (longer = rarer = better). Kept byte-identical -/// here because `engine::parse` is a private module unreachable from `yara`. -fn anchor_score(b: &[u8]) -> usize { - let mut seen = [false; 256]; - let mut distinct = 0usize; - for &x in b { - if !seen[x as usize] { - seen[x as usize] = true; - distinct += 1; - } - } - match distinct { - 0 | 1 => 1, - 2 => 3.min(b.len()), - _ => b.len(), - } -} - /// Picks the most selective window of length `min(bytes.len(), MAX_ATOM)` from a -/// required literal. Any contiguous substring of a required literal is itself -/// required, so this preserves soundness while choosing a rarer anchor. +/// required literal, by its bytes alone ([`structural`]). Any contiguous +/// substring of a required literal is itself required, so this preserves +/// soundness while choosing a rarer anchor. fn best_window(bytes: &[u8]) -> &[u8] { let len = bytes.len().min(MAX_ATOM); if bytes.len() <= MAX_ATOM { @@ -86,7 +66,7 @@ fn best_window(bytes: &[u8]) -> &[u8] { let mut best_start = 0; let mut best_score = 0; for start in 0..=(bytes.len() - len) { - let score = anchor_score(&bytes[start..start + len]); + let score = structural(&bytes[start..start + len]); if score > best_score { best_score = score; best_start = start; @@ -98,7 +78,7 @@ fn best_window(bytes: &[u8]) -> &[u8] { /// Gate for a `Literal` pattern (plain/`nocase`/`wide`, no `xor`). Each non-empty /// needle contributes one required-literal window; the pattern matches iff SOME /// needle matches, so the atom set is the disjunction over needles. An `xor` -/// pattern transforms the bytes and cannot be gated by a plain automaton, so its +/// pattern transforms the bytes and cannot be gated by a plain literal, so its /// caller passes `xor_present = true` → ungated. pub(crate) fn literal_gate(needles: &[Needle], xor_present: bool) -> Gate { if xor_present { @@ -257,9 +237,9 @@ fn required_literals(hir: ®ex_syntax::hir::Hir, kind: ExtractKind) -> Option< /// Picks the more selective of two candidate required-literal sets. A set is only /// as strong as its weakest (lowest-scoring) atom, since ANY atom hit triggers a -/// scan, so we rank by the minimum [`anchor_score`] across the set. +/// scan, so we rank by the minimum [`structural`] score across the set. fn pick_more_selective(a: Option>>, b: Option>>) -> Option>> { - let score = |s: &[Vec]| s.iter().map(|a| anchor_score(a)).min().unwrap_or(0); + let score = |s: &[Vec]| s.iter().map(|a| structural(a)).min().unwrap_or(0); match (a, b) { (None, x) | (x, None) => x, (Some(x), Some(y)) => { @@ -273,12 +253,12 @@ fn pick_more_selective(a: Option>>, b: Option>>) -> Opti } // --------------------------------------------------------------------------- -// The compiled gate (one automaton over all atoms) +// The compiled gate (one index over all atoms) // --------------------------------------------------------------------------- -/// De-duplicating pool of atoms feeding one Aho-Corasick automaton. Distinct -/// atoms map to automaton values; each value carries the list of pattern ids that -/// share that atom (`groups`). +/// De-duplicating pool of the atoms of one case. Distinct atoms are values of +/// the index; each value carries the list of pattern ids that share that atom +/// (`groups`). #[derive(Default)] struct Pool { index: HashMap, usize>, @@ -302,116 +282,83 @@ impl Pool { self.groups[idx].push(pid); } } - - /// Builds the automaton. On a (very unlikely) build failure the pool's - /// patterns are force-ungated so they are still scanned, never silently - /// dropped. - fn finish(self, ungated: &mut [bool]) -> Lane { - if self.atoms.is_empty() { - return Lane::default(); - } - match DoubleArrayAhoCorasickBuilder::new() - .match_kind(daachorse::MatchKind::Standard) - .build_with_values(self.atoms.iter().zip(0u32..)) - { - Ok(ac) => Lane { - ac: Some(ac), - groups: self.groups, - runs: crate::engine::longest_runs(&self.atoms), - }, - Err(_) => { - for g in &self.groups { - for &pid in g { - ungated[pid] = true; - } - } - Lane::default() - } - } - } -} - -/// One automaton of the gate: over its atoms, the patterns behind each, and -/// the longest run of each byte value in them (see [`crate::engine::sweep`]). -struct Lane { - ac: Option>, - groups: Vec>, - runs: [u32; 256], -} - -impl Default for Lane { - fn default() -> Self { - Lane { - ac: None, - groups: Vec::new(), - runs: [0; 256], - } - } } -/// The prefilter gate for a whole compiled rule set. Case-sensitive atoms are -/// searched in the raw buffer; `nocase` atoms in its bytes lowercased, both in -/// the one read of it. +/// The prefilter gate for a whole compiled rule set: one index over every +/// atom, the case-sensitive ones in partition 0 and the `nocase` ones, +/// lowercased, in partition 1, found in the one read of the object. pub(crate) struct PatternGate { - cs: Lane, - ci: Lane, + grams: Grams, + /// Per partition, per atom, the patterns behind it. + groups: [Vec>; 2], /// Per pattern id: `true` = always scan (ungated). ungated: Vec, - /// Number of patterns with a required-literal gate (diagnostics). - gated: usize, - /// Kill switch: when set (via `EXAV_YARA_NO_GATE`), the prefilter is bypassed - /// and every pattern is scanned. A safety valve: results are identical to a - /// gated scan by construction, but this lets an operator rule the prefilter - /// out if a gate bug is ever suspected. Read once at build time (never on the - /// hot path). - bypass: bool, } impl PatternGate { /// Builds the gate from one [`Gate`] per pattern (index == pattern id). pub(crate) fn from_gates(gates: &[Gate]) -> PatternGate { - let n = gates.len(); - let mut ungated = vec![false; n]; - let mut cs = Pool::default(); - let mut ci = Pool::default(); + let mut ungated = vec![false; gates.len()]; + let mut pools: [Pool; 2] = Default::default(); for (pid, g) in gates.iter().enumerate() { match g { - Gate::Ungated => ungated[pid] = true, - Gate::Atoms { atoms, nocase } => { - let pool = if *nocase { &mut ci } else { &mut cs }; + // A pattern with no atom at all would never be slated to run. + Gate::Atoms { atoms, nocase } + if !atoms.is_empty() && atoms.iter().all(|a| !a.is_empty()) => + { for a in atoms { - pool.add(a.clone(), pid); + pools[*nocase as usize].add(a.clone(), pid); } } + _ => ungated[pid] = true, } } - let cs = cs.finish(&mut ungated); - let ci = ci.finish(&mut ungated); - // Whatever remains not-ungated after the (possibly failing) builds is - // in fact gated. - let gated = ungated.iter().filter(|&&u| !u).count(); + let atoms = || pools.iter().flat_map(|p| &p.atoms); + let stats = GramStats::build(atoms().count(), |f| atoms().for_each(|a| f(a))); + let anchors: Vec = pools + .iter() + .enumerate() + .flat_map(|(part, p)| { + p.atoms + .iter() + .zip(0u32..) + .map(move |(bytes, value)| IndexAnchor { + part: part as u32, + nocase: part == 1, + value, + bytes, + }) + }) + .collect(); + let grams = Grams::build(&anchors, &stats); + let [cs, ci] = pools; PatternGate { - cs, - ci, + grams, + groups: [cs.groups, ci.groups], ungated, - gated, - bypass: std::env::var_os("EXAV_YARA_NO_GATE").is_some(), } } + /// Whether the gate is one for `patterns` patterns: one entry each, and + /// every gated one behind some atom, so it can be slated to run. + pub(crate) fn fits(&self, patterns: usize) -> bool { + let mut reached = self.ungated.clone(); + for &pid in self.groups.iter().flatten().flatten() { + reached[pid] = true; + } + self.ungated.len() == patterns && reached.iter().all(|&r| r) + } + /// Number of patterns that carry a required-literal gate. #[allow(dead_code)] pub(crate) fn gated_count(&self) -> usize { - self.gated + self.ungated.iter().filter(|&&u| !u).count() } /// Returns, per pattern id, whether `find_all` must be run: `true` for every /// ungated pattern, plus every gated pattern one of whose atoms occurs in /// `data`. A gated pattern whose atoms are all absent provably has no match. pub(crate) fn select(&self, src: &dyn ByteSource) -> Vec { - if self.bypass { - return vec![true; self.ungated.len()]; - } let mut run = self.ungated.clone(); // Number of gated patterns still to resolve; lets us stop scanning once // every pattern is already slated to run (dense buffers). @@ -419,20 +366,12 @@ impl PatternGate { if pending == 0 { return run; } - let (mut lanes, mut groups) = (Vec::new(), Vec::new()); - for (lane, fold) in [(&self.cs, false), (&self.ci, true)] { - if let Some(ac) = &lane.ac { - lanes.push(crate::engine::Lane { - ac, - fold, - runs: &lane.runs, - }); - groups.push(&lane.groups); - } - } - crate::engine::sweep(&lanes, src, &mut |k, value, _, _| { - mark(groups[k], value, &mut run, &mut pending) - }); + sweep( + &self.grams, + &[true, true], + src, + &mut |part, value, _, _, _| mark(&self.groups[part], value, &mut run, &mut pending), + ); run } } @@ -452,85 +391,44 @@ fn mark(groups: &[Vec], value: u32, run: &mut [bool], pending: &mut usize // Serialization of the compiled gate // --------------------------------------------------------------------------- // -// The atom automaton is the expensive artifact this whole module exists to -// avoid rebuilding on load, so the gate is serialized in its COMPILED form: each -// daachorse automaton travels as its own byte serialization (`serialize`), and -// the per-atom pattern-id groups + the ungated bitmap travel as plain data. On -// load the automaton is reconstructed via daachorse's checked `deserialize` (no -// rebuild). `bypass` is NOT serialized: it is an operator kill switch read -// afresh from the environment on load, exactly as at build time. - -/// True when the prefilter kill switch is set. Read once (build or load time), -/// never on the hot path. -fn bypass_env() -> bool { - std::env::var_os("EXAV_YARA_NO_GATE").is_some() -} +// The index travels as its own binary form ([`Grams::write`]), the per-atom +// pattern-id groups and the ungated bitmap as plain data. A load checks every +// id against what it indexes, so a corrupt blob fails loudly, never misreads. impl Serialize for PatternGate { fn serialize(&self, s: S) -> Result { use crate::database::Blob; - let ac_cs = self.cs.ac.as_ref().map(|ac| ac.serialize()); - let ac_ci = self.ci.ac.as_ref().map(|ac| ac.serialize()); - ( - ac_cs.as_deref().map(Blob), - &self.cs.groups, - &self.cs.runs[..], - ac_ci.as_deref().map(Blob), - &self.ci.groups, - &self.ci.runs[..], - &self.ungated, - self.gated, - ) - .serialize(s) + let mut index = Vec::new(); + self.grams + .write(&mut index) + .map_err(serde::ser::Error::custom)?; + (Blob(&index), &self.groups, &self.ungated).serialize(s) } } impl<'de> Deserialize<'de> for PatternGate { fn deserialize>(d: D) -> Result { use crate::database::BlobBuf; - type Wire = ( - Option, - Vec>, - Vec, - Option, - Vec>, - Vec, - Vec, - usize, - ); - let (ac_cs, groups_cs, runs_cs, ac_ci, groups_ci, runs_ci, ungated, gated): Wire = + let (index, groups, ungated): (BlobBuf, [Vec>; 2], Vec) = Deserialize::deserialize(d)?; - - fn rebuild( - bytes: Option, - groups: Vec>, - runs: Vec, - ) -> Result { - let runs = runs - .try_into() - .map_err(|_| serde::de::Error::custom("bad atom run table"))?; - let ac = match bytes { - None => None, - Some(b) => { - // Checked deserialize: rejects data that would cause - // out-of-bounds access (the database is a trusted artifact, - // but a corrupt/mismatched blob must fail loudly, not misread). - let (ac, _rest) = - DoubleArrayAhoCorasick::::deserialize(&b.0).map_err(|e| { - serde::de::Error::custom(format!("bad atom automaton: {e}")) - })?; - Some(ac) - } - }; - Ok(Lane { ac, groups, runs }) + if groups + .iter() + .flatten() + .flatten() + .any(|&pid| pid >= ungated.len()) + { + return Err(serde::de::Error::custom("bad atom pattern id")); + } + let mut index = &index.0[..]; + let grams = Grams::read(&mut index, &[groups[0].len(), groups[1].len()]) + .map_err(|e| serde::de::Error::custom(format!("bad atom index: {e}")))?; + if !index.is_empty() { + return Err(serde::de::Error::custom("bad atom index: trailing bytes")); } - Ok(PatternGate { - cs: rebuild(ac_cs, groups_cs, runs_cs)?, - ci: rebuild(ac_ci, groups_ci, runs_ci)?, + grams, + groups, ungated, - gated, - bypass: bypass_env(), }) } } @@ -598,4 +496,38 @@ mod tests { assert_eq!(gate.select(&cache), want, "{hay:?}"); } } + + /// A pattern with no atom runs always. A gate read back is refused unless + /// it has an entry per pattern, a way to slate each to run, and nothing + /// after its index. + #[test] + fn a_gate_that_cannot_run_a_pattern_is_refused() { + let gates = [ + Gate::Atoms { + atoms: vec![b"abcd".to_vec()], + nocase: false, + }, + Gate::Atoms { + atoms: vec![], + nocase: false, + }, + ]; + let gate = PatternGate::from_gates(&gates); + assert_eq!(gate.select(&&b"nothing"[..]), [false, true]); + assert!(gate.fits(2) && !gate.fits(3)); + let bytes = rmp_serde::to_vec(&gate).unwrap(); + let mut orphan: PatternGate = rmp_serde::from_slice(&bytes).unwrap(); + orphan.groups[0].clear(); + assert!(!orphan.fits(2), "a gated pattern behind no atom"); + let (index, groups, ungated): (crate::database::BlobBuf, [Vec>; 2], Vec) = + rmp_serde::from_slice(&bytes).unwrap(); + let mut longer = index.0.clone(); + longer.push(0); + let tampered = + rmp_serde::to_vec(&(crate::database::Blob(&longer), groups, ungated)).unwrap(); + assert!( + rmp_serde::from_slice::(&tampered).is_err(), + "trailing bytes" + ); + } } diff --git a/crates/exav-core/src/yara/matcher.rs b/crates/exav-core/src/yara/matcher.rs index f0e01b29..c5f03e24 100644 --- a/crates/exav-core/src/yara/matcher.rs +++ b/crates/exav-core/src/yara/matcher.rs @@ -92,7 +92,7 @@ pub(crate) struct Base64Sub { // *definition* (regex source + flags, or the literal/base64 needle bytes), from // which [`PatternDef::compile`] rebuilds an identical `PatternMatcher` on load. // Rebuilding a single pattern's regex is cheap; the aggregate cost that the -// serialized database avoids is the daachorse atom-automaton build (see +// serialized database avoids is the atom index build (see // [`crate::yara::atoms`]), which is serialized separately. /// The serializable definition of one `base64`/`base64wide` sub-pattern. Mirrors diff --git a/crates/exav-core/src/yara/mod.rs b/crates/exav-core/src/yara/mod.rs index 851d1393..9ba4c972 100644 --- a/crates/exav-core/src/yara/mod.rs +++ b/crates/exav-core/src/yara/mod.rs @@ -10,7 +10,7 @@ //! database-build time ([`YaraDb::finalize`]) the rules are compiled and the //! compiled rule set is serialized into `YaraDb::compiled_blob`, so a prebuilt //! `.exavdb` loads the YARA engine WITHOUT recompiling, in particular without -//! rebuilding the expensive daachorse atom automaton. The daachorse automaton + +//! extracting the atoms and building their index again. The atom index + //! owned IR + per-pattern definitions travel in the blob; only the per-pattern //! regexes (whose compiled automata are not serializable) are recompiled from //! their stored source on load. The rule *sources* are also retained, so a build @@ -88,13 +88,14 @@ pub fn compile(src: &str) -> Result { use serde::{Deserialize, Serialize}; /// Format version of [`YaraDb::compiled_blob`]. Bumped whenever the serialized -/// compiled-rules layout (the `Rules` blob: IR, pattern defs, atom automaton) +/// compiled-rules layout (the `Rules` blob: IR, pattern defs, atom index) /// changes in an incompatible way. A loaded blob whose version does not match /// this build is ignored and the rules are recompiled from source, so a stale /// blob is never misread: the coverage is preserved, just without the load-time /// shortcut. (This is independent of the outer `.exavdb` format serial in -/// `crate::database`, which guards the whole file.) Version 2 stores the atom -/// automatons as binary blobs, with their run tables. +/// `crate::database`, which guards the whole file.) Version 1 is exav 0.0.1's; +/// version 2 replaced the atom automaton with the anchor index the signature +/// engine uses. Bumped once per release at most. #[cfg(feature = "yara")] const YARA_BLOB_VERSION: u32 = 2; @@ -122,7 +123,7 @@ pub struct YaraDb { /// The serialized COMPILED rule set, produced by [`Self::finalize`] at /// database-build time (only when built with the `yara` feature). When /// present and version-compatible, [`Self::rules`] deserializes it (skipping - /// the expensive daachorse atom-automaton build) instead of recompiling from + /// the atom extraction and index build) instead of recompiling from /// `sources`. This is what makes a prebuilt `.exavdb` load its YARA engine /// without recompiling, consistent with the rest of the database (which also /// serializes its compiled form). Absent when the database was built without @@ -177,7 +178,7 @@ impl YaraDb { /// validate them and record which rules were rejected (so the coverage gap is /// captured in the built database) and (b) SERIALIZE the compiled rule set /// into `Self::compiled_blob`, so a prebuilt `.exavdb` loads the YARA engine - /// without recompiling (skipping the expensive atom-automaton build), exactly + /// without recompiling (skipping the atom extraction and index build), exactly /// as the rest of the database serializes its compiled form. Idempotent. pub fn finalize(&mut self) { #[cfg(not(feature = "yara"))] @@ -218,7 +219,7 @@ impl YaraDb { return None; } // Fast path: a version-compatible prebuilt compiled blob loads - // the rule set (incl. the atom automaton) without recompiling. + // the rule set (incl. the atom index) without recompiling. if let Some(blob) = &self.compiled_blob { if self.blob_version == YARA_BLOB_VERSION { match Rules::deserialize(blob) { @@ -397,7 +398,7 @@ fn preview(rejected: &[String]) -> String { // differential (~17k rules, 0 disagreements). The reasons are structural: // // * The AV pattern matcher is non-exhaustive by design (it reports the first -// hit and stops, and prunes on Aho-Corasick prefilter economics). YARA +// hit and stops, and prunes on anchor prefilter economics). YARA // conditions need *exhaustive* per-pattern match sets: counts (`#a`), the // i-th offset/length (`@a[i]`/`!a[i]`), `$a at N`, `$a in (a..b)`, and // `N of`/`P% of` quantifiers all require every match, not just existence. diff --git a/crates/exav-core/src/yara/scanner.rs b/crates/exav-core/src/yara/scanner.rs index 92e643ab..cf4a9010 100644 --- a/crates/exav-core/src/yara/scanner.rs +++ b/crates/exav-core/src/yara/scanner.rs @@ -41,14 +41,14 @@ pub struct Rules { pub(crate) imported: Vec, /// Whether any rule uses the `entrypoint` keyword. pub(crate) uses_entrypoint: bool, - /// Required-literal Aho-Corasick prefilter: which patterns even need to be + /// Required-literal prefilter: which patterns even need to be /// scanned for a given buffer (see [`crate::yara::atoms`]). pub(crate) gate: PatternGate, } /// The serialized form of a compiled [`Rules`] set. Everything here is cheap to -/// (de)serialize; the expensive-to-rebuild atom automaton is inside `gate` -/// (serialized as daachorse's own byte form), and the per-pattern regexes are +/// (de)serialize; the atom index is inside `gate` (in its own binary form), and +/// the per-pattern regexes are /// recompiled from `defs` on load. Serialized with borrowed fields, deserialized /// into the owned twin [`RulesBlobOwned`]. #[derive(Serialize)] @@ -73,8 +73,8 @@ struct RulesBlobOwned { impl Rules { /// Serializes the compiled rule set to MessagePack bytes for embedding in the - /// on-disk database. The atom automaton travels in its compiled (daachorse) - /// form; only the per-pattern regex SOURCES travel, not the compiled regex + /// on-disk database. The atom index travels in its built form; only the + /// per-pattern regex SOURCES travel, not the compiled regex /// automata. See [`Rules::deserialize`] for the reverse. pub(crate) fn serialize(&self) -> Vec { let blob = RulesBlobRef { @@ -85,20 +85,25 @@ impl Rules { uses_entrypoint: self.uses_entrypoint, gate: &self.gate, }; - // Infallible for this data (no maps with non-string keys, no unsupported - // types); `rmp_serde` only errors on serializer I/O, and Vec never fails. - rmp_serde::to_vec(&blob).expect("serialize compiled YARA rules") + // Infallible for this data: every length is known and fits, and + // writing to a Vec never fails. + crate::database::to_vec_indexed(&blob).expect("serialize compiled YARA rules") } /// Reconstructs a compiled rule set from [`Rules::serialize`] bytes. The - /// daachorse atom automaton is deserialized (NOT rebuilt); the IR and rule + /// atom index is read back (NOT rebuilt from the atoms); the IR and rule /// structs are deserialized; and each pattern's `PatternMatcher` is /// recompiled from its stored [`PatternDef`]. This skips the aggregate build - /// cost (source parse, AST->IR lowering, atom extraction, and the daachorse - /// automaton build), paying only per-pattern regex compilation. + /// cost (source parse, AST->IR lowering, atom extraction, and the atom + /// index build), paying only per-pattern regex compilation. pub(crate) fn deserialize(bytes: &[u8]) -> Result { let blob: RulesBlobOwned = rmp_serde::from_slice(bytes) .map_err(|e| Error::new(format!("invalid serialized YARA rules: {e}")))?; + if !blob.gate.fits(blob.defs.len()) { + return Err(Error::new( + "invalid serialized YARA rules: the prefilter does not fit the patterns", + )); + } let mut patterns = Vec::with_capacity(blob.defs.len()); for def in &blob.defs { patterns.push(def.compile()?); @@ -946,15 +951,15 @@ mod serialize_tests { } /// Load-time measurement (not a correctness gate): compiles a large real - /// ruleset from source (which includes the expensive daachorse atom-automaton - /// build) vs. deserializing the serialized compiled form (which skips it), and + /// ruleset from source (which includes the atom index build) vs. + /// deserializing the serialized compiled form (which skips it), and /// prints the timings + blob size. Ignored by default; run with: /// /// ```text /// cargo test -p exav-core --features yara --lib -- --ignored --nocapture load_time /// ``` /// - /// The source ruleset path can be overridden with `EXAV_YARA_BENCH_RULES` + /// The source ruleset path can be overridden with `EXAV_DEBUG_YARA_BENCH_RULES` /// (e.g. a concatenated signature-base tree); it defaults to the in-repo /// `corpus/securiteinfo/securiteinfo.yara`. #[test] @@ -963,7 +968,7 @@ mod serialize_tests { use crate::yara::compiler::Compiler; use std::time::Instant; - let path = std::env::var("EXAV_YARA_BENCH_RULES").unwrap_or_else(|_| { + let path = std::env::var("EXAV_DEBUG_YARA_BENCH_RULES").unwrap_or_else(|_| { format!( "{}/../../corpus/securiteinfo/securiteinfo.yara", env!("CARGO_MANIFEST_DIR") @@ -989,8 +994,8 @@ mod serialize_tests { let bytes = fresh.serialize(); let serialize = t1.elapsed(); - // Deserialize (the load path): reconstruct the atom automaton + IR, and - // recompile only the per-pattern regexes, skipping the automaton BUILD. + // Deserialize (the load path): read back the atom index + IR, and + // recompile only the per-pattern regexes, skipping the index BUILD. let t2 = Instant::now(); let loaded = Rules::deserialize(&bytes).expect("deserialize"); let deserialize = t2.elapsed(); @@ -1003,9 +1008,9 @@ mod serialize_tests { "rules: {n_rules}, patterns: {n_patterns}, blob: {} bytes", bytes.len() ); - eprintln!("compile from source (incl. atom-automaton build): {compile_from_source:?}"); - eprintln!("serialize compiled form: {serialize:?}"); - eprintln!("deserialize (load, skips automaton build): {deserialize:?}"); + eprintln!("compile from source (incl. atom index build): {compile_from_source:?}"); + eprintln!("serialize compiled form: {serialize:?}"); + eprintln!("deserialize (load, skips index build): {deserialize:?}"); if deserialize.as_nanos() > 0 { eprintln!( "speedup (compile / deserialize): {:.1}x", diff --git a/crates/exav-core/tests/suites/bytecode_gating.rs b/crates/exav-core/tests/suites/bytecode_gating.rs index bdfed809..6ba0c83f 100644 --- a/crates/exav-core/tests/suites/bytecode_gating.rs +++ b/crates/exav-core/tests/suites/bytecode_gating.rs @@ -150,6 +150,51 @@ fn trigger_gated_detection_in_a_scan() { } } +/// A program gated on `EXAVTESTMARKER` that `malloc`s 32 bytes and `write`s +/// them, then names what `write` returned: `BC.Test.Write.AllOk` for 32, +/// `BC.Test.Write.Fail1` for anything else (`BC.Test.Write.MallocNull` if +/// `malloc` fails). Assembled with the format's encoders. +const WRITE_32: &str = "\ +ClamBCaghfdeiodke|aa```c``abhc``|ancflfafmfbfcfmb`cnbicicnbbc``ajaap`clamcoincidencejb:4096 +BC.Test.Write;Engine:56-255,Target:0;0;45584156544553544d41524b4552 +Tedaacb`bbadb`baabbadb`bdbiaahdbdaahdbdaah +Ebbaacaebed|amcgefdgfgifbgegcgnfafmfef``bbabfd|agmfaflflfofcf``acbed|afggbgifdgef`` +Gagag`@`bgdBbdBcdBnbBdeBefBcgBdgBnbBgeBbgBifBdgBefBnbBmdBafBlfBlfBofBcfBndBegBlfBlf@`bad@Aa`bhdBbdBcdBnbBdeBefBcgBdgBnbBgeBbgBifBdgBefBnbBadBlfBlfBodBkf@`bad@Ac`bidBbdBcdBnbBdeBefBcgBdgBnbBgeBbgBifBdgBefBnbBfdBafBifBlfBac@`bad@Ae` +A`b`bLagbad`aa`b`b`b`b`aa`b`b`b`b`Falae +Bbad`ababbaB`bdaaaaeabad`@`Taaaaaaab +Bb`bababbaeAb`BhadTcab`b@d +Bb`bacabbac`B`bdaaadfab`bacB`bdTaaadadac +Bb`baeabbaeAd`BcadTcab`b@d +Bb`bafabbaeAf`BcadTcab`b@dE +"; + +/// A scan's limits reach the programs it runs: a `write` past the largest +/// buffer the scan may hold, or past the bytes it has left to scan, fails, +/// and the program goes on to see it. +#[test] +fn a_write_past_the_scan_limits_fails() { + let mut loader = exav_core::loader::Builder::new(); + loader.add_named_bytes("w.cbc", WRITE_32.as_bytes(), true); + let db = loader.build().unwrap(); + let input = b"..EXAVTESTMARKER.."; + let found = |opts: &ScanOptions| match analyze(&db, input, opts).verdict { + Verdict::Infected { signature, .. } => Some(signature), + v => panic!("{v:?}"), + }; + let ok = Some("BC.Test.Write.AllOk".to_string()); + let failed = Some("BC.Test.Write.Fail1".to_string()); + assert_eq!(found(&ScanOptions::default()), ok); + let mut opts = ScanOptions::default(); + opts.limits.max_buffer_bytes = 31; + assert_eq!(found(&opts), failed); + // What is extracted counts against it, not the input. + let mut opts = ScanOptions::default(); + opts.limits.max_scanned_bytes = 31; + assert_eq!(found(&opts), failed); + opts.limits.max_scanned_bytes = 32; + assert_eq!(found(&opts), ok); +} + #[test] fn forced_mode_runs_regardless_of_trigger() { let cbc = synth_cbc("Synth.BC.Detect", b"MALWARE"); diff --git a/crates/exav-core/tests/suites/one_scan.rs b/crates/exav-core/tests/suites/one_scan.rs index 457257d6..7691c3b8 100644 --- a/crates/exav-core/tests/suites/one_scan.rs +++ b/crates/exav-core/tests/suites/one_scan.rs @@ -24,7 +24,7 @@ fn both(blob: &[u8], opts: &ScanOptions) -> [(&'static str, Verdict); 2] { fn zip_with(member: &[u8], comment: &str) -> Vec { let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); - z.set_comment(comment); + z.set_comment(comment).unwrap(); z.start_file( "member.bin", zip::write::SimpleFileOptions::default() diff --git a/crates/exav-core/tests/yara_coverage_difftest.rs b/crates/exav-core/tests/yara_coverage_difftest.rs index 3297939a..6dd72bbd 100644 --- a/crates/exav-core/tests/yara_coverage_difftest.rs +++ b/crates/exav-core/tests/yara_coverage_difftest.rs @@ -1,12 +1,12 @@ //! Coarse correctness spot-check over a real corpus. //! -//! For every rule file in `$EXAV_YARA_CORPUS` that exav compiles with ZERO +//! For every rule file in `$EXAV_DEBUG_YARA_CORPUS` that exav compiles with ZERO //! rejections *and* that yara-x also accepts, both engines are asked to scan //! the same handful of byte inputs; the set of matching rule identifiers must be //! identical. This confirms the coverage numbers aren't hiding silent //! mis-compiles (a rule that "compiles" but matches differently). //! -//! No-op unless `EXAV_YARA_CORPUS` points at a directory of `.yar`/`.yara` +//! No-op unless `EXAV_DEBUG_YARA_CORPUS` points at a directory of `.yar`/`.yara` //! files, so it never runs in the default `cargo test`, and skipped when the //! `yr` binary is absent. See `yara_difftest.rs` for why yara-x is a program //! here rather than a dependency. @@ -53,10 +53,10 @@ fn exav_set(rules: &exav_core::yara::Rules, data: &[u8]) -> BTreeSet { .collect() } -/// The oracle, invoked as a program rather than linked as a library — see -/// `yara_difftest.rs` for why. +/// The oracle, invoked as a program rather than linked as a library (see +/// `yara_difftest.rs` for why). fn yr_bin() -> String { - std::env::var("EXAV_YR_BIN").unwrap_or_else(|_| "yr".to_string()) + std::env::var("EXAV_DEBUG_YR_BIN").unwrap_or_else(|_| "yr".to_string()) } fn have_yr() -> bool { @@ -67,8 +67,8 @@ fn have_yr() -> bool { } /// Matching rule names from `yr scan`, or `None` when the oracle refused the -/// rule file. Refusal is expected on a real corpus — unknown external -/// variables and modules exav does not implement make it bail — and those +/// rule file. Refusal is expected on a real corpus (unknown external +/// variables and modules exav does not implement make it bail), and those /// files are skipped rather than counted as disagreements. fn yr_set(rules: &str, data: &[u8]) -> Option> { let dir = std::env::temp_dir().join(format!("exav-yr-cov-{}", std::process::id())); @@ -99,8 +99,8 @@ fn yr_set(rules: &str, data: &[u8]) -> Option> { #[test] fn spot_check_real_corpus() { - let Ok(dir) = std::env::var("EXAV_YARA_CORPUS") else { - eprintln!("EXAV_YARA_CORPUS not set; skipping corpus spot-check"); + let Ok(dir) = std::env::var("EXAV_DEBUG_YARA_CORPUS") else { + eprintln!("EXAV_DEBUG_YARA_CORPUS not set; skipping corpus spot-check"); return; }; if !have_yr() { diff --git a/crates/exav-core/tests/yara_difftest.rs b/crates/exav-core/tests/yara_difftest.rs index 6246f4d2..56df6ce9 100644 --- a/crates/exav-core/tests/yara_difftest.rs +++ b/crates/exav-core/tests/yara_difftest.rs @@ -2,14 +2,14 @@ //! engine and to yara-x, and assert the set of matching rule identifiers is //! identical. //! -//! yara-x is invoked as a PROGRAM — the `yr` binary — rather than linked as a +//! yara-x is invoked as a PROGRAM (the `yr` binary) rather than linked as a //! library. As a dependency it brings a ~200-crate WebAssembly subtree in for //! the benefit of two test files, and a crate in the graph can end up in a //! shipped artifact by accident in a way a binary on `PATH` cannot. This is the //! same relationship exav already has with `clamscan`: run the reference tool, //! compare its output. //! -//! Skipped when `yr` is not installed. Point `EXAV_YR_BIN` at a specific build +//! Skipped when `yr` is not installed. Point `EXAV_DEBUG_YR_BIN` at a specific build //! to override the lookup. //! //! cargo install yara-x-cli # provides `yr` @@ -120,11 +120,11 @@ fn exav_matches(rules: &exav_core::yara::Rules, data: &[u8]) -> BTreeSet /// Where the oracle lives. Overridable so a checkout can point at a specific /// build rather than whatever is on `PATH`. fn yr_bin() -> String { - std::env::var("EXAV_YR_BIN").unwrap_or_else(|_| "yr".to_string()) + std::env::var("EXAV_DEBUG_YR_BIN").unwrap_or_else(|_| "yr".to_string()) } /// Whether the oracle is available at all. Absent is not a failure: the same -/// rule the clamd differential follows — a harness that needs a tool nobody +/// rule the clamd differential follows: a harness that needs a tool nobody /// installed should skip, not fail, or it becomes noise everyone learns to /// ignore. fn have_yr() -> bool { @@ -138,7 +138,7 @@ fn have_yr() -> bool { /// /// Shelling out rather than linking `yara-x`: as a library it drags a /// ~200-crate WebAssembly subtree into the dependency graph for the benefit of -/// two test files. As a binary it is exactly what it should be — an external +/// two test files. As a binary it is exactly what it should be: an external /// oracle, the same relationship exav already has with `clamscan`, and one that /// cannot end up in a shipped artifact by accident. fn yr_matches(rules: &str, data: &[u8], defines: &[(&str, &str)]) -> BTreeSet { @@ -183,7 +183,7 @@ fn differential_matches_agree() { /// The `elf` module, cross-checked against the real engine. Linux-focused /// rulesets lean on this module heavily, so every field exav exposes has to -/// agree with yara-x — including the undefined-propagation behaviour on inputs +/// agree with yara-x, including the undefined-propagation behaviour on inputs /// that are not ELF at all. const ELF_RULES: &str = r#" import "elf" @@ -511,7 +511,7 @@ fn differential_external_variables() { } // exav pre-declares the standard externals in `Compiler::new`, but // declaring them explicitly keeps the comparison independent of that - // default — the oracle is given the same values on its command line. + // default. The oracle is given the same values on its command line. let mut ec = exav_core::yara::Compiler::new(); ec.define_external("filename", ""); ec.define_external("filepath", ""); diff --git a/crates/exav-grep/NOTICE b/crates/exav-grep/NOTICE index 97811f32..34612eb8 100644 --- a/crates/exav-grep/NOTICE +++ b/crates/exav-grep/NOTICE @@ -1,483 +1,7 @@ -exav — THIRD-PARTY NOTICES -========================== +exav-grep: third-party notices -exav itself is licensed under the MIT License (see the LICENSE file). +This crate is part of exav and contains no third-party source. The notices +for the components exav links are in the NOTICE file at the root of the exav +repository, which also ships in every release archive: -The exav binary statically links a number of third-party Rust crates. Those -crates retain their own licenses; the binary distribution must carry their -notices. This file enumerates the major components and reproduces the notice -texts that the BSD-3-Clause and Apache-2.0-WITH-LLVM-exception licenses require. - -This is a hand-maintained summary of the principal components, not a byte-exact -dump of every transitive crate's license. To regenerate a complete, per-crate -attribution list from the resolved dependency graph, install one of: - - cargo install cargo-about # then: cargo about generate about.hbs - cargo install cargo-bundle-licenses - -and run it against this workspace. `cargo deny check licenses` (run in CI) -confirms every dependency resolves to a permissive license on the allowlist in -deny.toml. - -Versions below reflect Cargo.lock at the time of writing (last verified -2026-09-09). - - -------------------------------------------------------------------------------- -1. yara-x — BSD-3-Clause (source derivative, and a development dependency) -------------------------------------------------------------------------------- - -exav's YARA engine is native, in `crates/exav-core/src/yara/`. Parts of it are -derived from yara-x's source: the module implementations (`pe`, `elf`, `dotnet`, -`math`, `hash`, `string`) and the compiler's base64 handling, including field -semantics and the RVA-to-offset and import-hash routines. Each such file carries -a per-file attribution header, and the license text is reproduced below and in -`crates/exav-core/LICENSE-YARA-X`. - -`yara-x-parser` is a regular OPTIONAL DEPENDENCY, enabled by the default-on -`yara` feature of `crates/exav-core`, so a default build does link it. No other -yara-x crate is a dependency of any kind. - -The differential test that compares exav's engine against yara-x -(`crates/exav-core/tests/yara_difftest.rs`) invokes yara-x as a PROGRAM — the -`yr` binary from `PATH` — rather than linking it, and skips when that binary is -absent. - -Components: yara-x-parser 1.18. -Upstream: https://github.com/VirusTotal/yara-x -Copyright (c) VirusTotal and contributors. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other BSD-3-Clause components linked by exav (same 3-clause terms above, with -their respective copyright holders): subtle, tlsh2. - - -------------------------------------------------------------------------------- -1b. libarchive RAR5 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR5 (unpack50) decompressor in -crates/exav-unpack/src/formats/rar5_unpack.rs is a Rust port of the -decompression core of libarchive's archive_read_support_format_rar5.c. As a -source derivative, BSD-2-Clause requires retaining the upstream copyright and -licence text reproduced below. - -Upstream: https://github.com/libarchive/libarchive -Copyright (c) 2018 Grzegorz Antoniak (http://antoniak.org) -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1c. nwaples/rardecode RAR3 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR3/RAR4 (unpack29) decompressor in -crates/exav-unpack/src/formats/rar3_unpack.rs is a Rust port of the -decompression core of the Go package nwaples/rardecode. As a source -derivative, BSD-2-Clause requires retaining the upstream copyright and licence -text reproduced below. - -Upstream: https://github.com/nwaples/rardecode -Copyright (c) 2015, Nicholas Waples -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1d. ppmd-rust PPMd7 (var.H) model — CC0-1.0 OR MIT-0 (source derivative) -------------------------------------------------------------------------------- - -The PPMd variant H decoder in crates/exav-unpack/src/formats/ppmd7/ is vendored -and adapted from the `ppmd-rust` crate (version 1.4.0), generalised over a range -decoder trait so it can be driven by either the 7-Zip or the RAR range coder. -`ppmd-rust` is dual-licensed CC0-1.0 OR MIT-0 (public-domain-equivalent) and is -itself a Rust port of Igor Pavlov's 7-Zip PPMd7, which implements -Dmitry Shkarin's public-domain PPMd var.H (2001). It does NOT derive from -UnRAR/RARLAB. - -Upstream: https://crates.io/crates/ppmd-rust -License: CC0-1.0 OR MIT-0 (https://spdx.org/licenses/CC0-1.0, - https://spdx.org/licenses/MIT-0). Both place the work in (or treat it - as) the public domain; no attribution is legally required, and this - entry is recorded for completeness. - -The RAR-specific framing wired into crates/exav-unpack/src/formats/rar3_unpack.rs -(the PPMd block header: order/sub-allocator-memory/escape bits; the RAR -Subbotin carry-less range decoder in ppmd7/rangedec.rs; and the escape-symbol -handling for the PPMd↔LZSS conversion) is reproduced from the public format -description in libarchive's BSD-2-Clause archive_read_support_format_rar.c / -archive_ppmd7.c (already attributed in section 1b above), NOT from UnRAR. - - -------------------------------------------------------------------------------- -1e. dmg-core UDIF reader — Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The UDIF/DMG block reader in crates/exav-unpack/src/formats/udif.rs is vendored -and adapted from the `dmg-core` crate (version 0.1.2), changed to use -`lzma-rust2` instead of `lzma-rs` for XZ decompression. `dmg-core` is licensed -Apache-2.0; the Apache License requires retaining the attribution notices and -reproducing any upstream NOTICE text. - -Upstream: https://crates.io/crates/dmg-core -Copyright (c) Albert Hui / SecurityRonin. -Licensed under the Apache License, Version 2.0. You may obtain a copy at -http://www.apache.org/licenses/LICENSE-2.0 (see also the Apache-2.0 terms -referenced in section 2 below). Upstream ships no separate NOTICE file. - - -------------------------------------------------------------------------------- -1f. rust-cab CAB reader — MIT (source derivative) -------------------------------------------------------------------------------- - -The CAB (Microsoft Cabinet) reader in crates/exav-unpack/src/formats/cab_parse/ -is vendored and adapted from the `rust-cab` crate — write support, the datetime -module, unused public types, and tests removed; bounds checks added for -folder data-block and short-read cases. Licensed MIT; the MIT License requires -retaining the copyright and permission notice (reproduced in the exav LICENSE -file) in distributions. - -Upstream: https://github.com/mdsteele/rust-cab -Copyright (c) Matthew D. Steele . - - -------------------------------------------------------------------------------- -1g. unarc-rs ARJ reader — MIT OR Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The ARJ reader in crates/exav-unpack/src/formats/arj_parse/ is vendored and -adapted from the `unarc-rs` crate — multi-volume support, volume providers, and -external error types removed; wired to `delharc` for LH6 decompression. -Dual-licensed MIT OR Apache-2.0; the MIT or Apache terms apply at the -distributor's choice (both reproduced/referenced elsewhere in this file). - -The ZOO structures in crates/exav-unpack/src/formats/zoo_parse.rs are vendored -and adapted from the same crate, for the same reason the ARJ reader was: linking -it brings in a second `zip` with default features, and Cargo's feature -unification then enables the `zip` crate's own LZMA/bzip2/deflate64 decoders -inside exav, bypassing exav's per-member budget. Adapted to exav's error -handling, with the variable-length part of a directory entry (long filename, -directory name) actually read. - -The ZOO test archives in crates/exav-unpack/tests/fixtures/zoo/ (`store.zoo`, -`default.zoo`, `high_per.zoo`) are that project's own test corpus, redistributed -under the same terms. They hold one identical member under the three -compression methods, which is what lets the codecs be checked against stored -bytes rather than against themselves. - -Upstream: https://github.com/mkrueger/unarc-rs -Copyright (c) Mike Krüger. - - -------------------------------------------------------------------------------- -1h1. unshield / explode — MIT (linked dependency + test fixtures) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/ishield_z.rs reads InstallShield `.z` archives -through the `unshield` crate, whose only real dependency is `explode` — a leaf -decoder for the PKWARE Data Compression Library "implode" codec those archives -use. Both MIT. - -The fixtures in crates/exav-unpack/tests/fixtures/ishieldz/ are that project's -own examples, redistributed under the same terms. `undhr.z` is shipped alongside -`undhr.md`, the uncompressed original, which is what lets the decoder be checked -against the real file rather than against itself. - -This is a DIFFERENT format from the InstallShield `ISc(` InstallScript cabinet, -whose only implementation is the LGPL `unshield` C project — never read, and the -reason that format stays recognised-but-unopened. - -Upstream: https://github.com/agrif/unshield , https://github.com/agrif/explode -Copyright (c) Aaron Griffith. - - -------------------------------------------------------------------------------- -1h2. iced-x86 — MIT (generated opcode tables) -------------------------------------------------------------------------------- - -crates/exav-x86/src/generated.rs holds the SIMD, system and VEX regions of the -x86 opcode maps. It is not hand-written and not transcribed from a manual: it is -emitted by crates/exav-x86/tools/gen-tables, which decodes a probe for every -(prefix, opcode, /digit, mod) combination with `iced-x86` and records what came -back. The tables are therefore derived from `iced-x86`'s data, and the MIT -License requires retaining its copyright and permission notice (the MIT text is -in the exav LICENSE file). - -`iced-x86` is also the oracle that exav-x86's differential tests and the -`x86_decode` fuzz target compare against — including the decoding that -exav-pe-emu drives. It is a dev-dependency only: the decoder exav-pe-emu uses at -run time is exav-x86, and no iced code reaches a shipped artifact. - -Upstream: https://github.com/icedland/iced -Copyright (C) 2018-present iced project and contributors. - - -------------------------------------------------------------------------------- -1h. oletools test sample — BSD-2-Clause (test fixture) -------------------------------------------------------------------------------- - -The Excel 4.0 macro test fixture -crates/exav-unpack/tests/fixtures/xlm_excel4_sample.xls is a benign sample -redistributed from the `oletools` project's test corpus, used to verify the XLM -macro-sheet surfacing in formats/xlm.rs. Licensed BSD-2-Clause. - -Upstream: https://github.com/decalage2/oletools -Copyright (c) Philippe Lagadec. - - -------------------------------------------------------------------------------- -1i. aPLib reference (depack) — public domain (freeware) -------------------------------------------------------------------------------- - -The aPLib depacker in crates/exav-unpack/src/formats/aplib.rs is a clean-room -implementation of the published aPLib on-disk format. The bit/gamma reader -follows the algorithm published in the freely-distributable aPLib package's -reference `depack.c` by Jørgen Ibsen, which the author released for unrestricted -use ("free to use"). No attribution is legally required; this entry is recorded -for completeness. It does not derive from any GPL source. - -Upstream: https://ibsensoftware.com/products_aPLib.html - - -------------------------------------------------------------------------------- -1j. Microsoft Script Encoder decode tables — public domain -------------------------------------------------------------------------------- - -The substitution/decode tables in crates/exav-unpack/src/formats/screnc.rs are -the published Microsoft Script Encoder (`#@~^`) constants, as documented in -Didier Stevens' public-domain `decode-vbe.py`. Recorded here for completeness; -no attribution is legally required and it does not derive from any GPL source. - -Upstream: https://blog.didierstevens.com/programs/ - - -------------------------------------------------------------------------------- -1k. AutoIt-Ripper — MIT (reference implementation) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/autoit.rs implements the compiled-AutoIt3 script -layout — the EA05 record structure, the Mersenne-Twister keystream and its seeds -— from the MIT-licensed AutoIt-Ripper reference together with the public format -description. - -Upstream: https://github.com/nazywam/AutoIt-Ripper -Copyright (c) nazywam and contributors. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - - -------------------------------------------------------------------------------- -1l. NSIS format headers — zlib -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/nsis.rs implements the installer layout — the -firstheader and block-table structures — from NSIS's own zlib-licensed format -header `Source/exehead/fileform.h`, together with public reverse-engineering. - -Upstream: https://github.com/kichik/nsis -Copyright (c) Nullsoft and contributors. - - This software is provided 'as-is', without any express or implied warranty. - In no event will the authors be held liable for any damages arising from the - use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it freely, - subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a - product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - -------------------------------------------------------------------------------- -1m. sevenz-rust2 — Apache-2.0 (design reference) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/sevenz/ takes its reader structure from -sevenz-rust2, re-implemented natively so the crate can keep -`#![forbid(unsafe_code)]` and use exav's own PPMd7 model. - -Upstream: https://github.com/hasenbanck/sevenz-rust2 -Copyright (c) the sevenz-rust2 authors. - -Licensed under the Apache License, Version 2.0. A copy is available at -http://www.apache.org/licenses/LICENSE-2.0. The license requires retaining the -copyright, patent, trademark and attribution notices from the source, and -reproducing any NOTICE text the upstream distributes. - - -------------------------------------------------------------------------------- -2. ISC-licensed components -------------------------------------------------------------------------------- - -ring 0.17.14 (Apache-2.0 AND ISC), rustls 0.23.41, rustls-webpki, untrusted. -These carry ISC permission-notice terms; reproduce the copyright notice and the -permission/disclaimer notice in distributions. - - -------------------------------------------------------------------------------- -3. MIT / Apache-2.0 dual-licensed components (the bulk of the graph) -------------------------------------------------------------------------------- - -The majority of exav's dependencies are dual-licensed "MIT OR Apache-2.0" (or a -superset including Unlicense / 0BSD / Zlib), so the MIT or Apache terms apply at -the distributor's choice. Notable examples: - - aho-corasick, memchr, daachorse pattern matching / Aho-Corasick - serde, rmp-serde serialization - clap CLI parsing - sha1, sha2, md-5, digest (RustCrypto) hashing - flate2, miniz_oxide, bzip2, zip decompression (sevenz-rust2 is dev-only: - it builds 7z test fixtures; the shipped - 7z reader is native, §1m) - lzfse_rust, delharc, lzxd, DMG/LHA/CAB codecs - bitstream-io, stringprep - cfb OLE/CFB compound-file reader - fatfs, ext4-view FAT and ext2/3/4 filesystem readers - walkdir directory traversal - regex path-filter regexes, and the search - engine behind exav-grep - libc syscall bindings (daemon) - ureq, rustls, webpki-roots HTTPS for the signature updater - wasm-bindgen, js-sys, web-sys, WebAssembly/JS bindings — these reach - wasm-bindgen-futures the published npm artifact - -MIT-only components (the MIT copyright and permission notice must be retained; -the MIT text is in the exav LICENSE file): cfb, hfsplus, apfs (the two latter -via DMG/HFS+/APFS support). - -RustCrypto crates are MIT OR Apache-2.0. Zlib-licensed components present: -miniz_oxide, tinyvec. Unicode-3.0: the ICU / unicode-ident -crates. Other permissive licenses in the graph: CC0-1.0 and MIT-0 (ppmd-rust via -sevenz-rust2), CDLA-Permissive-2.0 (webpki-roots, TLS root store), bzip2-1.0.6 -(libbz2-rs-sys). - -Format documentation consulted ------------------------------ - -Some decoders were written from format descriptions rather than from any code. -Recorded here because a decoder's provenance is part of what makes it auditable: - - EGG ESTsoft Corp., "EGG Format Specification, Version 1.0" — the - vendor's own published document. No ESTsoft code was used. - - AZO ESTsoft's own compression algorithm, absent from the EGG spec. - `crates/exav-unpack/src/formats/azo.rs` and `azo_tables.rs` are a - Rust port of the AZO decoder in `EggDotNet` - (https://github.com/akolman/EggDotNet), MIT, Copyright (c) Andrew - Kolman. ESTsoft's own UnEgg library was NOT used: its licence - forbids commercial use without approval and forbids using it to - develop a compression algorithm. - - HWP3 Preamble offsets taken from `java-hwp` - (https://github.com/ddoleye/java-hwp), Apache-2.0. - Copyright the java-hwp authors. Validated against - `testHWP_3.0.hwp` from the Apache Tika test corpus - (https://github.com/apache/tika), Apache-2.0, Copyright the - Apache Software Foundation — vendored as - `crates/exav-unpack/tests/fixtures/hwp3/tika_testHWP_3.0.hwp`. - - ALZ Structure cross-checked against `unalz` (zlib license) and - `unar`, used as external oracles rather than as sources. - -Full MIT and Apache-2.0 license texts are widely available and are reproduced in -each crate's source distribution; the exav LICENSE file contains the MIT text. - - -------------------------------------------------------------------------------- -Build-only tooling -------------------------------------------------------------------------------- - -Everything above concerns code that reaches a shipped artifact. The -documentation site under `www/` builds with an npm toolchain whose lockfile -includes packages under stronger terms, LGPL-3.0-or-later among them. None of it -is linked into, bundled with, or distributed alongside any exav binary, crate or -WASM module: it runs at build time to produce static HTML and is listed here so -that a licence scan of the repository has an answer rather than a finding. +https://github.com/sylvinus/exav/blob/main/NOTICE diff --git a/crates/exav-pe-emu/NOTICE b/crates/exav-pe-emu/NOTICE index 97811f32..d61ed5d6 100644 --- a/crates/exav-pe-emu/NOTICE +++ b/crates/exav-pe-emu/NOTICE @@ -1,483 +1,7 @@ -exav — THIRD-PARTY NOTICES -========================== +exav-pe-emu: third-party notices -exav itself is licensed under the MIT License (see the LICENSE file). +This crate is part of exav and contains no third-party source. The notices +for the components exav links are in the NOTICE file at the root of the exav +repository, which also ships in every release archive: -The exav binary statically links a number of third-party Rust crates. Those -crates retain their own licenses; the binary distribution must carry their -notices. This file enumerates the major components and reproduces the notice -texts that the BSD-3-Clause and Apache-2.0-WITH-LLVM-exception licenses require. - -This is a hand-maintained summary of the principal components, not a byte-exact -dump of every transitive crate's license. To regenerate a complete, per-crate -attribution list from the resolved dependency graph, install one of: - - cargo install cargo-about # then: cargo about generate about.hbs - cargo install cargo-bundle-licenses - -and run it against this workspace. `cargo deny check licenses` (run in CI) -confirms every dependency resolves to a permissive license on the allowlist in -deny.toml. - -Versions below reflect Cargo.lock at the time of writing (last verified -2026-09-09). - - -------------------------------------------------------------------------------- -1. yara-x — BSD-3-Clause (source derivative, and a development dependency) -------------------------------------------------------------------------------- - -exav's YARA engine is native, in `crates/exav-core/src/yara/`. Parts of it are -derived from yara-x's source: the module implementations (`pe`, `elf`, `dotnet`, -`math`, `hash`, `string`) and the compiler's base64 handling, including field -semantics and the RVA-to-offset and import-hash routines. Each such file carries -a per-file attribution header, and the license text is reproduced below and in -`crates/exav-core/LICENSE-YARA-X`. - -`yara-x-parser` is a regular OPTIONAL DEPENDENCY, enabled by the default-on -`yara` feature of `crates/exav-core`, so a default build does link it. No other -yara-x crate is a dependency of any kind. - -The differential test that compares exav's engine against yara-x -(`crates/exav-core/tests/yara_difftest.rs`) invokes yara-x as a PROGRAM — the -`yr` binary from `PATH` — rather than linking it, and skips when that binary is -absent. - -Components: yara-x-parser 1.18. -Upstream: https://github.com/VirusTotal/yara-x -Copyright (c) VirusTotal and contributors. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other BSD-3-Clause components linked by exav (same 3-clause terms above, with -their respective copyright holders): subtle, tlsh2. - - -------------------------------------------------------------------------------- -1b. libarchive RAR5 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR5 (unpack50) decompressor in -crates/exav-unpack/src/formats/rar5_unpack.rs is a Rust port of the -decompression core of libarchive's archive_read_support_format_rar5.c. As a -source derivative, BSD-2-Clause requires retaining the upstream copyright and -licence text reproduced below. - -Upstream: https://github.com/libarchive/libarchive -Copyright (c) 2018 Grzegorz Antoniak (http://antoniak.org) -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1c. nwaples/rardecode RAR3 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR3/RAR4 (unpack29) decompressor in -crates/exav-unpack/src/formats/rar3_unpack.rs is a Rust port of the -decompression core of the Go package nwaples/rardecode. As a source -derivative, BSD-2-Clause requires retaining the upstream copyright and licence -text reproduced below. - -Upstream: https://github.com/nwaples/rardecode -Copyright (c) 2015, Nicholas Waples -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1d. ppmd-rust PPMd7 (var.H) model — CC0-1.0 OR MIT-0 (source derivative) -------------------------------------------------------------------------------- - -The PPMd variant H decoder in crates/exav-unpack/src/formats/ppmd7/ is vendored -and adapted from the `ppmd-rust` crate (version 1.4.0), generalised over a range -decoder trait so it can be driven by either the 7-Zip or the RAR range coder. -`ppmd-rust` is dual-licensed CC0-1.0 OR MIT-0 (public-domain-equivalent) and is -itself a Rust port of Igor Pavlov's 7-Zip PPMd7, which implements -Dmitry Shkarin's public-domain PPMd var.H (2001). It does NOT derive from -UnRAR/RARLAB. - -Upstream: https://crates.io/crates/ppmd-rust -License: CC0-1.0 OR MIT-0 (https://spdx.org/licenses/CC0-1.0, - https://spdx.org/licenses/MIT-0). Both place the work in (or treat it - as) the public domain; no attribution is legally required, and this - entry is recorded for completeness. - -The RAR-specific framing wired into crates/exav-unpack/src/formats/rar3_unpack.rs -(the PPMd block header: order/sub-allocator-memory/escape bits; the RAR -Subbotin carry-less range decoder in ppmd7/rangedec.rs; and the escape-symbol -handling for the PPMd↔LZSS conversion) is reproduced from the public format -description in libarchive's BSD-2-Clause archive_read_support_format_rar.c / -archive_ppmd7.c (already attributed in section 1b above), NOT from UnRAR. - - -------------------------------------------------------------------------------- -1e. dmg-core UDIF reader — Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The UDIF/DMG block reader in crates/exav-unpack/src/formats/udif.rs is vendored -and adapted from the `dmg-core` crate (version 0.1.2), changed to use -`lzma-rust2` instead of `lzma-rs` for XZ decompression. `dmg-core` is licensed -Apache-2.0; the Apache License requires retaining the attribution notices and -reproducing any upstream NOTICE text. - -Upstream: https://crates.io/crates/dmg-core -Copyright (c) Albert Hui / SecurityRonin. -Licensed under the Apache License, Version 2.0. You may obtain a copy at -http://www.apache.org/licenses/LICENSE-2.0 (see also the Apache-2.0 terms -referenced in section 2 below). Upstream ships no separate NOTICE file. - - -------------------------------------------------------------------------------- -1f. rust-cab CAB reader — MIT (source derivative) -------------------------------------------------------------------------------- - -The CAB (Microsoft Cabinet) reader in crates/exav-unpack/src/formats/cab_parse/ -is vendored and adapted from the `rust-cab` crate — write support, the datetime -module, unused public types, and tests removed; bounds checks added for -folder data-block and short-read cases. Licensed MIT; the MIT License requires -retaining the copyright and permission notice (reproduced in the exav LICENSE -file) in distributions. - -Upstream: https://github.com/mdsteele/rust-cab -Copyright (c) Matthew D. Steele . - - -------------------------------------------------------------------------------- -1g. unarc-rs ARJ reader — MIT OR Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The ARJ reader in crates/exav-unpack/src/formats/arj_parse/ is vendored and -adapted from the `unarc-rs` crate — multi-volume support, volume providers, and -external error types removed; wired to `delharc` for LH6 decompression. -Dual-licensed MIT OR Apache-2.0; the MIT or Apache terms apply at the -distributor's choice (both reproduced/referenced elsewhere in this file). - -The ZOO structures in crates/exav-unpack/src/formats/zoo_parse.rs are vendored -and adapted from the same crate, for the same reason the ARJ reader was: linking -it brings in a second `zip` with default features, and Cargo's feature -unification then enables the `zip` crate's own LZMA/bzip2/deflate64 decoders -inside exav, bypassing exav's per-member budget. Adapted to exav's error -handling, with the variable-length part of a directory entry (long filename, -directory name) actually read. - -The ZOO test archives in crates/exav-unpack/tests/fixtures/zoo/ (`store.zoo`, -`default.zoo`, `high_per.zoo`) are that project's own test corpus, redistributed -under the same terms. They hold one identical member under the three -compression methods, which is what lets the codecs be checked against stored -bytes rather than against themselves. - -Upstream: https://github.com/mkrueger/unarc-rs -Copyright (c) Mike Krüger. - - -------------------------------------------------------------------------------- -1h1. unshield / explode — MIT (linked dependency + test fixtures) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/ishield_z.rs reads InstallShield `.z` archives -through the `unshield` crate, whose only real dependency is `explode` — a leaf -decoder for the PKWARE Data Compression Library "implode" codec those archives -use. Both MIT. - -The fixtures in crates/exav-unpack/tests/fixtures/ishieldz/ are that project's -own examples, redistributed under the same terms. `undhr.z` is shipped alongside -`undhr.md`, the uncompressed original, which is what lets the decoder be checked -against the real file rather than against itself. - -This is a DIFFERENT format from the InstallShield `ISc(` InstallScript cabinet, -whose only implementation is the LGPL `unshield` C project — never read, and the -reason that format stays recognised-but-unopened. - -Upstream: https://github.com/agrif/unshield , https://github.com/agrif/explode -Copyright (c) Aaron Griffith. - - -------------------------------------------------------------------------------- -1h2. iced-x86 — MIT (generated opcode tables) -------------------------------------------------------------------------------- - -crates/exav-x86/src/generated.rs holds the SIMD, system and VEX regions of the -x86 opcode maps. It is not hand-written and not transcribed from a manual: it is -emitted by crates/exav-x86/tools/gen-tables, which decodes a probe for every -(prefix, opcode, /digit, mod) combination with `iced-x86` and records what came -back. The tables are therefore derived from `iced-x86`'s data, and the MIT -License requires retaining its copyright and permission notice (the MIT text is -in the exav LICENSE file). - -`iced-x86` is also the oracle that exav-x86's differential tests and the -`x86_decode` fuzz target compare against — including the decoding that -exav-pe-emu drives. It is a dev-dependency only: the decoder exav-pe-emu uses at -run time is exav-x86, and no iced code reaches a shipped artifact. - -Upstream: https://github.com/icedland/iced -Copyright (C) 2018-present iced project and contributors. - - -------------------------------------------------------------------------------- -1h. oletools test sample — BSD-2-Clause (test fixture) -------------------------------------------------------------------------------- - -The Excel 4.0 macro test fixture -crates/exav-unpack/tests/fixtures/xlm_excel4_sample.xls is a benign sample -redistributed from the `oletools` project's test corpus, used to verify the XLM -macro-sheet surfacing in formats/xlm.rs. Licensed BSD-2-Clause. - -Upstream: https://github.com/decalage2/oletools -Copyright (c) Philippe Lagadec. - - -------------------------------------------------------------------------------- -1i. aPLib reference (depack) — public domain (freeware) -------------------------------------------------------------------------------- - -The aPLib depacker in crates/exav-unpack/src/formats/aplib.rs is a clean-room -implementation of the published aPLib on-disk format. The bit/gamma reader -follows the algorithm published in the freely-distributable aPLib package's -reference `depack.c` by Jørgen Ibsen, which the author released for unrestricted -use ("free to use"). No attribution is legally required; this entry is recorded -for completeness. It does not derive from any GPL source. - -Upstream: https://ibsensoftware.com/products_aPLib.html - - -------------------------------------------------------------------------------- -1j. Microsoft Script Encoder decode tables — public domain -------------------------------------------------------------------------------- - -The substitution/decode tables in crates/exav-unpack/src/formats/screnc.rs are -the published Microsoft Script Encoder (`#@~^`) constants, as documented in -Didier Stevens' public-domain `decode-vbe.py`. Recorded here for completeness; -no attribution is legally required and it does not derive from any GPL source. - -Upstream: https://blog.didierstevens.com/programs/ - - -------------------------------------------------------------------------------- -1k. AutoIt-Ripper — MIT (reference implementation) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/autoit.rs implements the compiled-AutoIt3 script -layout — the EA05 record structure, the Mersenne-Twister keystream and its seeds -— from the MIT-licensed AutoIt-Ripper reference together with the public format -description. - -Upstream: https://github.com/nazywam/AutoIt-Ripper -Copyright (c) nazywam and contributors. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - - -------------------------------------------------------------------------------- -1l. NSIS format headers — zlib -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/nsis.rs implements the installer layout — the -firstheader and block-table structures — from NSIS's own zlib-licensed format -header `Source/exehead/fileform.h`, together with public reverse-engineering. - -Upstream: https://github.com/kichik/nsis -Copyright (c) Nullsoft and contributors. - - This software is provided 'as-is', without any express or implied warranty. - In no event will the authors be held liable for any damages arising from the - use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it freely, - subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a - product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - -------------------------------------------------------------------------------- -1m. sevenz-rust2 — Apache-2.0 (design reference) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/sevenz/ takes its reader structure from -sevenz-rust2, re-implemented natively so the crate can keep -`#![forbid(unsafe_code)]` and use exav's own PPMd7 model. - -Upstream: https://github.com/hasenbanck/sevenz-rust2 -Copyright (c) the sevenz-rust2 authors. - -Licensed under the Apache License, Version 2.0. A copy is available at -http://www.apache.org/licenses/LICENSE-2.0. The license requires retaining the -copyright, patent, trademark and attribution notices from the source, and -reproducing any NOTICE text the upstream distributes. - - -------------------------------------------------------------------------------- -2. ISC-licensed components -------------------------------------------------------------------------------- - -ring 0.17.14 (Apache-2.0 AND ISC), rustls 0.23.41, rustls-webpki, untrusted. -These carry ISC permission-notice terms; reproduce the copyright notice and the -permission/disclaimer notice in distributions. - - -------------------------------------------------------------------------------- -3. MIT / Apache-2.0 dual-licensed components (the bulk of the graph) -------------------------------------------------------------------------------- - -The majority of exav's dependencies are dual-licensed "MIT OR Apache-2.0" (or a -superset including Unlicense / 0BSD / Zlib), so the MIT or Apache terms apply at -the distributor's choice. Notable examples: - - aho-corasick, memchr, daachorse pattern matching / Aho-Corasick - serde, rmp-serde serialization - clap CLI parsing - sha1, sha2, md-5, digest (RustCrypto) hashing - flate2, miniz_oxide, bzip2, zip decompression (sevenz-rust2 is dev-only: - it builds 7z test fixtures; the shipped - 7z reader is native, §1m) - lzfse_rust, delharc, lzxd, DMG/LHA/CAB codecs - bitstream-io, stringprep - cfb OLE/CFB compound-file reader - fatfs, ext4-view FAT and ext2/3/4 filesystem readers - walkdir directory traversal - regex path-filter regexes, and the search - engine behind exav-grep - libc syscall bindings (daemon) - ureq, rustls, webpki-roots HTTPS for the signature updater - wasm-bindgen, js-sys, web-sys, WebAssembly/JS bindings — these reach - wasm-bindgen-futures the published npm artifact - -MIT-only components (the MIT copyright and permission notice must be retained; -the MIT text is in the exav LICENSE file): cfb, hfsplus, apfs (the two latter -via DMG/HFS+/APFS support). - -RustCrypto crates are MIT OR Apache-2.0. Zlib-licensed components present: -miniz_oxide, tinyvec. Unicode-3.0: the ICU / unicode-ident -crates. Other permissive licenses in the graph: CC0-1.0 and MIT-0 (ppmd-rust via -sevenz-rust2), CDLA-Permissive-2.0 (webpki-roots, TLS root store), bzip2-1.0.6 -(libbz2-rs-sys). - -Format documentation consulted ------------------------------ - -Some decoders were written from format descriptions rather than from any code. -Recorded here because a decoder's provenance is part of what makes it auditable: - - EGG ESTsoft Corp., "EGG Format Specification, Version 1.0" — the - vendor's own published document. No ESTsoft code was used. - - AZO ESTsoft's own compression algorithm, absent from the EGG spec. - `crates/exav-unpack/src/formats/azo.rs` and `azo_tables.rs` are a - Rust port of the AZO decoder in `EggDotNet` - (https://github.com/akolman/EggDotNet), MIT, Copyright (c) Andrew - Kolman. ESTsoft's own UnEgg library was NOT used: its licence - forbids commercial use without approval and forbids using it to - develop a compression algorithm. - - HWP3 Preamble offsets taken from `java-hwp` - (https://github.com/ddoleye/java-hwp), Apache-2.0. - Copyright the java-hwp authors. Validated against - `testHWP_3.0.hwp` from the Apache Tika test corpus - (https://github.com/apache/tika), Apache-2.0, Copyright the - Apache Software Foundation — vendored as - `crates/exav-unpack/tests/fixtures/hwp3/tika_testHWP_3.0.hwp`. - - ALZ Structure cross-checked against `unalz` (zlib license) and - `unar`, used as external oracles rather than as sources. - -Full MIT and Apache-2.0 license texts are widely available and are reproduced in -each crate's source distribution; the exav LICENSE file contains the MIT text. - - -------------------------------------------------------------------------------- -Build-only tooling -------------------------------------------------------------------------------- - -Everything above concerns code that reaches a shipped artifact. The -documentation site under `www/` builds with an npm toolchain whose lockfile -includes packages under stronger terms, LGPL-3.0-or-later among them. None of it -is linked into, bundled with, or distributed alongside any exav binary, crate or -WASM module: it runs at build time to produce static HTML and is listed here so -that a licence scan of the repository has an answer rather than a finding. +https://github.com/sylvinus/exav/blob/main/NOTICE diff --git a/crates/exav-unpack-wasm/Cargo.lock b/crates/exav-unpack-wasm/Cargo.lock index af08e26a..2d2ce59f 100644 --- a/crates/exav-unpack-wasm/Cargo.lock +++ b/crates/exav-unpack-wasm/Cargo.lock @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -56,15 +56,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -73,9 +64,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bitstream-io" @@ -127,9 +118,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", "shlex", @@ -148,9 +139,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chrono" @@ -229,9 +220,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] @@ -266,7 +257,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3658b90877f637514897c3dcbd628e0d290bcc7fee551a398b3994ebdc62d51e" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", "chrono", "memchr", ] @@ -313,7 +304,6 @@ dependencies = [ "aes", "apfs", "base64", - "bincode", "bitstream-io", "byteorder", "cbc", @@ -341,7 +331,6 @@ dependencies = [ "quick-xml", "ruzstd", "salzweg", - "serde", "sha1", "sha2", "stringprep", @@ -384,7 +373,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecf18f4817ea4baf95d700eb763642e65f77a94e630867028024b6725848a028" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", "crc", ] @@ -411,18 +400,19 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -433,21 +423,21 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-task", @@ -470,7 +460,7 @@ dependencies = [ "indexmap", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -527,9 +517,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown", @@ -547,9 +537,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -558,15 +548,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lzfse_rust" @@ -576,9 +566,9 @@ checksum = "5edeac3ec657925dececc672bdd58fa2212291886ddcb3e08aa46924c78340c6" [[package]] name = "lzma-rust2" -version = "0.16.4" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce716bf1a316f47a280fc76295f6495b5bea4752bca01c3b3885e101b1c23c02" +checksum = "ca93e534d1142d1d0dcca6d25fe302508a5dfb40b302802904577725ea0b695b" [[package]] name = "lznt1" @@ -597,9 +587,9 @@ checksum = "c17f346186eccb574ba5581acefc514f0c70a642db4f96e245034a0a158a7168" [[package]] name = "mail-parser" -version = "0.11.4" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c0e7e0704500930be5b6c629f30d23fd1dde4d1800e138e04b3fa302e64d51" +checksum = "4ec00bda90c6e645a54506c630c2820cd6b1890cfd2b0a169b50f74b2b8c7c86" dependencies = [ "hashify", ] @@ -616,15 +606,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -672,9 +662,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -690,9 +680,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -705,9 +695,9 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ruzstd" @@ -724,36 +714,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29a9ec85db1460d952e6556b945c1f3e287ddb44b79d6d72171daac24d4950d2" -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "sha1" version = "0.11.0" @@ -784,9 +744,9 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "slab" @@ -807,9 +767,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -828,44 +799,44 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] -name = "tinyvec_macros" -version = "0.1.1" +name = "tokio" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "pin-project-lite", +] [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" [[package]] name = "typed-path" @@ -887,9 +858,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-normalization" @@ -918,9 +889,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.23.4" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "js-sys", "wasm-bindgen", @@ -928,9 +899,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -941,19 +912,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -961,31 +933,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -1022,7 +994,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1033,7 +1005,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1062,9 +1034,9 @@ dependencies = [ [[package]] name = "xz4rust" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1053c1562c16e3760559c390d3b690ae6b524c200eb9a7ddb5d43c77d000832d" +checksum = "606930b71bad6379f366994e4e6252a3462fa28bfc3695c33b07bef8f967d6d6" [[package]] name = "zip" @@ -1079,3 +1051,9 @@ dependencies = [ "memchr", "typed-path", ] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" diff --git a/crates/exav-unpack-wasm/NOTICE b/crates/exav-unpack-wasm/NOTICE index 97811f32..26b7e3aa 100644 --- a/crates/exav-unpack-wasm/NOTICE +++ b/crates/exav-unpack-wasm/NOTICE @@ -1,483 +1,471 @@ -exav — THIRD-PARTY NOTICES -========================== +exav-unpack-wasm: third-party notices +===================================== -exav itself is licensed under the MIT License (see the LICENSE file). +exav-unpack-wasm and the @exav/unpack-wasm npm package are licensed under the +MIT License (see LICENSE). The package's WebAssembly module is a build of +exav-unpack, its "standard" preset, so it contains the third-party code in +exav-unpack's source and the crates it links. This file gives their notices. +The MIT License and the Apache License 2.0 texts the entries refer to are at +the end of this file; they apply to those components only. -The exav binary statically links a number of third-party Rust crates. Those -crates retain their own licenses; the binary distribution must carry their -notices. This file enumerates the major components and reproduces the notice -texts that the BSD-3-Clause and Apache-2.0-WITH-LLVM-exception licenses require. -This is a hand-maintained summary of the principal components, not a byte-exact -dump of every transitive crate's license. To regenerate a complete, per-crate -attribution list from the resolved dependency graph, install one of: +Part 1: code in exav-unpack's source +==================================== - cargo install cargo-about # then: cargo about generate about.hbs - cargo install cargo-bundle-licenses +libarchive RAR5 reader, BSD-2-Clause (source derivative) +-------------------------------------------------------- -and run it against this workspace. `cargo deny check licenses` (run in CI) -confirms every dependency resolves to a permissive license on the allowlist in -deny.toml. - -Versions below reflect Cargo.lock at the time of writing (last verified -2026-09-09). - - -------------------------------------------------------------------------------- -1. yara-x — BSD-3-Clause (source derivative, and a development dependency) -------------------------------------------------------------------------------- - -exav's YARA engine is native, in `crates/exav-core/src/yara/`. Parts of it are -derived from yara-x's source: the module implementations (`pe`, `elf`, `dotnet`, -`math`, `hash`, `string`) and the compiler's base64 handling, including field -semantics and the RVA-to-offset and import-hash routines. Each such file carries -a per-file attribution header, and the license text is reproduced below and in -`crates/exav-core/LICENSE-YARA-X`. - -`yara-x-parser` is a regular OPTIONAL DEPENDENCY, enabled by the default-on -`yara` feature of `crates/exav-core`, so a default build does link it. No other -yara-x crate is a dependency of any kind. - -The differential test that compares exav's engine against yara-x -(`crates/exav-core/tests/yara_difftest.rs`) invokes yara-x as a PROGRAM — the -`yr` binary from `PATH` — rather than linking it, and skips when that binary is -absent. - -Components: yara-x-parser 1.18. -Upstream: https://github.com/VirusTotal/yara-x -Copyright (c) VirusTotal and contributors. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other BSD-3-Clause components linked by exav (same 3-clause terms above, with -their respective copyright holders): subtle, tlsh2. - - -------------------------------------------------------------------------------- -1b. libarchive RAR5 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR5 (unpack50) decompressor in -crates/exav-unpack/src/formats/rar5_unpack.rs is a Rust port of the -decompression core of libarchive's archive_read_support_format_rar5.c. As a -source derivative, BSD-2-Clause requires retaining the upstream copyright and -licence text reproduced below. +exav-unpack's RAR5 decompressor is a Rust port of the decompression core of +libarchive's archive_read_support_format_rar5.c, and the RAR framing of its +PPMd decoder follows libarchive's archive_read_support_format_rar.c and +archive_ppmd7.c, under the same terms. Upstream: https://github.com/libarchive/libarchive + Copyright (c) 2018 Grzegorz Antoniak (http://antoniak.org) All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1c. nwaples/rardecode RAR3 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR3/RAR4 (unpack29) decompressor in -crates/exav-unpack/src/formats/rar3_unpack.rs is a Rust port of the -decompression core of the Go package nwaples/rardecode. As a source -derivative, BSD-2-Clause requires retaining the upstream copyright and licence -text reproduced below. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +nwaples/rardecode RAR3 reader, BSD-2-Clause (source derivative) +--------------------------------------------------------------- + +exav-unpack's RAR3 decompressor is a Rust port of the decompression core of +the Go package nwaples/rardecode. Upstream: https://github.com/nwaples/rardecode + Copyright (c) 2015, Nicholas Waples All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1d. ppmd-rust PPMd7 (var.H) model — CC0-1.0 OR MIT-0 (source derivative) -------------------------------------------------------------------------------- - -The PPMd variant H decoder in crates/exav-unpack/src/formats/ppmd7/ is vendored -and adapted from the `ppmd-rust` crate (version 1.4.0), generalised over a range -decoder trait so it can be driven by either the 7-Zip or the RAR range coder. -`ppmd-rust` is dual-licensed CC0-1.0 OR MIT-0 (public-domain-equivalent) and is -itself a Rust port of Igor Pavlov's 7-Zip PPMd7, which implements -Dmitry Shkarin's public-domain PPMd var.H (2001). It does NOT derive from -UnRAR/RARLAB. - -Upstream: https://crates.io/crates/ppmd-rust -License: CC0-1.0 OR MIT-0 (https://spdx.org/licenses/CC0-1.0, - https://spdx.org/licenses/MIT-0). Both place the work in (or treat it - as) the public domain; no attribution is legally required, and this - entry is recorded for completeness. - -The RAR-specific framing wired into crates/exav-unpack/src/formats/rar3_unpack.rs -(the PPMd block header: order/sub-allocator-memory/escape bits; the RAR -Subbotin carry-less range decoder in ppmd7/rangedec.rs; and the escape-symbol -handling for the PPMd↔LZSS conversion) is reproduced from the public format -description in libarchive's BSD-2-Clause archive_read_support_format_rar.c / -archive_ppmd7.c (already attributed in section 1b above), NOT from UnRAR. - - -------------------------------------------------------------------------------- -1e. dmg-core UDIF reader — Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The UDIF/DMG block reader in crates/exav-unpack/src/formats/udif.rs is vendored -and adapted from the `dmg-core` crate (version 0.1.2), changed to use -`lzma-rust2` instead of `lzma-rs` for XZ decompression. `dmg-core` is licensed -Apache-2.0; the Apache License requires retaining the attribution notices and -reproducing any upstream NOTICE text. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +dmg-core UDIF reader, Apache-2.0 (source derivative) +---------------------------------------------------- + +exav-unpack's UDIF (DMG) reader is vendored from dmg-core 0.1.2 and changed to +decode XZ with lzma-rust2 instead of lzma-rs. Licensed under the Apache +License, Version 2.0 (text below). Upstream ships no NOTICE file. Upstream: https://crates.io/crates/dmg-core Copyright (c) Albert Hui / SecurityRonin. -Licensed under the Apache License, Version 2.0. You may obtain a copy at -http://www.apache.org/licenses/LICENSE-2.0 (see also the Apache-2.0 terms -referenced in section 2 below). Upstream ships no separate NOTICE file. - - -------------------------------------------------------------------------------- -1f. rust-cab CAB reader — MIT (source derivative) -------------------------------------------------------------------------------- - -The CAB (Microsoft Cabinet) reader in crates/exav-unpack/src/formats/cab_parse/ -is vendored and adapted from the `rust-cab` crate — write support, the datetime -module, unused public types, and tests removed; bounds checks added for -folder data-block and short-read cases. Licensed MIT; the MIT License requires -retaining the copyright and permission notice (reproduced in the exav LICENSE -file) in distributions. - -Upstream: https://github.com/mdsteele/rust-cab -Copyright (c) Matthew D. Steele . - - -------------------------------------------------------------------------------- -1g. unarc-rs ARJ reader — MIT OR Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The ARJ reader in crates/exav-unpack/src/formats/arj_parse/ is vendored and -adapted from the `unarc-rs` crate — multi-volume support, volume providers, and -external error types removed; wired to `delharc` for LH6 decompression. -Dual-licensed MIT OR Apache-2.0; the MIT or Apache terms apply at the -distributor's choice (both reproduced/referenced elsewhere in this file). - -The ZOO structures in crates/exav-unpack/src/formats/zoo_parse.rs are vendored -and adapted from the same crate, for the same reason the ARJ reader was: linking -it brings in a second `zip` with default features, and Cargo's feature -unification then enables the `zip` crate's own LZMA/bzip2/deflate64 decoders -inside exav, bypassing exav's per-member budget. Adapted to exav's error -handling, with the variable-length part of a directory entry (long filename, -directory name) actually read. - -The ZOO test archives in crates/exav-unpack/tests/fixtures/zoo/ (`store.zoo`, -`default.zoo`, `high_per.zoo`) are that project's own test corpus, redistributed -under the same terms. They hold one identical member under the three -compression methods, which is what lets the codecs be checked against stored -bytes rather than against themselves. - -Upstream: https://github.com/mkrueger/unarc-rs -Copyright (c) Mike Krüger. - - -------------------------------------------------------------------------------- -1h1. unshield / explode — MIT (linked dependency + test fixtures) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/ishield_z.rs reads InstallShield `.z` archives -through the `unshield` crate, whose only real dependency is `explode` — a leaf -decoder for the PKWARE Data Compression Library "implode" codec those archives -use. Both MIT. - -The fixtures in crates/exav-unpack/tests/fixtures/ishieldz/ are that project's -own examples, redistributed under the same terms. `undhr.z` is shipped alongside -`undhr.md`, the uncompressed original, which is what lets the decoder be checked -against the real file rather than against itself. - -This is a DIFFERENT format from the InstallShield `ISc(` InstallScript cabinet, -whose only implementation is the LGPL `unshield` C project — never read, and the -reason that format stays recognised-but-unopened. - -Upstream: https://github.com/agrif/unshield , https://github.com/agrif/explode -Copyright (c) Aaron Griffith. - - -------------------------------------------------------------------------------- -1h2. iced-x86 — MIT (generated opcode tables) -------------------------------------------------------------------------------- - -crates/exav-x86/src/generated.rs holds the SIMD, system and VEX regions of the -x86 opcode maps. It is not hand-written and not transcribed from a manual: it is -emitted by crates/exav-x86/tools/gen-tables, which decodes a probe for every -(prefix, opcode, /digit, mod) combination with `iced-x86` and records what came -back. The tables are therefore derived from `iced-x86`'s data, and the MIT -License requires retaining its copyright and permission notice (the MIT text is -in the exav LICENSE file). - -`iced-x86` is also the oracle that exav-x86's differential tests and the -`x86_decode` fuzz target compare against — including the decoding that -exav-pe-emu drives. It is a dev-dependency only: the decoder exav-pe-emu uses at -run time is exav-x86, and no iced code reaches a shipped artifact. - -Upstream: https://github.com/icedland/iced -Copyright (C) 2018-present iced project and contributors. - - -------------------------------------------------------------------------------- -1h. oletools test sample — BSD-2-Clause (test fixture) -------------------------------------------------------------------------------- - -The Excel 4.0 macro test fixture -crates/exav-unpack/tests/fixtures/xlm_excel4_sample.xls is a benign sample -redistributed from the `oletools` project's test corpus, used to verify the XLM -macro-sheet surfacing in formats/xlm.rs. Licensed BSD-2-Clause. - -Upstream: https://github.com/decalage2/oletools -Copyright (c) Philippe Lagadec. - - -------------------------------------------------------------------------------- -1i. aPLib reference (depack) — public domain (freeware) -------------------------------------------------------------------------------- - -The aPLib depacker in crates/exav-unpack/src/formats/aplib.rs is a clean-room -implementation of the published aPLib on-disk format. The bit/gamma reader -follows the algorithm published in the freely-distributable aPLib package's -reference `depack.c` by Jørgen Ibsen, which the author released for unrestricted -use ("free to use"). No attribution is legally required; this entry is recorded -for completeness. It does not derive from any GPL source. - -Upstream: https://ibsensoftware.com/products_aPLib.html - - -------------------------------------------------------------------------------- -1j. Microsoft Script Encoder decode tables — public domain -------------------------------------------------------------------------------- - -The substitution/decode tables in crates/exav-unpack/src/formats/screnc.rs are -the published Microsoft Script Encoder (`#@~^`) constants, as documented in -Didier Stevens' public-domain `decode-vbe.py`. Recorded here for completeness; -no attribution is legally required and it does not derive from any GPL source. - -Upstream: https://blog.didierstevens.com/programs/ - - -------------------------------------------------------------------------------- -1k. AutoIt-Ripper — MIT (reference implementation) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/autoit.rs implements the compiled-AutoIt3 script -layout — the EA05 record structure, the Mersenne-Twister keystream and its seeds -— from the MIT-licensed AutoIt-Ripper reference together with the public format -description. -Upstream: https://github.com/nazywam/AutoIt-Ripper -Copyright (c) nazywam and contributors. - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: +MIT-licensed sources +-------------------- - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. +Each under the MIT License (MIT OR Apache-2.0 ones used under MIT): - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. +- rust-cab (CAB reader, vendored): Copyright (c) Matthew D. Steele + . https://github.com/mdsteele/rust-cab +- unarc-rs (ARJ and ZOO readers, vendored): Copyright (c) Mike Krüger. + https://github.com/mkrueger/unarc-rs +- bzip2-rs (bzip2 decoder, vendored): Copyright (c) 2021 Paolo Barbolini. + https://github.com/paolobarbolini/bzip2-rs +- EggDotNet (AZO decoder, ported): Copyright (c) Andrew Kolman. + https://github.com/akolman/EggDotNet +- AutoIt-Ripper (compiled AutoIt3 layout and tables): Copyright (c) nazywam + and contributors. https://github.com/nazywam/AutoIt-Ripper -------------------------------------------------------------------------------- -1l. NSIS format headers — zlib -------------------------------------------------------------------------------- +NSIS format headers, zlib +------------------------- -crates/exav-unpack/src/formats/nsis.rs implements the installer layout — the -firstheader and block-table structures — from NSIS's own zlib-licensed format -header `Source/exehead/fileform.h`, together with public reverse-engineering. +exav-unpack's NSIS reader implements the installer layout from NSIS's format +header Source/exehead/fileform.h. Upstream: https://github.com/kichik/nsis -Copyright (c) Nullsoft and contributors. - - This software is provided 'as-is', without any express or implied warranty. - In no event will the authors be held liable for any damages arising from the - use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it freely, - subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a - product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - -------------------------------------------------------------------------------- -1m. sevenz-rust2 — Apache-2.0 (design reference) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/sevenz/ takes its reader structure from -sevenz-rust2, re-implemented natively so the crate can keep -`#![forbid(unsafe_code)]` and use exav's own PPMd7 model. - -Upstream: https://github.com/hasenbanck/sevenz-rust2 -Copyright (c) the sevenz-rust2 authors. - -Licensed under the Apache License, Version 2.0. A copy is available at -http://www.apache.org/licenses/LICENSE-2.0. The license requires retaining the -copyright, patent, trademark and attribution notices from the source, and -reproducing any NOTICE text the upstream distributes. - - -------------------------------------------------------------------------------- -2. ISC-licensed components -------------------------------------------------------------------------------- - -ring 0.17.14 (Apache-2.0 AND ISC), rustls 0.23.41, rustls-webpki, untrusted. -These carry ISC permission-notice terms; reproduce the copyright notice and the -permission/disclaimer notice in distributions. - - -------------------------------------------------------------------------------- -3. MIT / Apache-2.0 dual-licensed components (the bulk of the graph) -------------------------------------------------------------------------------- - -The majority of exav's dependencies are dual-licensed "MIT OR Apache-2.0" (or a -superset including Unlicense / 0BSD / Zlib), so the MIT or Apache terms apply at -the distributor's choice. Notable examples: - - aho-corasick, memchr, daachorse pattern matching / Aho-Corasick - serde, rmp-serde serialization - clap CLI parsing - sha1, sha2, md-5, digest (RustCrypto) hashing - flate2, miniz_oxide, bzip2, zip decompression (sevenz-rust2 is dev-only: - it builds 7z test fixtures; the shipped - 7z reader is native, §1m) - lzfse_rust, delharc, lzxd, DMG/LHA/CAB codecs - bitstream-io, stringprep - cfb OLE/CFB compound-file reader - fatfs, ext4-view FAT and ext2/3/4 filesystem readers - walkdir directory traversal - regex path-filter regexes, and the search - engine behind exav-grep - libc syscall bindings (daemon) - ureq, rustls, webpki-roots HTTPS for the signature updater - wasm-bindgen, js-sys, web-sys, WebAssembly/JS bindings — these reach - wasm-bindgen-futures the published npm artifact - -MIT-only components (the MIT copyright and permission notice must be retained; -the MIT text is in the exav LICENSE file): cfb, hfsplus, apfs (the two latter -via DMG/HFS+/APFS support). - -RustCrypto crates are MIT OR Apache-2.0. Zlib-licensed components present: -miniz_oxide, tinyvec. Unicode-3.0: the ICU / unicode-ident -crates. Other permissive licenses in the graph: CC0-1.0 and MIT-0 (ppmd-rust via -sevenz-rust2), CDLA-Permissive-2.0 (webpki-roots, TLS root store), bzip2-1.0.6 -(libbz2-rs-sys). - -Format documentation consulted ------------------------------ - -Some decoders were written from format descriptions rather than from any code. -Recorded here because a decoder's provenance is part of what makes it auditable: - - EGG ESTsoft Corp., "EGG Format Specification, Version 1.0" — the - vendor's own published document. No ESTsoft code was used. - - AZO ESTsoft's own compression algorithm, absent from the EGG spec. - `crates/exav-unpack/src/formats/azo.rs` and `azo_tables.rs` are a - Rust port of the AZO decoder in `EggDotNet` - (https://github.com/akolman/EggDotNet), MIT, Copyright (c) Andrew - Kolman. ESTsoft's own UnEgg library was NOT used: its licence - forbids commercial use without approval and forbids using it to - develop a compression algorithm. - - HWP3 Preamble offsets taken from `java-hwp` - (https://github.com/ddoleye/java-hwp), Apache-2.0. - Copyright the java-hwp authors. Validated against - `testHWP_3.0.hwp` from the Apache Tika test corpus - (https://github.com/apache/tika), Apache-2.0, Copyright the - Apache Software Foundation — vendored as - `crates/exav-unpack/tests/fixtures/hwp3/tika_testHWP_3.0.hwp`. - - ALZ Structure cross-checked against `unalz` (zlib license) and - `unar`, used as external oracles rather than as sources. - -Full MIT and Apache-2.0 license texts are widely available and are reproduced in -each crate's source distribution; the exav LICENSE file contains the MIT text. - - -------------------------------------------------------------------------------- -Build-only tooling -------------------------------------------------------------------------------- - -Everything above concerns code that reaches a shipped artifact. The -documentation site under `www/` builds with an npm toolchain whose lockfile -includes packages under stronger terms, LGPL-3.0-or-later among them. None of it -is linked into, bundled with, or distributed alongside any exav binary, crate or -WASM module: it runs at build time to produce static HTML and is listed here so -that a licence scan of the repository has an answer rather than a finding. +Copyright (c) Nullsoft and contributors + +This software is provided 'as-is', without any express or implied warranty. +In no event will the authors be held liable for any damages arising from the +use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it freely, +subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim + that you wrote the original software. If you use this software in a + product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. + + +Apache-2.0 references +--------------------- + +- java-hwp (HWP 3.0 preamble offsets): Copyright the java-hwp authors. + https://github.com/ddoleye/java-hwp +- sevenz-rust2 (7z reader structure, written anew): Copyright the sevenz-rust2 + authors. https://github.com/hasenbanck/sevenz-rust2 + + +Public-domain sources, recorded for completeness +------------------------------------------------ + +- ppmd-rust 1.4.0 (PPMd7 model, vendored), CC0-1.0 OR MIT-0. +- aPLib's reference depack.c by Jørgen Ibsen (bit and gamma readers), released + free to use. +- Didier Stevens' decode-vbe.py (Microsoft Script Encoder tables), public + domain. + + +Part 2: crates linked into the module +===================================== + +Build-time crates (proc-macro2, quote, syn, unicode-ident, thiserror-impl, +wasm-bindgen-macro, wasm-bindgen-macro-support) generate code but do not +reach the module and are not listed. + +Under the MIT License, each offering it (alone, or among MIT OR Apache-2.0, +Unlicense OR MIT, 0BSD OR MIT OR Apache-2.0, Zlib OR Apache-2.0 OR MIT), with +the copyright its license file states, or its authors where it states none: + + adler2 Jonas Schievink + aes 2018 Artyom Pavlov; 2018-2024 The RustCrypto Project Developers + apfs 2026 Dil4rd + arraydeque 2018 Andy Lok + base64 2015 Alice Maz + bitflags 2014 The Rust Project Developers + bitstream-io 2017 Brian Langenberger + block-buffer 2018-2025 The RustCrypto Project Developers + block-padding 2018-2025 The RustCrypto Project Developers + bumpalo 2019 Nick Fitzgerald + byteorder 2015 Andrew Gallant + cbc 2018 Artyom Pavlov; 2018-2022 RustCrypto Developers + cfb 2017 Matthew D. Steele + cfg-if 2014 Alex Crichton + chrono 2014 Kang Seonghoon + cipher 2016-2025 RustCrypto Developers + cmov 2022-2026 The RustCrypto Project Developers + const-oid 2020-2026 The RustCrypto Project Developers + cpubits 2023-2026 The RustCrypto Project Developers + crc 2017 crc-rs Developers + crc-catalog 2019 Akhil Velagapudi + crc32fast 2018 Sam Rijs, Alex Crichton and contributors + crypto-common 2021-2026 RustCrypto Developers + ctutils 2025-2026 The RustCrypto Project Developers + deflate64 .NET Foundation and Contributors; 2023 anatawa12 + delharc 2023 Rafal Michalski + des 2017 Gulshan Singh, Antoni Boucher, Artyom Pavlov; + 2017-2024 The RustCrypto Project Developers + digest 2017 Artyom Pavlov; 2017-2025 RustCrypto Developers + equivalent 2016-2023 the equivalent authors + explode 2020 Aaron Griffith + ext4-view 2023 Google LLC + fatfs 2017 Rafał Harabień + filetime 2014 Alex Crichton + flate2 2014-2026 Alex Crichton + fnv 2017 Contributors + futures-core 2016 Alex Crichton; 2017 The Tokio Authors + futures-task 2016 Alex Crichton; 2017 The Tokio Authors + futures-util 2016 Alex Crichton; 2017 The Tokio Authors + hashbrown 2016 Amanieu d'Antras + hfsplus 2026 Dil4rd + hmac 2017 Artyom Pavlov + hybrid-array 2022-2026 The RustCrypto Project Developers + indexmap 2016-2017 the indexmap authors + inout 2022 Artyom Pavlov; 2022-2025 The RustCrypto Project Developers + js-sys 2014 Alex Crichton + log 2014 The Rust Project Developers + lzfse_rust 2020 Vin Singh Ghataore + lznt1 Cody Wyatt Neiman + lzxd 2020 Lonami + mail-parser Stalwart Labs + md-5 2006-2009 Graydon Hoare; 2009-2013 Mozilla Foundation; + 2016 Artyom Pavlov; 2016-2026 The RustCrypto Project Developers + memchr 2015 Andrew Gallant + miniz_oxide 2017 Frommi; 2017-2024 oyvindln; 2010-2014 Rich Geldreich + and Tenacious Software LLC; 2013-2014 RAD Game Tools and + Valve Software + no_std_io2 2020-2021 Brendan Molloy + num-traits 2014 The Rust Project Developers + once_cell Aleksey Kladov + pbkdf2 2017 Artyom Pavlov; 2018-2023 The RustCrypto Project Developers + pin-project-lite the pin-project-lite authors + quick-xml 2016 Johann Tuffe + ruzstd 2019 Moritz Borcherding + salzweg Redwarp + sha1 2006-2009 Graydon Hoare; 2009-2013 Mozilla Foundation; + 2016 Artyom Pavlov; 2016-2026 The RustCrypto Project Developers + sha2 2006-2009 Graydon Hoare; 2009-2013 Mozilla Foundation; + 2016 Artyom Pavlov; 2016-2026 The RustCrypto Project Developers + simd-adler32 2021 Marvin Countryman + slab 2019 Carl Lerche + stringprep 2017 The rust-stringprep Developers + tar The tar-rs Project Contributors + thiserror David Tolnay + tinyvec 2019 Daniel "Lokathor" Gee + twox-hash 2015 Jake Goulding + typed-path Chip Senkbeil + typenum 2014 Paho Lurie-Gregg + unicode-bidi 2015 The Rust Project Developers + unicode-normalization 2015 The Rust Project Developers + unicode-properties 2015 The Rust Project Developers + unshield 2020 Aaron Griffith + uuid 2014 The Rust Project Developers; 2018 Ashley Mannix, + Christopher Armstrong, Dylan DPC, Hunar Roop Kahlon + wasm-bindgen 2014 Alex Crichton + wasm-bindgen-futures 2014 Alex Crichton + wasm-bindgen-shared 2014 Alex Crichton + web-sys 2014 Alex Crichton + web-time 2023 dAxpeDDa + xz4rust 2025 Alexander Schütz; 2018 Sam Rijs, Alex Crichton and + contributors + zip 2014 Mathijs van de Nes + +Under the Apache License, Version 2.0 (text below), its only license; it ships +no NOTICE file: + + lzma-rust2 the lzma-rust2 authors + +Under CC0-1.0 (public domain; no notice required, recorded for completeness): + + constant_time_eq Cesar Eduardo Barros + + +MIT License +----------- + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Apache License, Version 2.0 +--------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/crates/exav-unpack-wasm/src/lib.rs b/crates/exav-unpack-wasm/src/lib.rs index a7527fe3..28110b2f 100644 --- a/crates/exav-unpack-wasm/src/lib.rs +++ b/crates/exav-unpack-wasm/src/lib.rs @@ -181,9 +181,8 @@ fn read_entry( let (data, partial) = content.into_bytes(meta, budget)?; entry.data = data; if partial && entry.unsupported.is_none() { - entry.unsupported = Some( - "member failed to decode part way; the bytes before the failure are kept", - ); + entry.unsupported = + Some("member failed to decode part way; the bytes before the failure are kept"); } } Ok(entry) @@ -382,21 +381,26 @@ impl Archive { let out = Array::new(); let mut index = 0; let mut failed = None; - let walked = walk(self.fmt, self.bytes.source(), &mut budget, &mut |meta, content, _| { - let size = match &content { - Some(Member::Bytes(data)) => Some(data.len() as u64), - _ => meta.size, - }; - match member_to_js(meta, index, size) { - Ok(m) => out.push(&m), - Err(e) => { - failed = Some(e); - return Some(()); - } - }; - index += 1; - None - }); + let walked = walk( + self.fmt, + self.bytes.source(), + &mut budget, + &mut |meta, content, _| { + let size = match &content { + Some(Member::Bytes(data)) => Some(data.len() as u64), + _ => meta.size, + }; + match member_to_js(meta, index, size) { + Ok(m) => out.push(&m), + Err(e) => { + failed = Some(e); + return Some(()); + } + }; + index += 1; + None + }, + ); if let Some(e) = failed { return Err(e); } @@ -416,13 +420,18 @@ impl Archive { ) -> Result { let mut budget = self.budget(passwords); let mut at = 0; - let found = walk(self.fmt, self.bytes.source(), &mut budget, &mut |meta, content, b| { - if at < index { - at += 1; - return None; - } - Some(read_entry(meta, content, b)) - }); + let found = walk( + self.fmt, + self.bytes.source(), + &mut budget, + &mut |meta, content, b| { + if at < index { + at += 1; + return None; + } + Some(read_entry(meta, content, b)) + }, + ); match found { Ok(Some(Ok(entry))) => { self.extracted += entry.data.len() as u64; @@ -445,23 +454,28 @@ impl Archive { let out = Array::new(); let mut taken = 0; let mut failed = None; - let walked = walk(self.fmt, self.bytes.source(), &mut budget, &mut |meta, content, b| { - let entry = match read_entry(meta, content, b) { - Ok(entry) => entry, - Err(hit) => return Some(Err(hit)), - }; - taken += entry.data.len() as u64; - match unpack_entry_to_js(&entry) { - Ok(e) => { - out.push(&e); - None + let walked = walk( + self.fmt, + self.bytes.source(), + &mut budget, + &mut |meta, content, b| { + let entry = match read_entry(meta, content, b) { + Ok(entry) => entry, + Err(hit) => return Some(Err(hit)), + }; + taken += entry.data.len() as u64; + match unpack_entry_to_js(&entry) { + Ok(e) => { + out.push(&e); + None + } + Err(e) => { + failed = Some(e); + Some(Ok(())) + } } - Err(e) => { - failed = Some(e); - Some(Ok(())) - } - } - }); + }, + ); self.extracted += taken; if let Some(e) = failed { return Err(e); diff --git a/crates/exav-unpack/Cargo.toml b/crates/exav-unpack/Cargo.toml index 8edcdc19..fc005f72 100644 --- a/crates/exav-unpack/Cargo.toml +++ b/crates/exav-unpack/Cargo.toml @@ -18,21 +18,24 @@ readme = "README.md" # would otherwise download it. Clone the repository to run the suites. exclude = ["tests/fixtures/**", "test_data/**"] -# The extractor CLI ships with the library rather than as a separate package. -# It is a thin shell over the public API and needs nothing the library does not -# already have (no argument parser, no error crate), so a library consumer pays -# nothing for it (Cargo builds only the lib target for a dependency), and -# `cargo install exav-unpack` gives the command, which is where people look for -# it. +# The extractor CLI ships with the library rather than as a separate package, +# so `cargo install exav-unpack` gives the command, which is where people look +# for it. Its one dependency of its own (`chrono`, for local times) is behind +# the `cli` feature, which a library consumer turning default features off +# never builds. [[bin]] name = "exav-unpack" path = "src/main.rs" +required-features = ["cli"] [dependencies] # Always-on (format-agnostic) deps. thiserror = "2" # fast substring search (locating the UPX PackHeader magic, etc.) memchr = "2" +# The CLI's local times: a DOS timestamp is a local time, and `-l` lists in +# local time, as unzip does. Already in the tree through `delharc`. +chrono = { version = "0.4", default-features = false, features = ["clock", "std"], optional = true } # The x86 emulator that runs a packer's stub (`exav-pe-emu`). Its own crate because # it is useful on its own and because it is the only thing that needs a CPU @@ -93,8 +96,6 @@ base64 = { version = "0.22", optional = true } quick-xml = { version = "0.41", optional = true } des = { version = "0.9", default-features = false, optional = true } cbc = { version = "0.2", optional = true } -bincode = { version = "1", optional = true } -serde = { version = "1.0.228", features = ["derive"], optional = true } flate2 = { version = "1.1.9", optional = true } # ZIP method 9 (Deflate64 / "enhanced deflate"). A leaf dep: zero runtime # dependencies, MIT, and `#![forbid(unsafe_code)]` like this crate. It is the @@ -113,7 +114,9 @@ ruzstd = { version = "0.8", optional = true } crc32fast = { version = "1.5.0", optional = true } [features] -default = ["all-formats", "decrypt"] +default = ["all-formats", "decrypt", "cli"] +# The `exav-unpack` command. +cli = ["dep:chrono"] # Makes a reserved byte prefix panic from inside the format dispatch, so the # `catch_unwind` boundary in `walk` can be tested for what it is: the # thing that turns an arbitrary decoder panic into a reported limit instead of a @@ -239,7 +242,7 @@ email = ["dep:mail-parser", "dep:base64"] # the `decrypt` feature, not here. dmg = [ "dep:hfsplus", "dep:apfs", "dep:lzfse_rust", "dep:xz4rust", "bzip2-decoder", - "dep:flate2", "dep:base64", "dep:quick-xml", "dep:bincode", "dep:serde", + "dep:flate2", "dep:base64", "dep:quick-xml", ] # Decryption of encrypted members (ZIP ZipCrypto/WinZip-AES, PDF standard @@ -291,7 +294,7 @@ required-features = ["pe-emu"] # independent, but they run in every other feature pass. [[test]] name = "cli" -required-features = ["zip"] +required-features = ["zip", "cli"] [dev-dependencies] # Used only in tests to compare decoder output against external-oracle digests. diff --git a/crates/exav-unpack/NOTICE b/crates/exav-unpack/NOTICE index 97811f32..5dbbcdb8 100644 --- a/crates/exav-unpack/NOTICE +++ b/crates/exav-unpack/NOTICE @@ -1,483 +1,407 @@ -exav — THIRD-PARTY NOTICES -========================== +exav-unpack: third-party notices +================================ -exav itself is licensed under the MIT License (see the LICENSE file). +exav-unpack is licensed under the MIT License (see LICENSE). Parts of its +source are derived from other projects, or written from their code or format +headers; this file gives their notices. The MIT License and the Apache License +2.0 texts the entries refer to are at the end of this file; they apply to those +components only. The notices for the components the exav binaries link are in +the NOTICE file at the root of the exav repository: +https://github.com/sylvinus/exav/blob/main/NOTICE -The exav binary statically links a number of third-party Rust crates. Those -crates retain their own licenses; the binary distribution must carry their -notices. This file enumerates the major components and reproduces the notice -texts that the BSD-3-Clause and Apache-2.0-WITH-LLVM-exception licenses require. -This is a hand-maintained summary of the principal components, not a byte-exact -dump of every transitive crate's license. To regenerate a complete, per-crate -attribution list from the resolved dependency graph, install one of: +libarchive RAR5 reader, BSD-2-Clause (source derivative) +-------------------------------------------------------- - cargo install cargo-about # then: cargo about generate about.hbs - cargo install cargo-bundle-licenses - -and run it against this workspace. `cargo deny check licenses` (run in CI) -confirms every dependency resolves to a permissive license on the allowlist in -deny.toml. - -Versions below reflect Cargo.lock at the time of writing (last verified -2026-09-09). - - -------------------------------------------------------------------------------- -1. yara-x — BSD-3-Clause (source derivative, and a development dependency) -------------------------------------------------------------------------------- - -exav's YARA engine is native, in `crates/exav-core/src/yara/`. Parts of it are -derived from yara-x's source: the module implementations (`pe`, `elf`, `dotnet`, -`math`, `hash`, `string`) and the compiler's base64 handling, including field -semantics and the RVA-to-offset and import-hash routines. Each such file carries -a per-file attribution header, and the license text is reproduced below and in -`crates/exav-core/LICENSE-YARA-X`. - -`yara-x-parser` is a regular OPTIONAL DEPENDENCY, enabled by the default-on -`yara` feature of `crates/exav-core`, so a default build does link it. No other -yara-x crate is a dependency of any kind. - -The differential test that compares exav's engine against yara-x -(`crates/exav-core/tests/yara_difftest.rs`) invokes yara-x as a PROGRAM — the -`yr` binary from `PATH` — rather than linking it, and skips when that binary is -absent. - -Components: yara-x-parser 1.18. -Upstream: https://github.com/VirusTotal/yara-x -Copyright (c) VirusTotal and contributors. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other BSD-3-Clause components linked by exav (same 3-clause terms above, with -their respective copyright holders): subtle, tlsh2. - - -------------------------------------------------------------------------------- -1b. libarchive RAR5 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR5 (unpack50) decompressor in -crates/exav-unpack/src/formats/rar5_unpack.rs is a Rust port of the -decompression core of libarchive's archive_read_support_format_rar5.c. As a -source derivative, BSD-2-Clause requires retaining the upstream copyright and -licence text reproduced below. +src/formats/rar5_unpack.rs is a Rust port of the decompression core of +libarchive's archive_read_support_format_rar5.c. The RAR framing of the PPMd +decoder (the PPMd block header in src/formats/rar3_unpack.rs, the range decoder +in src/formats/ppmd7/rangedec.rs and the PPMd/LZSS escape handling) follows +libarchive's archive_read_support_format_rar.c and archive_ppmd7.c, under the +same terms. Upstream: https://github.com/libarchive/libarchive + Copyright (c) 2018 Grzegorz Antoniak (http://antoniak.org) All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1c. nwaples/rardecode RAR3 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR3/RAR4 (unpack29) decompressor in -crates/exav-unpack/src/formats/rar3_unpack.rs is a Rust port of the -decompression core of the Go package nwaples/rardecode. As a source -derivative, BSD-2-Clause requires retaining the upstream copyright and licence -text reproduced below. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +nwaples/rardecode RAR3 reader, BSD-2-Clause (source derivative) +--------------------------------------------------------------- + +src/formats/rar3_unpack.rs is a Rust port of the decompression core of the Go +package nwaples/rardecode. Upstream: https://github.com/nwaples/rardecode + Copyright (c) 2015, Nicholas Waples All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1d. ppmd-rust PPMd7 (var.H) model — CC0-1.0 OR MIT-0 (source derivative) -------------------------------------------------------------------------------- - -The PPMd variant H decoder in crates/exav-unpack/src/formats/ppmd7/ is vendored -and adapted from the `ppmd-rust` crate (version 1.4.0), generalised over a range -decoder trait so it can be driven by either the 7-Zip or the RAR range coder. -`ppmd-rust` is dual-licensed CC0-1.0 OR MIT-0 (public-domain-equivalent) and is -itself a Rust port of Igor Pavlov's 7-Zip PPMd7, which implements -Dmitry Shkarin's public-domain PPMd var.H (2001). It does NOT derive from -UnRAR/RARLAB. - -Upstream: https://crates.io/crates/ppmd-rust -License: CC0-1.0 OR MIT-0 (https://spdx.org/licenses/CC0-1.0, - https://spdx.org/licenses/MIT-0). Both place the work in (or treat it - as) the public domain; no attribution is legally required, and this - entry is recorded for completeness. - -The RAR-specific framing wired into crates/exav-unpack/src/formats/rar3_unpack.rs -(the PPMd block header: order/sub-allocator-memory/escape bits; the RAR -Subbotin carry-less range decoder in ppmd7/rangedec.rs; and the escape-symbol -handling for the PPMd↔LZSS conversion) is reproduced from the public format -description in libarchive's BSD-2-Clause archive_read_support_format_rar.c / -archive_ppmd7.c (already attributed in section 1b above), NOT from UnRAR. - - -------------------------------------------------------------------------------- -1e. dmg-core UDIF reader — Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The UDIF/DMG block reader in crates/exav-unpack/src/formats/udif.rs is vendored -and adapted from the `dmg-core` crate (version 0.1.2), changed to use -`lzma-rust2` instead of `lzma-rs` for XZ decompression. `dmg-core` is licensed -Apache-2.0; the Apache License requires retaining the attribution notices and -reproducing any upstream NOTICE text. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +dmg-core UDIF reader, Apache-2.0 (source derivative) +---------------------------------------------------- + +src/formats/udif.rs is vendored from dmg-core 0.1.2 and changed to decode XZ +with lzma-rust2 instead of lzma-rs. Licensed under the Apache License, Version +2.0 (text below). Upstream ships no NOTICE file. Upstream: https://crates.io/crates/dmg-core Copyright (c) Albert Hui / SecurityRonin. -Licensed under the Apache License, Version 2.0. You may obtain a copy at -http://www.apache.org/licenses/LICENSE-2.0 (see also the Apache-2.0 terms -referenced in section 2 below). Upstream ships no separate NOTICE file. -------------------------------------------------------------------------------- -1f. rust-cab CAB reader — MIT (source derivative) -------------------------------------------------------------------------------- +rust-cab CAB reader, MIT (source derivative) +-------------------------------------------- -The CAB (Microsoft Cabinet) reader in crates/exav-unpack/src/formats/cab_parse/ -is vendored and adapted from the `rust-cab` crate — write support, the datetime -module, unused public types, and tests removed; bounds checks added for -folder data-block and short-read cases. Licensed MIT; the MIT License requires -retaining the copyright and permission notice (reproduced in the exav LICENSE -file) in distributions. +src/formats/cab_parse/ is vendored from rust-cab, with write support, the +datetime module and unused public types removed, and bounds checks added. Upstream: https://github.com/mdsteele/rust-cab -Copyright (c) Matthew D. Steele . - +Copyright (c) Matthew D. Steele -------------------------------------------------------------------------------- -1g. unarc-rs ARJ reader — MIT OR Apache-2.0 (source derivative) -------------------------------------------------------------------------------- -The ARJ reader in crates/exav-unpack/src/formats/arj_parse/ is vendored and -adapted from the `unarc-rs` crate — multi-volume support, volume providers, and -external error types removed; wired to `delharc` for LH6 decompression. -Dual-licensed MIT OR Apache-2.0; the MIT or Apache terms apply at the -distributor's choice (both reproduced/referenced elsewhere in this file). +unarc-rs ARJ and ZOO readers, MIT OR Apache-2.0 (source derivative) +------------------------------------------------------------------- -The ZOO structures in crates/exav-unpack/src/formats/zoo_parse.rs are vendored -and adapted from the same crate, for the same reason the ARJ reader was: linking -it brings in a second `zip` with default features, and Cargo's feature -unification then enables the `zip` crate's own LZMA/bzip2/deflate64 decoders -inside exav, bypassing exav's per-member budget. Adapted to exav's error -handling, with the variable-length part of a directory entry (long filename, -directory name) actually read. - -The ZOO test archives in crates/exav-unpack/tests/fixtures/zoo/ (`store.zoo`, -`default.zoo`, `high_per.zoo`) are that project's own test corpus, redistributed -under the same terms. They hold one identical member under the three -compression methods, which is what lets the codecs be checked against stored -bytes rather than against themselves. +src/formats/arj_parse/ and src/formats/zoo_parse.rs are vendored from unarc-rs +and adapted (multi-volume support and volume providers removed, exav's own +error handling, LH6 through delharc). Used under the MIT License. Upstream: https://github.com/mkrueger/unarc-rs -Copyright (c) Mike Krüger. - - -------------------------------------------------------------------------------- -1h1. unshield / explode — MIT (linked dependency + test fixtures) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/ishield_z.rs reads InstallShield `.z` archives -through the `unshield` crate, whose only real dependency is `explode` — a leaf -decoder for the PKWARE Data Compression Library "implode" codec those archives -use. Both MIT. - -The fixtures in crates/exav-unpack/tests/fixtures/ishieldz/ are that project's -own examples, redistributed under the same terms. `undhr.z` is shipped alongside -`undhr.md`, the uncompressed original, which is what lets the decoder be checked -against the real file rather than against itself. - -This is a DIFFERENT format from the InstallShield `ISc(` InstallScript cabinet, -whose only implementation is the LGPL `unshield` C project — never read, and the -reason that format stays recognised-but-unopened. - -Upstream: https://github.com/agrif/unshield , https://github.com/agrif/explode -Copyright (c) Aaron Griffith. - - -------------------------------------------------------------------------------- -1h2. iced-x86 — MIT (generated opcode tables) -------------------------------------------------------------------------------- - -crates/exav-x86/src/generated.rs holds the SIMD, system and VEX regions of the -x86 opcode maps. It is not hand-written and not transcribed from a manual: it is -emitted by crates/exav-x86/tools/gen-tables, which decodes a probe for every -(prefix, opcode, /digit, mod) combination with `iced-x86` and records what came -back. The tables are therefore derived from `iced-x86`'s data, and the MIT -License requires retaining its copyright and permission notice (the MIT text is -in the exav LICENSE file). +Copyright (c) Mike Krüger -`iced-x86` is also the oracle that exav-x86's differential tests and the -`x86_decode` fuzz target compare against — including the decoding that -exav-pe-emu drives. It is a dev-dependency only: the decoder exav-pe-emu uses at -run time is exav-x86, and no iced code reaches a shipped artifact. -Upstream: https://github.com/icedland/iced -Copyright (C) 2018-present iced project and contributors. +bzip2-rs bzip2 decoder, MIT OR Apache-2.0 (source derivative) +------------------------------------------------------------- +src/formats/bzip2_rs/ is vendored from bzip2-rs 0.1.2 and adapted (the +streaming decoder buffers a whole compressed block). Used under the MIT +License. -------------------------------------------------------------------------------- -1h. oletools test sample — BSD-2-Clause (test fixture) -------------------------------------------------------------------------------- +Upstream: https://github.com/paolobarbolini/bzip2-rs +Copyright (c) 2021 Paolo Barbolini -The Excel 4.0 macro test fixture -crates/exav-unpack/tests/fixtures/xlm_excel4_sample.xls is a benign sample -redistributed from the `oletools` project's test corpus, used to verify the XLM -macro-sheet surfacing in formats/xlm.rs. Licensed BSD-2-Clause. -Upstream: https://github.com/decalage2/oletools -Copyright (c) Philippe Lagadec. +EggDotNet AZO decoder, MIT (source derivative) +---------------------------------------------- +src/formats/azo.rs and src/formats/azo_tables.rs are a Rust port of the AZO +decoder in EggDotNet. -------------------------------------------------------------------------------- -1i. aPLib reference (depack) — public domain (freeware) -------------------------------------------------------------------------------- +Upstream: https://github.com/akolman/EggDotNet +Copyright (c) Andrew Kolman -The aPLib depacker in crates/exav-unpack/src/formats/aplib.rs is a clean-room -implementation of the published aPLib on-disk format. The bit/gamma reader -follows the algorithm published in the freely-distributable aPLib package's -reference `depack.c` by Jørgen Ibsen, which the author released for unrestricted -use ("free to use"). No attribution is legally required; this entry is recorded -for completeness. It does not derive from any GPL source. -Upstream: https://ibsensoftware.com/products_aPLib.html +AutoIt-Ripper, MIT (reference implementation) +--------------------------------------------- - -------------------------------------------------------------------------------- -1j. Microsoft Script Encoder decode tables — public domain -------------------------------------------------------------------------------- - -The substitution/decode tables in crates/exav-unpack/src/formats/screnc.rs are -the published Microsoft Script Encoder (`#@~^`) constants, as documented in -Didier Stevens' public-domain `decode-vbe.py`. Recorded here for completeness; -no attribution is legally required and it does not derive from any GPL source. - -Upstream: https://blog.didierstevens.com/programs/ - - -------------------------------------------------------------------------------- -1k. AutoIt-Ripper — MIT (reference implementation) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/autoit.rs implements the compiled-AutoIt3 script -layout — the EA05 record structure, the Mersenne-Twister keystream and its seeds -— from the MIT-licensed AutoIt-Ripper reference together with the public format -description. +src/formats/autoit.rs implements the compiled AutoIt3 script layout from +AutoIt-Ripper, and src/formats/autoit_data.rs holds its keyword and function +tables, from autoit_data.py. Upstream: https://github.com/nazywam/AutoIt-Ripper -Copyright (c) nazywam and contributors. +Copyright (c) nazywam and contributors + - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: +NSIS format headers, zlib +------------------------- - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. +src/formats/nsis.rs implements the installer layout from NSIS's format header +Source/exehead/fileform.h. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. +Upstream: https://github.com/kichik/nsis +Copyright (c) Nullsoft and contributors +This software is provided 'as-is', without any express or implied warranty. +In no event will the authors be held liable for any damages arising from the +use of this software. -------------------------------------------------------------------------------- -1l. NSIS format headers — zlib -------------------------------------------------------------------------------- +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it freely, +subject to the following restrictions: -crates/exav-unpack/src/formats/nsis.rs implements the installer layout — the -firstheader and block-table structures — from NSIS's own zlib-licensed format -header `Source/exehead/fileform.h`, together with public reverse-engineering. +1. The origin of this software must not be misrepresented; you must not claim + that you wrote the original software. If you use this software in a + product, an acknowledgment in the product documentation would be + appreciated but is not required. -Upstream: https://github.com/kichik/nsis -Copyright (c) Nullsoft and contributors. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. - This software is provided 'as-is', without any express or implied warranty. - In no event will the authors be held liable for any damages arising from the - use of this software. +3. This notice may not be removed or altered from any source distribution. - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it freely, - subject to the following restrictions: - 1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a - product, an acknowledgment in the product documentation would be - appreciated but is not required. +java-hwp, Apache-2.0 (format reference) +--------------------------------------- - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. +The HWP 3.0 preamble offsets in src/formats/hwp3.rs are taken from java-hwp. - 3. This notice may not be removed or altered from any source distribution. +Upstream: https://github.com/ddoleye/java-hwp +Copyright the java-hwp authors. Licensed under the Apache License, Version 2.0 +(text below). -------------------------------------------------------------------------------- -1m. sevenz-rust2 — Apache-2.0 (design reference) -------------------------------------------------------------------------------- +sevenz-rust2, Apache-2.0 (design reference) +------------------------------------------- -crates/exav-unpack/src/formats/sevenz/ takes its reader structure from -sevenz-rust2, re-implemented natively so the crate can keep -`#![forbid(unsafe_code)]` and use exav's own PPMd7 model. +src/formats/sevenz/ takes its reader structure from sevenz-rust2, written anew. Upstream: https://github.com/hasenbanck/sevenz-rust2 -Copyright (c) the sevenz-rust2 authors. - -Licensed under the Apache License, Version 2.0. A copy is available at -http://www.apache.org/licenses/LICENSE-2.0. The license requires retaining the -copyright, patent, trademark and attribution notices from the source, and -reproducing any NOTICE text the upstream distributes. - - -------------------------------------------------------------------------------- -2. ISC-licensed components -------------------------------------------------------------------------------- - -ring 0.17.14 (Apache-2.0 AND ISC), rustls 0.23.41, rustls-webpki, untrusted. -These carry ISC permission-notice terms; reproduce the copyright notice and the -permission/disclaimer notice in distributions. - - -------------------------------------------------------------------------------- -3. MIT / Apache-2.0 dual-licensed components (the bulk of the graph) -------------------------------------------------------------------------------- - -The majority of exav's dependencies are dual-licensed "MIT OR Apache-2.0" (or a -superset including Unlicense / 0BSD / Zlib), so the MIT or Apache terms apply at -the distributor's choice. Notable examples: - - aho-corasick, memchr, daachorse pattern matching / Aho-Corasick - serde, rmp-serde serialization - clap CLI parsing - sha1, sha2, md-5, digest (RustCrypto) hashing - flate2, miniz_oxide, bzip2, zip decompression (sevenz-rust2 is dev-only: - it builds 7z test fixtures; the shipped - 7z reader is native, §1m) - lzfse_rust, delharc, lzxd, DMG/LHA/CAB codecs - bitstream-io, stringprep - cfb OLE/CFB compound-file reader - fatfs, ext4-view FAT and ext2/3/4 filesystem readers - walkdir directory traversal - regex path-filter regexes, and the search - engine behind exav-grep - libc syscall bindings (daemon) - ureq, rustls, webpki-roots HTTPS for the signature updater - wasm-bindgen, js-sys, web-sys, WebAssembly/JS bindings — these reach - wasm-bindgen-futures the published npm artifact - -MIT-only components (the MIT copyright and permission notice must be retained; -the MIT text is in the exav LICENSE file): cfb, hfsplus, apfs (the two latter -via DMG/HFS+/APFS support). - -RustCrypto crates are MIT OR Apache-2.0. Zlib-licensed components present: -miniz_oxide, tinyvec. Unicode-3.0: the ICU / unicode-ident -crates. Other permissive licenses in the graph: CC0-1.0 and MIT-0 (ppmd-rust via -sevenz-rust2), CDLA-Permissive-2.0 (webpki-roots, TLS root store), bzip2-1.0.6 -(libbz2-rs-sys). - -Format documentation consulted ------------------------------ - -Some decoders were written from format descriptions rather than from any code. -Recorded here because a decoder's provenance is part of what makes it auditable: - - EGG ESTsoft Corp., "EGG Format Specification, Version 1.0" — the - vendor's own published document. No ESTsoft code was used. - - AZO ESTsoft's own compression algorithm, absent from the EGG spec. - `crates/exav-unpack/src/formats/azo.rs` and `azo_tables.rs` are a - Rust port of the AZO decoder in `EggDotNet` - (https://github.com/akolman/EggDotNet), MIT, Copyright (c) Andrew - Kolman. ESTsoft's own UnEgg library was NOT used: its licence - forbids commercial use without approval and forbids using it to - develop a compression algorithm. - - HWP3 Preamble offsets taken from `java-hwp` - (https://github.com/ddoleye/java-hwp), Apache-2.0. - Copyright the java-hwp authors. Validated against - `testHWP_3.0.hwp` from the Apache Tika test corpus - (https://github.com/apache/tika), Apache-2.0, Copyright the - Apache Software Foundation — vendored as - `crates/exav-unpack/tests/fixtures/hwp3/tika_testHWP_3.0.hwp`. - - ALZ Structure cross-checked against `unalz` (zlib license) and - `unar`, used as external oracles rather than as sources. - -Full MIT and Apache-2.0 license texts are widely available and are reproduced in -each crate's source distribution; the exav LICENSE file contains the MIT text. - - -------------------------------------------------------------------------------- -Build-only tooling -------------------------------------------------------------------------------- - -Everything above concerns code that reaches a shipped artifact. The -documentation site under `www/` builds with an npm toolchain whose lockfile -includes packages under stronger terms, LGPL-3.0-or-later among them. None of it -is linked into, bundled with, or distributed alongside any exav binary, crate or -WASM module: it runs at build time to produce static HTML and is listed here so -that a licence scan of the repository has an answer rather than a finding. +Copyright the sevenz-rust2 authors. Licensed under the Apache License, Version +2.0 (text below). + + +Public-domain sources, recorded for completeness +------------------------------------------------ + +- src/formats/ppmd7/ is vendored and adapted from ppmd-rust 1.4.0 + (https://crates.io/crates/ppmd-rust), CC0-1.0 OR MIT-0. +- src/formats/aplib.rs reproduces the bit and gamma readers of the reference + depack.c from aPLib by Jørgen Ibsen, released free to use + (https://ibsensoftware.com/products_aPLib.html). +- The decode tables in src/formats/screnc.rs are the Microsoft Script Encoder + constants as published in Didier Stevens' public-domain decode-vbe.py + (https://blog.didierstevens.com/programs/). + + +MIT License +----------- + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Apache License, Version 2.0 +--------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/crates/exav-unpack/README.md b/crates/exav-unpack/README.md index df2a8c8c..718c39c0 100644 --- a/crates/exav-unpack/README.md +++ b/crates/exav-unpack/README.md @@ -38,7 +38,20 @@ if let Some(fmt) = detect(&data) { `extract` holds every member; `walk` hands them over one at a time, a large one as a reader, and reads its source through a `ByteSource` (a slice, or a -`source::BlockCache` over a file). +`source::BlockCache` over a file). A member's metadata (`MemberMeta`) carries +its modification time, Unix mode and link target where the format records +them. A split archive reads as one: `span::ZipSpan` for a `zip -s` set, and +`join_rar_volumes` for a RAR set. + +The crate also ships the `exav-unpack` command (the `cli` feature, on by +default): an extractor for every format above, whose command line is a subset +of `unzip`'s. + +```sh +cargo install exav-unpack +exav-unpack archive.7z -d out/ +exav-unpack -l set.part1.rar +``` Licensed under MIT. Full documentation: [exav.org](https://exav.org/subprojects/exav-unpack/). See [`NOTICE`](https://github.com/sylvinus/exav/blob/main/NOTICE) diff --git a/crates/exav-unpack/src/formats/cab.rs b/crates/exav-unpack/src/formats/cab.rs index 0a77a2e2..4f8a0e62 100644 --- a/crates/exav-unpack/src/formats/cab.rs +++ b/crates/exav-unpack/src/formats/cab.rs @@ -54,8 +54,8 @@ pub(crate) fn stream_cab( name: f.name().to_string(), comp_size: f.uncompressed_size as u64, size: Some(f.uncompressed_size as u64), - encrypted: false, unsupported: Some("CAB folder could not be read up to this member"), + ..MemberMeta::default() }; if let Some(t) = visit(&m, None, budget) { return Ok(Some(t)); @@ -91,12 +91,12 @@ pub(crate) fn stream_cab( name: f.name().to_string(), comp_size: f.uncompressed_size as u64, size: Some(f.uncompressed_size as u64), - encrypted: false, unsupported: Some(if failed { "CAB folder could not be read up to this member" } else { "CAB folder ended before this member's offset" }), + ..MemberMeta::default() }; if let Some(t) = visit(&m, None, budget) { return Ok(Some(t)); @@ -108,8 +108,7 @@ pub(crate) fn stream_cab( name: f.name().to_string(), comp_size: f.uncompressed_size as u64, size: Some(f.uncompressed_size as u64), - encrypted: false, - unsupported: None, + ..MemberMeta::default() }; let r = { let mut window = (&mut reader).take(f.uncompressed_size as u64); diff --git a/crates/exav-unpack/src/formats/dmg.rs b/crates/exav-unpack/src/formats/dmg.rs index ccbf9e6c..db1abae0 100644 --- a/crates/exav-unpack/src/formats/dmg.rs +++ b/crates/exav-unpack/src/formats/dmg.rs @@ -99,72 +99,59 @@ fn find_apfs_offset(data: &[u8]) -> Option { // ─── Encrypted DMG header parsing ──────────────────────────────────────────── -/// Apple encrypted DMG header, matching dmgwiz's `EncryptedDmgHeader`. -/// Deserialized via bincode (big-endian, fixint encoding). -#[derive(serde::Deserialize)] +/// The fields exav uses of an Apple encrypted DMG (`encrcdsa`, version 2) +/// header: 264 bytes, big-endian, laid out as dmgwiz's `EncryptedDmgHeader`. #[cfg(feature = "decrypt")] struct EncryptedDmgHeader { - #[allow(dead_code)] - signature: [char; 8], - #[allow(dead_code)] version: u32, - #[allow(dead_code)] - enc_iv_size: u32, - #[allow(dead_code)] - unk1: u32, - #[allow(dead_code)] - unk2: u32, data_enc_key_bits: u32, - #[allow(dead_code)] - unk4: u32, hmac_key_bits: u32, - #[allow(dead_code)] - uuid: [u8; 16], blocksize: u32, datasize: u64, dataoffset: u64, - #[allow(dead_code)] - unk6: [u8; 24], - #[allow(dead_code)] - kdf_algorithm: u32, - #[allow(dead_code)] - kdf_prng_algorithm: u32, kdf_iteration_count: u32, kdf_salt_len: u32, kdf_salt: [u8; 32], - #[allow(dead_code)] blob_enc_iv_size: u32, blob_enc_iv: [u8; 32], - #[allow(dead_code)] - blob_enc_key_bits: u32, - #[allow(dead_code)] - blob_enc_algorithm: u32, - #[allow(dead_code)] - blob_enc_padding: u32, - #[allow(dead_code)] - blob_enc_mode: u32, encrypted_keyblob_size: u32, encrypted_keyblob1: [u8; 32], encrypted_keyblob2: [u8; 32], } -/// Parse the encrcdsa header from raw bytes using bincode (matching dmgwiz). #[cfg(feature = "decrypt")] fn parse_encrypted_header(data: &[u8]) -> Result { - use bincode::Options; - - let mut cursor = Cursor::new(data); - let header: EncryptedDmgHeader = bincode::DefaultOptions::new() - .with_big_endian() - .with_fixint_encoding() - .deserialize_from(&mut cursor) - .map_err(|e| LimitHit::corrupt(format!("encrypted DMG header parse: {e}")))?; - - let sig: String = header.signature.iter().collect(); - if sig != "encrcdsa" { + let h = data + .get(..264) + .ok_or_else(|| LimitHit::corrupt("encrypted DMG header truncated".into()))?; + if &h[..8] != b"encrcdsa" { return Err(LimitHit::corrupt("missing encrcdsa signature".into())); } - + let u32_at = |o: usize| u32::from_be_bytes(h[o..o + 4].try_into().unwrap()); + let u64_at = |o: usize| u64::from_be_bytes(h[o..o + 8].try_into().unwrap()); + let bytes_at = |o: usize| -> [u8; 32] { h[o..o + 32].try_into().unwrap() }; + let header = EncryptedDmgHeader { + version: u32_at(8), + data_enc_key_bits: u32_at(24), + hmac_key_bits: u32_at(32), + blocksize: u32_at(52), + datasize: u64_at(56), + dataoffset: u64_at(64), + kdf_iteration_count: u32_at(104), + kdf_salt_len: u32_at(108), + kdf_salt: bytes_at(112), + blob_enc_iv_size: u32_at(144), + blob_enc_iv: bytes_at(148), + encrypted_keyblob_size: u32_at(196), + encrypted_keyblob1: bytes_at(200), + encrypted_keyblob2: bytes_at(232), + }; + // Both index fixed-size arrays further on. + if header.kdf_salt_len > 32 || header.encrypted_keyblob_size > 64 { + return Err(LimitHit::corrupt( + "encrypted DMG header sizes out of range".into(), + )); + } Ok(header) } @@ -578,6 +565,60 @@ mod tests { assert!(is_dmg(&buf)); } + /// Every field of an `encrcdsa` header is read from its big-endian place. + #[cfg(feature = "decrypt")] + #[test] + fn encrypted_header_fields() { + let mut h = b"encrcdsa".to_vec(); + // version .. hmac_key_bits: seven u32s, each its own index. + for i in 0..7u32 { + h.extend_from_slice(&(0x0101_0000 + i).to_be_bytes()); + } + h.extend_from_slice(&[0xaa; 16]); // uuid + h.extend_from_slice(&4096u32.to_be_bytes()); // blocksize + h.extend_from_slice(&0x0102_0304_0506_0708u64.to_be_bytes()); // datasize + h.extend_from_slice(&0x1112_1314_1516_1718u64.to_be_bytes()); // dataoffset + h.extend_from_slice(&[0xbb; 24]); + h.extend_from_slice(&1u32.to_be_bytes()); // kdf_algorithm + h.extend_from_slice(&2u32.to_be_bytes()); // kdf_prng_algorithm + h.extend_from_slice(&1000u32.to_be_bytes()); // kdf_iteration_count + h.extend_from_slice(&20u32.to_be_bytes()); // kdf_salt_len + h.extend((0..32).map(|i| i as u8)); // kdf_salt + h.extend_from_slice(&8u32.to_be_bytes()); // blob_enc_iv_size + h.extend((0..32).map(|i| 0x40 + i as u8)); // blob_enc_iv + for v in [192u32, 17, 7, 6] { + h.extend_from_slice(&v.to_be_bytes()); + } + h.extend_from_slice(&48u32.to_be_bytes()); // encrypted_keyblob_size + h.extend((0..64).map(|i| 0x80 + i as u8)); // encrypted_keyblob1, 2 + assert_eq!(h.len(), 264); + h.extend_from_slice(b"trailing data"); + let p = parse_encrypted_header(&h).unwrap(); + assert_eq!(p.version, 0x0101_0000); + assert_eq!(p.data_enc_key_bits, 0x0101_0004); + assert_eq!(p.hmac_key_bits, 0x0101_0006); + assert_eq!(p.blocksize, 4096); + assert_eq!(p.datasize, 0x0102_0304_0506_0708); + assert_eq!(p.dataoffset, 0x1112_1314_1516_1718); + assert_eq!(p.kdf_iteration_count, 1000); + assert_eq!(p.kdf_salt_len, 20); + assert_eq!(p.kdf_salt[31], 31); + assert_eq!(p.blob_enc_iv_size, 8); + assert_eq!(p.blob_enc_iv[0], 0x40); + assert_eq!(p.encrypted_keyblob_size, 48); + assert_eq!(p.encrypted_keyblob1[0], 0x80); + assert_eq!(p.encrypted_keyblob2[31], 0xbf); + assert!(parse_encrypted_header(&h[..263]).is_err(), "short"); + let mut bad = h.clone(); + bad[..8].copy_from_slice(b"encrcdsb"); + assert!(parse_encrypted_header(&bad).is_err(), "signature"); + for (at, v) in [(108, 33u32), (196, 65)] { + let mut bad = h.clone(); + bad[at..at + 4].copy_from_slice(&v.to_be_bytes()); + assert!(parse_encrypted_header(&bad).is_err(), "size at {at}"); + } + } + /// Write a well-formed HFS+ volume header at `off`. fn put_hfs(buf: &mut [u8], off: usize) { buf[off] = 0x48; diff --git a/crates/exav-unpack/src/formats/lha.rs b/crates/exav-unpack/src/formats/lha.rs index 90748274..92e1c874 100644 --- a/crates/exav-unpack/src/formats/lha.rs +++ b/crates/exav-unpack/src/formats/lha.rs @@ -86,8 +86,7 @@ pub(crate) fn walk( name: header.parse_pathname_to_str(), comp_size: header.compressed_size, size: Some(header.original_size), - encrypted: false, - unsupported: None, + ..MemberMeta::default() }; let visited = if dec.is_decoder_supported() { emit_stream(&meta, &mut dec, budget, visit)? diff --git a/crates/exav-unpack/src/formats/mod.rs b/crates/exav-unpack/src/formats/mod.rs index 91b19e1d..d5c6e6ea 100644 --- a/crates/exav-unpack/src/formats/mod.rs +++ b/crates/exav-unpack/src/formats/mod.rs @@ -216,7 +216,7 @@ pub(crate) use pepack::emulate_pe; #[cfg(feature = "pepack")] pub(crate) use pepack::{extract_pepack, is_pepack}; #[cfg(feature = "rar")] -pub(crate) use rar::extract_rar; +pub(crate) use rar::{extract_rar, join_volumes as join_rar_volumes}; #[cfg(feature = "rtf")] pub(crate) use rtf::extract_rtf; #[cfg(feature = "screnc")] diff --git a/crates/exav-unpack/src/formats/rar.rs b/crates/exav-unpack/src/formats/rar.rs index d90960bd..3f8b64ce 100644 --- a/crates/exav-unpack/src/formats/rar.rs +++ b/crates/exav-unpack/src/formats/rar.rs @@ -1,9 +1,10 @@ //! RAR extractor, RAR4 and RAR5, implemented from the public RAR file-format //! field layout. Stored members are copied; RAR3 (unpack 29, LZ + PPMd) members //! go to `rar3_unpack` and RAR5 members to `rar5_unpack`, solid groups -//! included. RAR 1.5/2.x compression, encrypted members and members split -//! across volumes are reported as members we could not extract (the caller -//! still scans the raw archive in place). +//! included. RAR 1.5/2.x compression and encrypted members are reported as +//! members we could not extract (the caller still scans the raw archive in +//! place), and so are members split across volumes, unless the volumes are +//! joined first ([`join_volumes`]). //! //! Layouts used (magic-byte / header-field level only): //! - RAR4 marker `52 61 72 21 1A 07 00`; blocks = `HEAD_CRC u16, HEAD_TYPE u8, @@ -94,10 +95,9 @@ fn push_stored( budget.commit(bytes.len() as u64); out.push(Entry { comp_size: pack, - encrypted: false, - unsupported: None, name, data: bytes, + ..Entry::default() }); Ok(()) } @@ -249,10 +249,9 @@ fn extract_rar4(data: &[u8], start: usize, budget: &mut Budget) -> Result { out.push(Entry { comp_size: pack, - encrypted: false, - unsupported: None, name, data: bytes, + ..Entry::default() }); } Some(_) => { @@ -444,10 +443,9 @@ fn extract_rar5(data: &[u8], start: usize, budget: &mut Budget) -> Result { out.push(Entry { comp_size: data_size, - encrypted: false, - unsupported: None, name: f.name, data: bytes, + ..Entry::default() }); } Some(_) => { @@ -598,6 +596,385 @@ fn rar5_extra_has_crypt(data: &[u8], end: usize, extra_size: u64) -> bool { false } +// ---- Volumes ---------------------------------------------------------------- +// +// A member split across volumes has a file header in each, flagged as +// continuing from the previous volume and/or in the next, with that volume's +// share of the packed data after it. Every part's header carries the CRC-32 of +// that part's packed data, except the last, whose CRC is the whole unpacked +// file's (RARLAB's technote). Joined, the shares are the member's whole packed +// stream, which the single-volume reader then decodes as it decodes any other. + +/// The volumes of a RAR set, in order, as one single-volume archive: each +/// split member's packed data joined behind its first header, which takes the +/// total size and the last part's CRC. `Err` names what does not fit together. +pub(crate) fn join_volumes(volumes: &[&[u8]]) -> Result, String> { + match volumes.first() { + Some(v) if v.starts_with(RAR5_MAGIC) => join_rar5(volumes), + Some(v) if v.starts_with(RAR4_MAGIC) => join_rar4(volumes), + _ => Err("the first part is not a RAR archive".to_string()), + } +} + +/// A member whose parts are still being gathered: its first header and the +/// packed data so far. +struct Pending { + header: Vec, + data: Vec, +} + +fn join_rar4(volumes: &[&[u8]]) -> Result, String> { + let mut out = RAR4_MAGIC.to_vec(); + let mut pending: Option = None; + for (k, v) in volumes.iter().enumerate() { + if !v.starts_with(RAR4_MAGIC) { + return Err(format!("part {} is not a RAR4 volume", k + 1)); + } + let mut pos = RAR4_MAGIC.len(); + while pos + 7 <= v.len() { + let (Some(flags), Some(head_size)) = (u16le(v, pos + 3), u16le(v, pos + 5)) else { + break; + }; + let (htype, head_size) = (v[pos + 2], head_size as usize); + if head_size < 7 || pos + head_size > v.len() { + break; + } + let mut add = if flags & 0x8000 != 0 { + u32le(v, pos + 7).unwrap_or(0) as u64 + } else { + 0 + }; + if htype == 0x74 && flags & 0x100 != 0 { + add |= (u32le(v, pos + 32).unwrap_or(0) as u64) << 32; + } + let end = pos + .checked_add(head_size) + .and_then(|e| e.checked_add(usize::try_from(add).ok()?)) + .filter(|&e| e <= v.len()) + .ok_or_else(|| format!("part {} ends inside a block", k + 1))?; + match htype { + // The archive header once; each volume's end marker never. + 0x73 if k == 0 => out.extend_from_slice(&v[pos..end]), + 0x73 | 0x7B => {} + 0x74 => { + let (before, after) = (flags & 0x01 != 0, flags & 0x02 != 0); + let (header, data) = (&v[pos..pos + head_size], &v[pos + head_size..end]); + match (before, &mut pending) { + (false, None) if after => { + pending = Some(Pending { + header: header.to_vec(), + data: data.to_vec(), + }) + } + (false, None) => out.extend_from_slice(&v[pos..end]), + (true, Some(p)) => p.data.extend_from_slice(data), + (false, Some(_)) => { + return Err(format!("part {} drops a member mid-way", k + 1)) + } + (true, None) => { + return Err(format!( + "part {} goes on with a member whose start is missing", + k + 1 + )) + } + } + if before && !after { + let p = pending.take().expect("a split member was pending"); + out.extend(rar4_whole(p, u32le(v, pos + 16).unwrap_or(0))?); + } + } + _ => out.extend_from_slice(&v[pos..end]), + } + if end <= pos { + break; + } + pos = end; + } + } + if pending.is_some() { + return Err("the last part ends inside a member".to_string()); + } + // ENDARC: CRC, type 0x7B, flags, size 7. + out.extend_from_slice(&[0x3d, 0x7b, 0x7b, 0x00, 0x40, 0x07, 0x00]); + Ok(out) +} + +/// A RAR4 member's whole header and data, from its parts. +fn rar4_whole(mut p: Pending, crc: u32) -> Result, String> { + let h = &mut p.header; + // A stored member's data is its content, so a part missing from between + // the ones given shows as a size the header does not record. (A compressed + // one fails its CRC once decoded.) + let large = u16::from_le_bytes([h[3], h[4]]) & 0x100 != 0; + let unp = u64::from(u32le(h, 11).unwrap_or(0)) + | if large { + u64::from(u32le(h, 36).unwrap_or(0)) << 32 + } else { + 0 + }; + if h.get(25) == Some(&0x30) && unp != p.data.len() as u64 { + return Err(format!( + "a stored member of {unp} bytes has {} in the parts given", + p.data.len() + )); + } + let flags = u16::from_le_bytes([h[3], h[4]]) & !0x03; + h[3..5].copy_from_slice(&flags.to_le_bytes()); + let total = p.data.len() as u64; + h[7..11].copy_from_slice(&(total as u32).to_le_bytes()); + match (flags & 0x100 != 0, h.len() >= 40) { + (true, true) => h[32..36].copy_from_slice(&((total >> 32) as u32).to_le_bytes()), + _ if total > u64::from(u32::MAX) => { + return Err("a member past 4 GiB with no 64-bit size".to_string()) + } + _ => {} + } + if h.len() < 20 { + return Err("a short file header".to_string()); + } + h[16..20].copy_from_slice(&crc.to_le_bytes()); + let mut out = p.header; + out.extend_from_slice(&p.data); + Ok(out) +} + +fn vint_bytes(mut x: u64) -> Vec { + let mut o = Vec::new(); + loop { + let b = (x & 0x7f) as u8; + x >>= 7; + if x == 0 { + o.push(b); + return o; + } + o.push(b | 0x80); + } +} + +/// One RAR5 block: where its header content starts and ends, its type and +/// flags, and where its data ends. +struct Rar5Block { + hdr: usize, + data_off: usize, + htype: u64, + hflags: u64, + end: usize, +} + +fn rar5_block(v: &[u8], pos: usize) -> Option { + let (hsize, n) = vint(v, pos + 4)?; + let hdr = pos + 4 + n; + let data_off = hdr.checked_add(usize::try_from(hsize).ok()?)?; + if hsize == 0 || data_off > v.len() { + return None; + } + let (htype, t1) = vint(v, hdr)?; + let (hflags, t2) = vint(v, hdr + t1)?; + let mut q = hdr + t1 + t2; + if hflags & 0x01 != 0 { + q += vint(v, q)?.1; + } + let data_size = if hflags & 0x02 != 0 { vint(v, q)?.0 } else { 0 }; + let end = data_off.checked_add(usize::try_from(data_size).ok()?)?; + (end <= v.len()).then_some(Rar5Block { + hdr, + data_off, + htype, + hflags, + end, + }) +} + +fn join_rar5(volumes: &[&[u8]]) -> Result, String> { + let mut out = RAR5_MAGIC.to_vec(); + // The pending member, with the header content of its first part. + let mut pending: Option = None; + // Service blocks met while a member is pending, which go after it. + let mut deferred = Vec::new(); + for (k, v) in volumes.iter().enumerate() { + if !v.starts_with(RAR5_MAGIC) { + return Err(format!("part {} is not a RAR5 volume", k + 1)); + } + let mut pos = RAR5_MAGIC.len(); + while pos + 4 < v.len() { + let Some(b) = rar5_block(v, pos) else { break }; + // A volume past the first records its number: it must be where + // it is given. + if b.htype == 1 && k > 0 { + if let Some(n) = rar5_volume_number(&v[b.hdr..b.data_off]) { + if n != k as u64 { + return Err(format!("part {} is volume {} of its set", k + 1, n + 1)); + } + } + } + match b.htype { + 1 if k == 0 => out.extend_from_slice(&v[pos..b.end]), + 1 | 5 => {} + // The quick-open record each volume ends with caches that + // volume's headers at offsets in it: it has no place in the + // joined archive. + 3 if b.hflags & 0x18 == 0 && rar5_name(&v[b.hdr..b.data_off]) == Some(b"QO") => {} + 2 | 3 => { + let (before, after) = (b.hflags & 0x08 != 0, b.hflags & 0x10 != 0); + let data = &v[b.data_off..b.end]; + match (before, &mut pending) { + (false, None) if after => { + pending = Some(Pending { + header: v[b.hdr..b.data_off].to_vec(), + data: data.to_vec(), + }) + } + (false, None) => out.extend_from_slice(&v[pos..b.end]), + (true, Some(p)) => p.data.extend_from_slice(data), + (false, Some(_)) if b.htype == 3 && !after => { + deferred.extend_from_slice(&v[pos..b.end]) + } + (false, Some(_)) => { + return Err(format!("part {} drops a member mid-way", k + 1)) + } + (true, None) => { + return Err(format!( + "part {} goes on with a member whose start is missing", + k + 1 + )) + } + } + if before && !after { + let p = pending.take().expect("a split member was pending"); + out.extend(rar5_whole(p, rar5_crc(v, &b))?); + out.append(&mut deferred); + } + } + _ => out.extend_from_slice(&v[pos..b.end]), + } + if b.end <= pos { + break; + } + pos = b.end; + } + } + if pending.is_some() { + return Err("the last part ends inside a member".to_string()); + } + // End of archive: type 5, no flags, no end-of-archive flags. + let content = [vint_bytes(5), vint_bytes(0), vint_bytes(0)].concat(); + out.extend_from_slice(&crc32_ieee(&content).to_le_bytes()); + out.extend(vint_bytes(content.len() as u64)); + out.extend(content); + Ok(out) +} + +/// The volume number a RAR5 main header records (1 for the second volume), +/// when it records one. +fn rar5_volume_number(c: &[u8]) -> Option { + let (_, t1) = vint(c, 0)?; + let (hflags, t2) = vint(c, t1)?; + let mut q = t1 + t2; + if hflags & 0x01 != 0 { + q += vint(c, q)?.1; + } + if hflags & 0x02 != 0 { + q += vint(c, q)?.1; + } + let (archive_flags, n) = vint(c, q)?; + (archive_flags & 0x02 != 0) + .then(|| vint(c, q + n).map(|v| v.0)) + .flatten() +} + +/// The name in a RAR5 file or service header's content. +fn rar5_name(c: &[u8]) -> Option<&[u8]> { + let f = rar5_fields_at(c)?; + let (file_flags, n) = vint(c, f.after_sizes)?; + let mut q = f.after_sizes + n; + q += vint(c, q)?.1; // unpacked size + q += vint(c, q)?.1; // attributes + q += if file_flags & 0x02 != 0 { 4 } else { 0 } + if file_flags & 0x04 != 0 { 4 } else { 0 }; + q += vint(c, q)?.1; // compression information + q += vint(c, q)?.1; // host OS + let (len, n) = vint(c, q)?; + c.get(q + n..q + n + usize::try_from(len).ok()?) +} + +/// The data CRC a RAR5 file block records, when it records one. +fn rar5_crc(v: &[u8], b: &Rar5Block) -> Option { + let f = rar5_fields_at(&v[b.hdr..b.data_off])?; + f.crc_at.map(|at| u32le(&v[b.hdr..], at).unwrap_or(0)) +} + +/// Where the parts of a RAR5 file header content are: the end of its +/// data-size field, the start of the file fields, and the CRC's offset. +struct Rar5Layout { + after_sizes: usize, + crc_at: Option, +} + +fn rar5_fields_at(c: &[u8]) -> Option { + let (_, t1) = vint(c, 0)?; + let (hflags, t2) = vint(c, t1)?; + let mut q = t1 + t2; + if hflags & 0x01 != 0 { + q += vint(c, q)?.1; + } + if hflags & 0x02 != 0 { + q += vint(c, q)?.1; + } + let after_sizes = q; + let (file_flags, n) = vint(c, q)?; + q += n; + q += vint(c, q)?.1; // unpacked size + q += vint(c, q)?.1; // attributes + if file_flags & 0x02 != 0 { + q += 4; // mtime + } + let crc_at = (file_flags & 0x04 != 0).then_some(q); + Some(Rar5Layout { + after_sizes, + crc_at, + }) +} + +/// A RAR5 member's whole block, from its parts: the first header with the +/// split flags off, the total data size, and the last part's CRC. +fn rar5_whole(p: Pending, crc: Option) -> Result, String> { + let c = &p.header; + let bad = || "a short file header".to_string(); + let layout = rar5_fields_at(c).ok_or_else(bad)?; + // As for RAR4: a stored member's size shows a missing part. + let (file_flags, n) = vint(c, layout.after_sizes).ok_or_else(bad)?; + let (unp, n2) = vint(c, layout.after_sizes + n).ok_or_else(bad)?; + let mut q = layout.after_sizes + n + n2; + q += vint(c, q).ok_or_else(bad)?.1; + q += if file_flags & 0x02 != 0 { 4 } else { 0 } + if file_flags & 0x04 != 0 { 4 } else { 0 }; + let (comp_info, _) = vint(c, q).ok_or_else(bad)?; + if (comp_info >> 7) & 0x7 == 0 && file_flags & 0x08 == 0 && unp != p.data.len() as u64 { + return Err(format!( + "a stored member of {unp} bytes has {} in the parts given", + p.data.len() + )); + } + let (htype, t1) = vint(c, 0).ok_or_else(bad)?; + let (hflags, t2) = vint(c, t1).ok_or_else(bad)?; + let mut content = vint_bytes(htype); + content.extend(vint_bytes((hflags & !0x18) | 0x02)); + if hflags & 0x01 != 0 { + let (extra, _) = vint(c, t1 + t2).ok_or_else(bad)?; + content.extend(vint_bytes(extra)); + } + content.extend(vint_bytes(p.data.len() as u64)); + let shift = content.len() as isize - layout.after_sizes as isize; + content.extend_from_slice(&c[layout.after_sizes..]); + if let (Some(at), Some(crc)) = (layout.crc_at, crc) { + let at = (at as isize + shift) as usize; + content[at..at + 4].copy_from_slice(&crc.to_le_bytes()); + } + let mut out = crc32_ieee(&content).to_le_bytes().to_vec(); + out.extend(vint_bytes(content.len() as u64)); + out.extend(content); + out.extend_from_slice(&p.data); + Ok(out) +} + /// CRC-32 (IEEE, poly 0xEDB88320) over `data`. Every decoded RAR member is /// checked against the CRC the archive records, in every build: a decoder that /// produces plausible-looking wrong bytes — which is exactly what happens when @@ -754,6 +1131,178 @@ mod tests { assert_eq!(e[0].data, b"EXAV_RAR5_MARKER_payload"); } + /// `arc`'s members cut into `chunk`-byte parts, a new volume at each cut, + /// as RARLAB's technote describes a set: the split flags, each part's + /// share of the data and size, and the CRC of its share but in the last. + fn split(arc: &[u8], chunk: usize) -> Vec> { + let rar5 = arc.starts_with(RAR5_MAGIC); + let magic = if rar5 { RAR5_MAGIC } else { RAR4_MAGIC }; + let mut vols: Vec> = vec![magic.to_vec()]; + let mut main = Vec::new(); + let mut pos = magic.len(); + while pos + 7 < arc.len() { + // (block bytes, file header?, header end, data end) + let (end, file, data_off) = if rar5 { + let Some(b) = rar5_block(arc, pos) else { break }; + (b.end, b.htype == 2, b.data_off) + } else { + let (flags, size) = ( + u16le(arc, pos + 3).unwrap(), + u16le(arc, pos + 5).unwrap() as usize, + ); + let add = if flags & 0x8000 != 0 { + u32le(arc, pos + 7).unwrap() as usize + } else { + 0 + }; + (pos + size + add, arc[pos + 2] == 0x74, pos + size) + }; + if !file { + if main.is_empty() { + main = arc[pos..end].to_vec(); + vols[0].extend_from_slice(&main); + } + pos = end; + continue; + } + let data = &arc[data_off..end]; + let parts: Vec<&[u8]> = data.chunks(chunk.max(1)).collect(); + let parts = if parts.is_empty() { + vec![&data[..0]] + } else { + parts + }; + for (j, part) in parts.iter().enumerate() { + let (before, after) = (j > 0, j + 1 < parts.len()); + if before { + let mut v = magic.to_vec(); + v.extend_from_slice(&main); + vols.push(v); + } + let v = vols.last_mut().unwrap(); + if rar5 { + let c = &arc[rar5_block(arc, pos).unwrap().hdr..data_off]; + let mut p = Pending { + header: c.to_vec(), + data: part.to_vec(), + }; + // Built as a whole member, then flagged as a part. + let crc = (!after) + .then(|| rar5_crc(arc, &rar5_block(arc, pos).unwrap())) + .flatten(); + let crc = crc.or_else(|| after.then(|| crc32_ieee(part))); + let flags_at = vint(&p.header, 0).unwrap().1; + let (fl, n) = vint(&p.header, flags_at).unwrap(); + let fl = fl | if before { 0x08 } else { 0 } | if after { 0x10 } else { 0 }; + let mut h = p.header[..flags_at].to_vec(); + h.extend(vint_bytes(fl)); + h.extend_from_slice(&p.header[flags_at + n..]); + p.header = h; + let whole = rar5_part(p, crc); + v.extend(whole); + } else { + let mut h = arc[pos..data_off].to_vec(); + let fl = u16::from_le_bytes([h[3], h[4]]) + | if before { 1 } else { 0 } + | if after { 2 } else { 0 }; + h[3..5].copy_from_slice(&fl.to_le_bytes()); + h[7..11].copy_from_slice(&(part.len() as u32).to_le_bytes()); + if after { + h[16..20].copy_from_slice(&crc32_ieee(part).to_le_bytes()); + } + v.extend(h); + v.extend_from_slice(part); + } + } + pos = end; + } + vols + } + + /// A RAR5 block from header content whose flags are already set, with + /// `p.data` as its data and `crc` in its CRC field. + fn rar5_part(p: Pending, crc: Option) -> Vec { + let c = &p.header; + let layout = rar5_fields_at(c).unwrap(); + let (htype, t1) = vint(c, 0).unwrap(); + let (hflags, t2) = vint(c, t1).unwrap(); + let mut content = vint_bytes(htype); + content.extend(vint_bytes(hflags | 0x02)); + if hflags & 0x01 != 0 { + content.extend(vint_bytes(vint(c, t1 + t2).unwrap().0)); + } + content.extend(vint_bytes(p.data.len() as u64)); + let shift = content.len() as isize - layout.after_sizes as isize; + content.extend_from_slice(&c[layout.after_sizes..]); + if let (Some(at), Some(crc)) = (layout.crc_at, crc) { + let at = (at as isize + shift) as usize; + content[at..at + 4].copy_from_slice(&crc.to_le_bytes()); + } + let mut out = crc32_ieee(&content).to_le_bytes().to_vec(); + out.extend(vint_bytes(content.len() as u64)); + out.extend(content); + out.extend_from_slice(&p.data); + out + } + + /// A set split from an archive, joined again, extracts as the archive + /// does: stored members, and the compressed members of a solid archive, + /// RAR4 and RAR5, cut at many sizes. A set missing a part is refused. + #[test] + fn a_volume_set_joins_into_its_archive() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/rar_solid"); + let arcs = [ + rar4_stored(b"hi.txt", &[b'z'; 1000]), + rar5_stored(b"a/b.bin", &[b'y'; 1000]), + std::fs::read(format!("{dir}/solid_rar4.rar")).unwrap(), + std::fs::read(format!("{dir}/solid_rar5.rar")).unwrap(), + ]; + let names = |e: &[Entry]| { + e.iter() + .map(|e| (e.name.clone(), e.data.clone(), e.unsupported)) + .collect::>() + }; + for (i, arc) in arcs.iter().enumerate() { + let want = names(&extract_rar(arc, &mut budget()).unwrap()); + assert!( + want.iter().all(|w| w.2.is_none() && !w.1.is_empty()), + "archive {i}: {want:?}" + ); + for chunk in [1, 7, 100, 333, 1 << 20] { + let vols = split(arc, chunk); + let refs: Vec<&[u8]> = vols.iter().map(Vec::as_slice).collect(); + let joined = join_volumes(&refs) + .unwrap_or_else(|e| panic!("archive {i} chunk {chunk}: {e}")); + assert_eq!( + names(&extract_rar(&joined, &mut budget()).unwrap()), + want, + "archive {i} chunk {chunk}" + ); + if refs.len() > 2 { + // A part gone from the middle: refused, or the member it + // held reported unreadable, never passed off as whole. + let missing: Vec<&[u8]> = refs + .iter() + .enumerate() + .filter(|(k, _)| *k != 1) + .map(|(_, v)| *v) + .collect(); + if let Ok(joined) = join_volumes(&missing) { + let got = names(&extract_rar(&joined, &mut budget()).unwrap()); + assert!( + got.iter().any(|g| g.2.is_some()), + "archive {i} chunk {chunk}: a missing part" + ); + } + assert!( + join_volumes(&refs[1..]).is_err(), + "archive {i} chunk {chunk}: no first part" + ); + } + } + } + } + #[test] fn truncated_does_not_panic() { let arc = rar5_stored(b"x", b"0123456789"); diff --git a/crates/exav-unpack/src/formats/sevenz/entry.rs b/crates/exav-unpack/src/formats/sevenz/entry.rs index d91d36ad..44b46ba7 100644 --- a/crates/exav-unpack/src/formats/sevenz/entry.rs +++ b/crates/exav-unpack/src/formats/sevenz/entry.rs @@ -315,6 +315,7 @@ fn stream_sevenz( size: None, encrypted: true, unsupported: Some("7z: encrypted header, unsupported codec"), + ..MemberMeta::default() }; return Ok(visit(&meta, None, budget)); } @@ -347,6 +348,7 @@ fn stream_sevenz( size: Some(file.size), encrypted: true, unsupported: Some(reason), + ..MemberMeta::default() }; if let Some(r) = visit(&meta, None, budget) { return Ok(Some(r)); @@ -445,7 +447,7 @@ fn stream_sevenz( comp_size: file.size, size: Some(file.size), encrypted: true, - unsupported: None, + ..MemberMeta::default() }; emit_bytes(&meta, Some(fd), budget, visit)? } @@ -456,6 +458,7 @@ fn stream_sevenz( size: Some(file.size), encrypted: true, unsupported: Some("7z: wrong or missing password"), + ..MemberMeta::default() }; visit(&meta, None, budget) } @@ -475,8 +478,8 @@ fn stream_sevenz( name, comp_size: file.size, size: Some(file.size), - encrypted: false, unsupported: Some("7z: member starts past the peak-buffer limit"), + ..MemberMeta::default() }; if let Some(t) = visit(&meta, None, budget) { return Ok(Some(t)); @@ -509,12 +512,12 @@ fn stream_sevenz( name, comp_size: file.size, size: Some(file.size), - encrypted: false, unsupported: Some(if failed { "7z: solid block failed to decode before this member" } else { "7z: solid block ended before this member" }), + ..MemberMeta::default() }; if let Some(t) = visit(&meta, None, budget) { return Ok(Some(t)); @@ -526,8 +529,7 @@ fn stream_sevenz( name, comp_size: file.size, size: Some(file.size), - encrypted: false, - unsupported: None, + ..MemberMeta::default() }; if let Some(t) = emit_stream(&meta, &mut window, budget, visit)? { return Ok(Some(t)); diff --git a/crates/exav-unpack/src/formats/swf.rs b/crates/exav-unpack/src/formats/swf.rs index 3f1f7a36..30df821c 100644 --- a/crates/exav-unpack/src/formats/swf.rs +++ b/crates/exav-unpack/src/formats/swf.rs @@ -38,8 +38,7 @@ pub(crate) fn walk( name: "movie.swf".to_string(), comp_size: (src.len() as u64).saturating_sub(8), size: Some(u32::from_le_bytes([hdr[4], hdr[5], hdr[6], hdr[7]]) as u64), - encrypted: false, - unsupported: None, + ..MemberMeta::default() }; if &hdr[0..3] == b"CWS" { let body = flate2::read::ZlibDecoder::new(Reader::range(src, 8, src.len())); diff --git a/crates/exav-unpack/src/formats/szdd.rs b/crates/exav-unpack/src/formats/szdd.rs index 89ce2908..7f2ce67d 100644 --- a/crates/exav-unpack/src/formats/szdd.rs +++ b/crates/exav-unpack/src/formats/szdd.rs @@ -57,8 +57,7 @@ pub(crate) fn walk( name: szdd_name(&hdr), comp_size: (src.len() - 14) as u64, size: Some(declared), - encrypted: false, - unsupported: None, + ..MemberMeta::default() }; emit_stream(&meta, &mut rdr, budget, visit) } diff --git a/crates/exav-unpack/src/formats/tar.rs b/crates/exav-unpack/src/formats/tar.rs index e1681d6e..50ef69cf 100644 --- a/crates/exav-unpack/src/formats/tar.rs +++ b/crates/exav-unpack/src/formats/tar.rs @@ -3,7 +3,7 @@ use std::io::{Seek, SeekFrom}; use crate::source::{ByteSource, Reader}; -use crate::stream::{emit_stream, MemberMeta, Visit}; +use crate::stream::{emit_stream, MemberMeta, Mtime, Visit}; use crate::{Budget, LimitHit}; pub(crate) fn walk( @@ -34,12 +34,31 @@ pub(crate) fn walk( .map(|p| p.to_string_lossy().into_owned()) .unwrap_or_else(|_| "tar-entry".to_string()); let size = entry.size(); + let header = entry.header(); + // The type bits `st_mode` has and a tar header keeps apart. A hard + // link has neither a type of its own nor content here: no mode. + let kind = match header.entry_type() { + ::tar::EntryType::Regular | ::tar::EntryType::Continuous => Some(0o100000), + ::tar::EntryType::Directory => Some(0o040000), + ::tar::EntryType::Symlink => Some(0o120000), + _ => None, + }; + let symlink = header.entry_type() == ::tar::EntryType::Symlink; let meta = MemberMeta { name, comp_size: size, size: Some(size), - encrypted: false, - unsupported: None, + mtime: header + .mtime() + .ok() + .and_then(|t| i64::try_from(t).ok()) + .map(Mtime::Unix), + mode: kind.and_then(|k| Some(k | (header.mode().ok()? & 0o7777))), + link: symlink + .then(|| entry.link_name().ok().flatten()) + .flatten() + .map(|l| l.to_string_lossy().into_owned()), + ..MemberMeta::default() }; if let Some(t) = emit_stream(&meta, &mut entry, budget, visit)? { return Ok(Some(t)); diff --git a/crates/exav-unpack/src/formats/udf.rs b/crates/exav-unpack/src/formats/udf.rs index 9aca7f52..48728cbc 100644 --- a/crates/exav-unpack/src/formats/udf.rs +++ b/crates/exav-unpack/src/formats/udf.rs @@ -946,8 +946,8 @@ pub(crate) fn stream_udf( name: path, comp_size: size, size: Some(size), - encrypted: false, unsupported: Some(reason), + ..MemberMeta::default() }; Ok(visit(&meta, None, budget)) } @@ -956,8 +956,7 @@ pub(crate) fn stream_udf( name: path, comp_size: size, size: Some(size), - encrypted: false, - unsupported: None, + ..MemberMeta::default() }; let mut reader = RunReader { img, diff --git a/crates/exav-unpack/src/formats/zip.rs b/crates/exav-unpack/src/formats/zip.rs index 9edb49d7..087d0001 100644 --- a/crates/exav-unpack/src/formats/zip.rs +++ b/crates/exav-unpack/src/formats/zip.rs @@ -936,10 +936,10 @@ fn parse_local_member( budget.commit(out.len() as u64); Ok(Some(Entry { comp_size: comp as u64, - encrypted: false, unsupported: part_way.then_some(PART_WAY), name, data: out, + ..Entry::default() })) } @@ -1161,9 +1161,9 @@ pub fn extract_zip_from( let entry = Entry { comp_size: comp, encrypted: true, - unsupported: None, name, data: plain, + ..Entry::default() }; if let Some(r) = visit(entry, budget) { return Ok(Some(r)); @@ -1258,10 +1258,10 @@ pub fn extract_zip_from( // Compressed size is meaningful for `.cdb` `FileSizeInContainer`. let entry = Entry { comp_size: comp, - encrypted: false, unsupported: part_way.then_some(PART_WAY), name, data: buf, + ..Entry::default() }; if let Some(r) = visit(entry, budget) { return Ok(Some(r)); @@ -1438,6 +1438,35 @@ fn read_spans( /// gigabytes unclaimed would otherwise turn opening it into a full read. const MAX_ORPHAN_SCAN: u64 = 16 * 1024 * 1024; +/// When a member was last modified: the Unix time of its extended-timestamp +/// or NTFS extra field, as `unzip` prefers, else its DOS wall-clock time. +fn zip_mtime<'a>( + extra: impl Iterator, + dos: Option<::zip::DateTime>, +) -> Option { + // FILETIME counts 100 ns ticks from 1601; the Unix epoch is this many + // seconds later. + const FILETIME_EPOCH: i64 = 11_644_473_600; + let unix = extra + .filter_map(|x| match x { + ::zip::ExtraField::ExtendedTimestamp(t) => t.mod_time().map(i64::from), + ::zip::ExtraField::Ntfs(n) => i64::try_from(n.mtime() / 10_000_000) + .ok() + .map(|s| s - FILETIME_EPOCH), + }) + .next(); + unix.map(crate::Mtime::Unix).or_else(|| { + dos.map(|d| crate::Mtime::Local { + year: d.year(), + month: d.month(), + day: d.day(), + hour: d.hour(), + minute: d.minute(), + second: d.second(), + }) + }) +} + /// Walk a ZIP off its source. Only the central directory and the members /// actually read are fetched. A cleartext member is handed over as the /// crate's decompressing reader, so a member decompressing to any size is @@ -1476,41 +1505,57 @@ pub(crate) fn walk( } let meta = MemberMeta { name: format!("zip entry {i}"), - comp_size: 0, - size: None, - encrypted: false, unsupported: Some("ZIP member header will not parse"), + ..MemberMeta::default() }; if let Some(t) = visit(&meta, None, budget) { return Ok(Some(t)); } continue; } - let (name, is_file, encrypted, comp, size, method) = { + let (name, is_dir, encrypted, comp, size, method, mtime, mode) = { let f = zip.by_index_raw(i).map_err(|e| zip_entry_error(i, &e))?; ( f.name().to_string(), - f.is_file(), + f.is_dir(), f.encrypted(), f.compressed_size(), f.size(), zip_method_code(&f.compression()), + zip_mtime(f.extra_data_fields(), f.last_modified()), + f.unix_mode(), ) }; - // `is_file()` is false purely because the name ends in '/'. A JAR packer + // `is_dir()` is true purely because the name ends in '/'. A JAR packer // buys exactly that: `kingDavid/9.class/` holds a real deflate-compressed // class that the JVM loads by name, while every ZIP tool discards it as a // folder. Skip only what carries nothing at all: a "directory" with // content is content. (The same guard lives in `extract_zip_from`.) - if !is_file && comp == 0 { - continue; // directory: counted toward the file budget above, skipped + if is_dir && comp == 0 { + // Counted toward the file budget above, and handed over only to + // a caller that makes directories. + if budget.visit_directories { + let meta = MemberMeta { + name, + mtime, + mode: Some(mode.map_or(0o040755, |m| 0o040000 | (m & 0o7777))), + ..MemberMeta::default() + }; + if let Some(t) = visit(&meta, None, budget) { + return Ok(Some(t)); + } + } + continue; } let meta = MemberMeta { name, comp_size: comp, size: Some(size), encrypted, - unsupported: None, + mtime, + mode, + zip_method: Some(method), + ..MemberMeta::default() }; if encrypted { if let Some(t) = walk_encrypted(&mut zip, i, meta, budget, visit)? { diff --git a/crates/exav-unpack/src/lib.rs b/crates/exav-unpack/src/lib.rs index 81f94d3d..d9902546 100644 --- a/crates/exav-unpack/src/lib.rs +++ b/crates/exav-unpack/src/lib.rs @@ -56,13 +56,14 @@ use formats::*; mod inflate; pub mod profile; pub mod source; +pub mod span; mod stream; pub mod volume; #[cfg(feature = "pdf")] pub use formats::has_obfuscated_name_object; #[allow(unused_imports)] pub(crate) use stream::Region; -pub use stream::{is_budget_overflow, walk, Member, MemberMeta, Visit}; +pub use stream::{is_budget_overflow, walk, Member, MemberMeta, Mtime, Visit}; /// Count of overlapping ZIP local file records, the signal behind ClamAV's /// `Heuristics.Zip.OverlappingFiles`. Zero for any well-formed archive. @@ -281,6 +282,10 @@ pub struct Budget { /// Only has effect with the `checksums` Cargo feature compiled in; without /// it, checksums are never verified regardless of this flag. pub(crate) verify_checksums: bool, + /// Hand the visitor the directory entries of a container that has them as + /// members of their own (ZIP), with no content. Off by default: a scan + /// has nothing to read in them. See [`Budget::set_visit_directories`]. + pub(crate) visit_directories: bool, } impl Budget { @@ -298,6 +303,7 @@ impl Budget { pe_emulation_steps: 0, passwords: Vec::new(), verify_checksums: false, + visit_directories: false, } } @@ -317,6 +323,13 @@ impl Budget { self } + /// Also visit directory entries (default off), for a caller that recreates + /// the tree, empty directories included. + pub fn set_visit_directories(&mut self, visit: bool) -> &mut Self { + self.visit_directories = visit; + self + } + /// Whether checksum verification is active: the `checksums` feature is /// compiled in **and** [`Budget::set_verify_checksums`] was enabled. pub fn should_verify_checksums(&self) -> bool { @@ -516,7 +529,10 @@ pub(crate) fn read_at( } /// One extracted member. -#[derive(Debug, Clone, PartialEq)] +/// +/// New fields may appear in any release: build one with [`Entry::new`] or +/// [`Entry::unsupported`] and assign the others. +#[derive(Debug, Clone, PartialEq, Default)] pub struct Entry { pub name: String, pub data: Vec, @@ -531,6 +547,10 @@ pub struct Entry { /// The scanner surfaces this as a distinct `Unscannable` verdict rather than /// silently treating the member as clean. pub unsupported: Option<&'static str>, + /// When the member was last modified (see [`MemberMeta::mtime`]). + pub mtime: Option, + /// Its Unix mode (see [`MemberMeta::mode`]). + pub mode: Option, } impl Entry { @@ -539,10 +559,9 @@ impl Entry { pub fn new(name: String, data: Vec) -> Self { Entry { comp_size: data.len() as u64, - encrypted: false, - unsupported: None, name, data, + ..Entry::default() } } @@ -557,10 +576,10 @@ impl Entry { ) -> Self { Entry { name, - data: Vec::new(), comp_size, encrypted, unsupported: Some(reason), + ..Entry::default() } } } @@ -1409,6 +1428,13 @@ fn detect_with(src: &dyn ByteSource, prescan: Option<&dyn Fn() -> Prescan>) -> O if let Some(data) = src.as_slice() { return detect_probe(&Probe::whole(data)); } + // The first checks need a few bytes, so an archive typed by them is not + // read `DETECT_HEAD` deep: over a network or browser reader that is most + // of a small archive. Only those `detect_probe` makes first: a later one, + // such as RAR's, could be overruled by an earlier check that needs more. + if let Some(fmt) = archive_magic(&src.window(0, 16)) { + return Some(fmt); + } let head = src.window(0, DETECT_HEAD); if head.len() == src.len() { return detect_probe(&Probe::whole(&head)); @@ -2153,6 +2179,8 @@ pub fn extract( comp_size: meta.comp_size, encrypted: meta.encrypted, unsupported: meta.unsupported, + mtime: meta.mtime, + mode: meta.mode, }; if let Some(content) = content { match content.into_bytes(meta, budget) { @@ -2177,6 +2205,16 @@ pub fn extract( } } +/// The volumes of a RAR set (`x.part1.rar`, `x.part2.rar`, ..., or `x.rar`, +/// `x.r00`, ...), in order, as one single-volume RAR archive: what [`walk`] +/// reads as [`Format::Rar`]. A member split across volumes is decoded whole +/// that way, where read one volume at a time it is reported unreadable. +/// `Err` names a part that does not fit the set. +#[cfg(feature = "rar")] +pub fn join_rar_volumes(volumes: &[&[u8]]) -> Result, String> { + formats::join_rar_volumes(volumes) +} + /// True if `data` looks like a UPX-packed executable (a valid `PackHeader` is /// present). Used by callers whose file-type classifier already recognises /// PE/ELF/Mach-O and wants to additionally unpack UPX. diff --git a/crates/exav-unpack/src/main.rs b/crates/exav-unpack/src/main.rs index 18ffb10f..7b0db867 100644 --- a/crates/exav-unpack/src/main.rs +++ b/crates/exav-unpack/src/main.rs @@ -1,175 +1,1524 @@ -//! `exav-unpack`: a memory-safe universal archive extractor. +//! `exav-unpack`: extract, list and test archives, with `unzip`'s command line. //! -//! Auto-detects the container format by magic and lists or extracts its members -//! using the pure-Rust `exav-unpack` library (the same extractors the exav -//! scanner uses — including a pure-Rust RAR and UPX). No native/C dependencies. +//! The command line is a subset of Info-ZIP `unzip` 6.00's: every option it +//! takes means what it means to `unzip`, and an `unzip` option it does not take +//! is refused (exit 10) rather than ignored. It adds long options only +//! (`--volume`, the `--max-*` limits, `--help`, `--version`), and reads every +//! archive format the `exav-unpack` library reads, not only ZIP. //! -//! Usage: -//! exav-unpack list -//! exav-unpack extract [output-dir] (default: current directory) +//! Where it differs from `unzip` on purpose: +//! - no password prompt (not implemented yet): an encrypted member with no +//! `-P` is skipped, as `unzip` does with no terminal, after the library's +//! built-in passwords; +//! - a symbolic link whose target leaves the extraction directory is not made; +//! - every member is `extracting:`, whatever its compression; +//! - a split archive is read from any of its parts. //! -//! `` may be any part of a byte-split set (`big.7z.001`, `.002`, …): -//! the volumes are rejoined from the same directory first, since no single part -//! is a readable archive on its own. +//! The archive is read from disk as it is needed and each member written as +//! it is decoded: neither is held in memory whole, except where a format's +//! decoder needs it. #![forbid(unsafe_code)] -use exav_unpack::{detect, extract, Budget, Format, Limits}; +use chrono::{Local, LocalResult, TimeZone}; +use exav_unpack::source::BlockCache; +use exav_unpack::{detect, walk, Budget, ByteSource, Format, Limits, Member, MemberMeta, Mtime}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, BufRead, Read, Seek, SeekFrom, Write}; use std::path::{Component, Path, PathBuf}; use std::process::ExitCode; +use std::time::{Duration, SystemTime}; + +const USAGE: &str = "\ +usage: exav-unpack [-opts[modifiers]] archive [list] [-x xlist] [-d exdir] + +Extracts the members in list (all by default), except those in xlist, into +exdir (the current directory by default). The options are a subset of +unzip's and mean what they mean to unzip; archive may be a wildcard, and of +any format exav-unpack reads. + + -l list members -t test members + -p extract members to stdout, no messages -c as -p, with messages + -Z1 list member names only -x exclude the members that follow + -d extract into exdir +modifiers: + -o overwrite files without prompting -n never overwrite files + -q quiet (-qq quieter) -P password to decrypt members + -j junk paths (no directories) -C match names case-insensitively + -D do not restore directory times (-DD: no times at all) + +exav-unpack only: + --volume FILE another part of a split archive (repeat for each part) + --max-size SIZE stop past SIZE decoded bytes in all (default 64G) + --max-memory SIZE largest member or container held in memory whole (1G) + --max-members N stop past N members (default 1000000) + --help, --version +SIZE takes a K, M, G or T suffix (powers of 1024)."; + +/// unzip's exit statuses, as far as they apply. +mod status { + pub const OK: u8 = 0; + pub const WARNING: u8 = 1; + pub const ERROR: u8 = 2; + pub const NO_PASSWORD: u8 = 5; + pub const NO_ARCHIVE: u8 = 9; + pub const BAD_OPTIONS: u8 = 10; + pub const NO_MATCH: u8 = 11; + pub const DISK_FULL: u8 = 50; + pub const UNSUPPORTED: u8 = 81; + pub const BAD_PASSWORD: u8 = 82; +} + +#[derive(Clone, Copy, PartialEq)] +enum Mode { + Extract, + List, + Test, + /// `-p`: the members' bytes to stdout. + Pipe, + /// `-c`: as `-p`, with messages. + Show, + /// `-Z1`: names only. + Names, +} + +struct Opts { + mode: Mode, + archive: String, + members: Vec, + excludes: Vec, + dir: Option, + /// `Some(true)` for `-o`, `Some(false)` for `-n`, `None` to ask. + overwrite: Option, + quiet: u8, + passwords: Vec, + junk: bool, + nocase: bool, + /// `-D` count: 1 keeps directory times off, 2 all times. + no_times: u8, + volumes: Vec, + limits: Limits, +} + +enum Parsed { + Usage(u8), + Version, + Run(Box), +} fn main() -> ExitCode { - let args: Vec = std::env::args().collect(); - let (extract_mode, archive, outdir) = match args.get(1).map(String::as_str) { - Some("list") if args.len() == 3 => (false, &args[2], None), - Some("extract") if args.len() == 3 || args.len() == 4 => (true, &args[2], args.get(3)), - _ => { - eprintln!( - "usage:\n exav-unpack list \n exav-unpack extract [output-dir]" - ); - return ExitCode::from(2); + let args: Vec = std::env::args().skip(1).collect(); + match parse(&args) { + Ok(Parsed::Usage(code)) => { + match code { + status::OK => println!("{USAGE}"), + _ => eprintln!("{USAGE}"), + } + ExitCode::from(code) } - }; - - let data = match read_archive(archive) { - Ok(d) => d, + Ok(Parsed::Version) => { + println!("exav-unpack {}", env!("CARGO_PKG_VERSION")); + ExitCode::SUCCESS + } + Ok(Parsed::Run(o)) => ExitCode::from(run(&o)), Err(e) => { - eprintln!("exav-unpack: {archive}: {e}"); - return ExitCode::FAILURE; + eprintln!("exav-unpack: {e}\n\n{USAGE}"); + ExitCode::from(status::BAD_OPTIONS) } - }; + } +} - // A packed executable is not a container by magic: "this is a PE" says - // nothing about whether the program on disk is the program that runs. The - // scanner routes those by content, and so does this, or `extract` on a - // packed dropper reports "unrecognised" for a file that unpacks fine. - let packed = || { - if exav_unpack::is_upx(&data) { - Some(Format::Upx) - } else if exav_unpack::is_pepack(&data) { - Some(Format::PePacked) - } else { - None +/// unzip's options that this subset refuses, with what they do. +const UNSUPPORTED: &[(char, &str)] = &[ + ('f', "freshen"), + ('u', "update"), + ('z', "archive comment"), + ('T', "archive timestamp"), + ('a', "text conversion"), + ('b', "binary/text mode"), + ('B', "backups"), + ('E', "MacOS extra field"), + ('F', "Acorn filetypes"), + ('i', "MacOS names"), + ('J', "file attributes"), + ('K', "setuid/setgid bits"), + ('L', "lowercase names"), + ('M', "pager"), + ('N', "Amiga filenotes"), + ('s', "spaces to underscores"), + ('S', "VMS Stream_LF"), + ('U', "Unicode escapes"), + ('V', "VMS version numbers"), + ('W', "wildcard stop at directories"), + ('X', "owners and ACLs"), + ('Y', "VMS versions"), + ('$', "volume labels"), + ('/', "Acorn extension list"), + (':', "paths outside the extraction directory"), + ('^', "control characters in names"), + ('2', "ODS names"), + ('A', "DLL help"), +]; + +fn parse(args: &[String]) -> Result { + if args.is_empty() { + return Ok(Parsed::Usage(status::OK)); + } + // An extractor writes to disk rather than holding what it decodes, so its + // totals can be far above the scanner's. The compression-ratio cap still + // stops a bomb long before them. + let mut limits = Limits::default(); + limits.max_extracted_bytes = 64 << 30; + limits.max_scanned_bytes = 64 << 30; + limits.max_buffer_bytes = 1 << 30; + limits.max_members = 1_000_000; + let mut o = Opts { + mode: Mode::Extract, + archive: String::new(), + members: Vec::new(), + excludes: Vec::new(), + dir: None, + overwrite: None, + quiet: 0, + passwords: Vec::new(), + junk: false, + nocase: false, + no_times: 0, + volumes: Vec::new(), + limits, + }; + let (mut modes, mut verbose, mut archive, mut excluding) = (Vec::new(), false, None, false); + let mut i = 0; + let next = |i: &mut usize, what: &str| -> Result { + *i += 1; + args.get(*i) + .cloned() + .ok_or_else(|| format!("{what} needs a value")) + }; + while i < args.len() { + let a = &args[i]; + // The long options are exav-unpack's own; unzip has none, so they + // are read anywhere. + if let Some(long) = a.strip_prefix("--").filter(|l| !l.is_empty()) { + let (name, inline) = match long.split_once('=') { + Some((n, v)) => (n, Some(v.to_string())), + None => (long, None), + }; + let value = |i: &mut usize| inline.clone().map_or_else(|| next(i, a), Ok); + match name { + "help" => return Ok(Parsed::Usage(status::OK)), + "version" => return Ok(Parsed::Version), + "volume" => o.volumes.push(PathBuf::from(value(&mut i)?)), + "max-size" => { + let n = size(&value(&mut i)?)?; + o.limits.max_extracted_bytes = n; + o.limits.max_scanned_bytes = n; + } + "max-memory" => o.limits.max_buffer_bytes = size(&value(&mut i)?)?, + "max-members" => { + o.limits.max_members = value(&mut i)? + .parse() + .map_err(|_| format!("{a} takes a number"))? + } + _ => return Err(format!("unknown option `{a}`")), + } + i += 1; + continue; + } + let option = a.len() > 1 && a.starts_with('-'); + // Past the archive, only `-x` and `-d` are options: anything else is + // a member name, as unzip has it. + if !option || (archive.is_some() && a != "-x" && !a.starts_with("-d")) { + match (&archive, excluding) { + (None, false) => archive = Some(a.clone()), + (_, true) => o.excludes.push(a.clone()), + (Some(_), false) => o.members.push(a.clone()), + } + i += 1; + continue; + } + excluding = false; + let cluster: Vec = a[1..].chars().collect(); + if cluster[0] == 'Z' { + match (i, &cluster[1..]) { + (0, ['1']) => modes.push(Mode::Names), + (0, _) => return Err("-Z is supported as -Z1 only".to_string()), + _ => return Err("-Z must be the first option".to_string()), + } + i += 1; + continue; } + let mut k = 0; + while k < cluster.len() { + let c = cluster[k]; + // The rest of the cluster, or the next argument. + let mut operand = |i: &mut usize| -> Result { + let rest: String = cluster[k + 1..].iter().collect(); + k = cluster.len(); + match rest.is_empty() { + true => next(i, &format!("-{c}")), + false => Ok(rest), + } + }; + match c { + 'l' => modes.push(Mode::List), + 't' => modes.push(Mode::Test), + 'p' => modes.push(Mode::Pipe), + 'c' => modes.push(Mode::Show), + 'v' => verbose = true, + 'o' => o.overwrite = Some(true), + 'n' => o.overwrite = Some(false), + 'q' => o.quiet += 1, + 'j' => o.junk = true, + 'C' => o.nocase = true, + 'D' => o.no_times += 1, + 'h' => return Ok(Parsed::Usage(status::OK)), + 'd' => o.dir = Some(PathBuf::from(operand(&mut i)?)), + 'P' => o.passwords.push(operand(&mut i)?), + 'x' => excluding = true, + _ => { + return Err(match UNSUPPORTED.iter().find(|u| u.0 == c) { + Some((_, what)) => { + format!("-{c} ({what}) is an unzip option exav-unpack does not take") + } + None => format!("unknown option -{c}"), + }) + } + } + k += 1; + } + i += 1; + } + if verbose { + // unzip's `-v` alone shows its version; with an archive it is the + // verbose listing, which this subset does not have. + return match (&archive, modes.is_empty()) { + (None, true) => Ok(Parsed::Version), + _ => { + Err("-v (verbose listing) is an unzip option exav-unpack does not take".to_string()) + } + }; + } + if modes.len() > 1 { + return Err("any combination of -c, -l, -p, -t and -Z1 is invalid".to_string()); + } + o.mode = modes.pop().unwrap_or(Mode::Extract); + let Some(archive) = archive else { + return Ok(Parsed::Usage(status::BAD_OPTIONS)); }; - let Some(fmt) = detect(&data).or_else(packed) else { - eprintln!("exav-unpack: {archive}: unrecognised archive format"); - return ExitCode::FAILURE; + o.archive = archive; + Ok(Parsed::Run(Box::new(o))) +} + +/// `n`, `nK`, `nM`, `nG` or `nT`, in bytes. +fn size(s: &str) -> Result { + let bad = || format!("`{s}` is not a size"); + let (digits, shift) = match s.char_indices().last() { + Some((i, c)) if c.is_ascii_alphabetic() => { + let shift = match c.to_ascii_uppercase() { + 'K' => 10, + 'M' => 20, + 'G' => 30, + 'T' => 40, + _ => return Err(bad()), + }; + (&s[..i], shift) + } + _ => (s, 0), }; - eprintln!("format: {fmt:?}"); + let n: u64 = digits.parse().map_err(|_| bad())?; + n.checked_mul(1 << shift).ok_or_else(bad) +} + +/// How the archives went, for unzip's closing tally. +#[derive(Default)] +struct Tally { + ok: u32, + warned: u32, + failed: u32, + worst: u8, +} - // Generous limits for an interactive extractor (the scanner uses tighter - // ones); still bounded to contain decompression bombs. - let mut budget = Budget::new(Limits::default()); - let entries = match extract(fmt, &data, &mut budget) { - Ok(e) => e, +fn run(o: &Opts) -> u8 { + let archives = match archives(&o.archive) { + Ok(a) => a, Err(e) => { - eprintln!("exav-unpack: extraction stopped: {e}"); - return ExitCode::FAILURE; + eprintln!("exav-unpack: {e}"); + return status::NO_ARCHIVE; } }; - - if !extract_mode { - println!("{} member(s):", entries.len()); - for e in &entries { - println!("{:>12} {}", e.data.len(), e.name); + if archives.len() > 1 && !o.volumes.is_empty() { + eprintln!( + "exav-unpack: --volume names the parts of one archive, not of {}", + o.archive + ); + return status::BAD_OPTIONS; + } + if o.dir.is_some() && matches!(o.mode, Mode::List | Mode::Test | Mode::Names) { + eprintln!("caution: not extracting; -d ignored"); + } + let mut prompt = Prompt::default(); + let mut t = Tally::default(); + for (n, path) in archives.iter().enumerate() { + if n > 0 && o.quiet == 0 && !matches!(o.mode, Mode::Pipe | Mode::Names) { + println!(); + } + let s = Archive::new(o, path, &mut prompt).run(); + match s { + status::OK => t.ok += 1, + status::WARNING => t.warned += 1, + _ => t.failed += 1, } - return ExitCode::SUCCESS; + t.worst = t.worst.max(s); } + if archives.len() > 1 && o.quiet == 0 && !matches!(o.mode, Mode::Pipe | Mode::Names) { + eprintln!(); + let line = |n: u32, one: &str, many: &str| match n { + 1 => eprintln!("1 archive {one}"), + n => eprintln!("{n} archives {many}"), + }; + line( + t.ok, + "was successfully processed.", + "were successfully processed.", + ); + if t.warned > 0 { + line( + t.warned, + "had warnings but no fatal errors.", + "had warnings but no fatal errors.", + ); + } + if t.failed > 0 { + line(t.failed, "had fatal errors.", "had fatal errors."); + } + } + t.worst +} - let root = PathBuf::from(outdir.map(String::as_str).unwrap_or(".")); - let mut failures = 0u32; - for e in &entries { - // Reject path traversal / absolute paths from hostile member names: keep - // only normal path components, dropping `..`, root, and prefixes. - let Some(rel) = safe_relative_path(&e.name) else { - eprintln!("skipping unsafe member name: {:?}", e.name); - failures += 1; - continue; +/// The archives `spec` names: itself, or failing that with `.zip` or `.ZIP` +/// added, as unzip tries them; or every file a wildcard matches. +fn archives(spec: &str) -> Result, String> { + let missing = || format!("cannot find or open {spec}, {spec}.zip or {spec}.ZIP."); + let path = Path::new(spec); + let name = path.file_name().and_then(|n| n.to_str()).unwrap_or(spec); + if name.contains(['*', '?', '[']) { + let dir = path + .parent() + .filter(|d| !d.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let mut found: Vec = fs::read_dir(dir) + .map_err(|_| missing())? + .flatten() + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| matches(name, n, false)) + }) + .map( + |e| match path.parent().filter(|d| !d.as_os_str().is_empty()) { + Some(_) => dir.join(e.file_name()), + None => PathBuf::from(e.file_name()), + }, + ) + .collect(); + found.sort(); + return match found.is_empty() { + true => Err(missing()), + false => Ok(found), }; - let path = root.join(rel); - if let Some(parent) = path.parent() { - if let Err(err) = std::fs::create_dir_all(parent) { - eprintln!("exav-unpack: mkdir {}: {err}", parent.display()); - failures += 1; + } + [ + spec.to_string(), + format!("{spec}.zip"), + format!("{spec}.ZIP"), + ] + .into_iter() + .map(PathBuf::from) + .find(|p| p.is_file()) + .map(|p| vec![p]) + .ok_or_else(missing) +} + +/// Whether `name` matches unzip's wildcard `pattern`: `*` any run of +/// characters, `/` included; `?` any one; `[...]` one of a set, `[!...]` one +/// outside it, with ranges. +fn matches(pattern: &str, name: &str, nocase: bool) -> bool { + let fold = |c: char| if nocase { c.to_ascii_lowercase() } else { c }; + let p: Vec = pattern.chars().map(fold).collect(); + let n: Vec = name.chars().map(fold).collect(); + // Classic backtracking on the last `*`: linear in practice. + let (mut pi, mut ni, mut star, mut mark) = (0, 0, None, 0); + while ni < n.len() { + let step = match p.get(pi) { + Some('*') => { + star = Some(pi); + mark = ni; + pi += 1; continue; } + Some('?') => Some(pi + 1), + Some('[') => class(&p, pi, n[ni]), + Some(&c) if c == n[ni] => Some(pi + 1), + _ => None, + }; + match (step, star) { + (Some(next), _) => { + pi = next; + ni += 1; + } + (None, Some(s)) => { + pi = s + 1; + mark += 1; + ni = mark; + } + (None, None) => return false, + } + } + p[pi..].iter().all(|&c| c == '*') +} + +/// Where the pattern goes on when the set at `p[at] == '['` takes `c`. +fn class(p: &[char], at: usize, c: char) -> Option { + let mut i = at + 1; + let negate = matches!(p.get(i), Some('!') | Some('^')); + if negate { + i += 1; + } + let mut hit = false; + let mut first = true; + while let Some(&x) = p.get(i) { + if x == ']' && !first { + return (hit != negate).then_some(i + 1); + } + first = false; + if p.get(i + 1) == Some(&'-') && p.get(i + 2).is_some_and(|&y| y != ']') { + hit |= (x..=p[i + 2]).contains(&c); + i += 3; + } else { + hit |= x == c; + i += 1; + } + } + // No closing `]`: a literal `[`. + (c == '[').then_some(at + 1) +} + +/// The answers to "replace?" that hold for the rest of the run. +#[derive(Default)] +struct Prompt { + all: Option, +} + +/// A link made once every member is out, so no member is written through it. +struct Deferred { + path: PathBuf, + shown: String, + target: String, +} + +/// One archive's run. +struct Archive<'a> { + o: &'a Opts, + path: &'a Path, + prompt: &'a mut Prompt, + status: u8, + /// Which of the include and exclude patterns matched something. + matched: Vec, + excluded: Vec, + files: u64, + bytes: u64, + links: Vec, + /// Directories whose times and modes are set last, once their contents + /// are written. + dirs: Vec<(PathBuf, Option, Option)>, + buf: Vec, + tested_bad: bool, + /// Members out (or tested) whole, and those skipped for a wrong password. + ok: u64, + bad_password: u64, +} + +impl<'a> Archive<'a> { + fn new(o: &'a Opts, path: &'a Path, prompt: &'a mut Prompt) -> Self { + Archive { + o, + path, + prompt, + status: status::OK, + matched: vec![false; o.members.len()], + excluded: vec![false; o.excludes.len()], + files: 0, + bytes: 0, + links: Vec::new(), + dirs: Vec::new(), + buf: vec![0; 1 << 20], + tested_bad: false, + ok: 0, + bad_password: 0, + } + } + + fn raise(&mut self, s: u8) { + self.status = self.status.max(s); + } + + fn run(mut self) -> u8 { + let o = self.o; + let messages = o.quiet == 0 && !matches!(o.mode, Mode::Pipe | Mode::Names); + if messages && !(o.mode == Mode::Test && o.quiet > 0) { + println!("Archive: {}", self.path.display()); + } + let src = match open(self.path, &o.volumes, o.limits.max_buffer_bytes) { + Ok(s) => s, + Err(e) => { + eprintln!("exav-unpack: cannot read {}: {e}", self.path.display()); + return status::NO_ARCHIVE; + } + }; + let Some(fmt) = detect(&*src).or_else(|| packed(&*src, o.limits.max_buffer_bytes)) else { + eprintln!( + "exav-unpack: {} is not an archive exav-unpack reads.", + self.path.display() + ); + return status::NO_ARCHIVE; + }; + if o.mode == Mode::List && o.quiet < 2 { + println!(" Length Date Time Name"); + println!("--------- ---------- ----- ----"); } - match std::fs::write(&path, &e.data) { - Ok(()) => println!("{} ({} bytes)", path.display(), e.data.len()), - Err(err) => { - eprintln!("exav-unpack: write {}: {err}", path.display()); - failures += 1; + let root = o.dir.clone().unwrap_or_else(|| PathBuf::from(".")); + if o.mode == Mode::Extract { + if let Err(e) = fs::create_dir_all(&root) { + eprintln!("exav-unpack: cannot create {}: {e}", root.display()); + return status::ERROR; } } + let mut budget = Budget::with_passwords(o.limits.clone(), o.passwords.clone()); + budget.set_visit_directories(true); + let stopped = walk(fmt, &*src, &mut budget, &mut |meta, content, _| { + self.member(&root, meta, content).err() + }); + match stopped { + Ok(None) => {} + Ok(Some(fatal)) => { + eprintln!("exav-unpack: {fatal}"); + self.raise(status::DISK_FULL); + } + Err(hit) => { + eprintln!(" error: {hit}"); + self.raise(status::ERROR); + } + } + self.finish(&root, messages); + self.status } - if failures > 0 { - ExitCode::FAILURE + + /// The links, directory times, the listing's totals, the test's verdict + /// and the patterns nothing matched. + fn finish(&mut self, root: &Path, messages: bool) { + let o = self.o; + if !self.links.is_empty() { + if messages { + println!("finishing deferred symbolic links:"); + } + for l in std::mem::take(&mut self.links) { + if messages { + println!(" {:<22} -> {}", l.shown, l.target); + } + if let Err(e) = make_link(root, &l.path, &l.target) { + eprintln!(" skipping: {:<22} {e}", l.shown); + self.raise(status::WARNING); + } + } + } + // Deepest first, so setting a directory's time is not undone by a + // write into it. + let mut dirs = std::mem::take(&mut self.dirs); + dirs.sort_by_key(|d| std::cmp::Reverse(d.0.components().count())); + for (path, mtime, mode) in dirs { + set_mode(&path, mode); + if o.no_times == 0 { + set_time(&path, mtime); + } + } + match o.mode { + Mode::List if o.quiet < 2 => { + println!("--------- -------"); + let s = if self.files == 1 { "" } else { "s" }; + println!( + "{:>9} {} file{s}", + self.bytes, self.files + ); + } + Mode::Test if o.quiet < 2 => { + let plural = |n: u64| if n == 1 { "file" } else { "files" }; + match (self.tested_bad, self.bad_password, self.ok) { + (true, ..) => println!( + "At least one error was detected in {}.", + self.path.display() + ), + (false, 0, _) => println!( + "No errors detected in compressed data of {}.", + self.path.display() + ), + (false, _, 0) => { + println!("Caution: zero files tested in {}.", self.path.display()) + } + (false, _, n) => println!( + "No errors detected in {} for the {n} {} tested.", + self.path.display(), + plural(n) + ), + } + if self.bad_password > 0 { + let n = self.bad_password; + println!("{n} {} skipped because of incorrect password.", plural(n)); + } + } + _ => {} + } + // As unzip has it: a wrong password is an error only when it left + // nothing to extract or test. + if self.bad_password > 0 { + self.raise(if self.ok == 0 { + status::BAD_PASSWORD + } else { + status::WARNING + }); + } + for (p, hit) in o.members.iter().zip(&self.matched) { + if !hit { + eprintln!("caution: filename not matched: {p}"); + self.status = self.status.max(status::NO_MATCH); + } + } + for (p, hit) in o.excludes.iter().zip(&self.excluded) { + if !hit { + eprintln!("caution: excluded filename not matched: {p}"); + } + } + } + + /// Whether member `name` is one the command line asks for. + fn wanted(&mut self, name: &str) -> bool { + let o = self.o; + let mut take = o.members.is_empty(); + for (p, hit) in o.members.iter().zip(&mut self.matched) { + if matches(p, name, o.nocase) { + *hit = true; + take = true; + } + } + for (p, hit) in o.excludes.iter().zip(&mut self.excluded) { + if matches(p, name, o.nocase) { + *hit = true; + take = false; + } + } + take + } + + /// List, test, print or write one member. `Err` ends the walk. + fn member( + &mut self, + root: &Path, + meta: &MemberMeta, + content: Option>, + ) -> Result<(), String> { + let o = self.o; + let name = meta.name.as_str(); + if !self.wanted(name) { + return Ok(()); + } + let kind = meta.mode.map(|m| m & 0o170000); + let dir = kind == Some(0o040000) + || ((name.ends_with('/') || name.ends_with('\\')) && meta.size.unwrap_or(0) == 0); + let link = kind == Some(0o120000); + let messages = o.quiet == 0; + match o.mode { + Mode::Names => { + println!("{name}"); + return Ok(()); + } + Mode::List => { + let size = match content { + Some(Member::Bytes(b)) => b.len() as u64, + // A size the archive declares is listed as declared, + // without decoding the member to measure it. + Some(Member::Stream(r)) => meta + .size + .unwrap_or_else(|| copy(r, &mut io::sink(), &mut self.buf).unwrap_or(0)), + None => meta.size.unwrap_or(0), + }; + self.files += 1; + self.bytes += size; + println!("{size:>9} {} {name}", when(meta.mtime)); + return Ok(()); + } + _ => {} + } + if dir { + match o.mode { + Mode::Test if messages => println!(" testing: {name:<22} OK"), + Mode::Extract if !o.junk => { + let Some(path) = target(root, name) else { + return self.unsafe_name(name); + }; + let shown = format!("{}/", shown(o, &path)); + let made = !path.is_dir(); + if let Err(e) = directories(root, &path, true) { + eprintln!("exav-unpack: {e}"); + self.raise(status::ERROR); + return Ok(()); + } + if messages && made { + println!(" creating: {shown}"); + } + self.dirs.push((path, meta.mtime, meta.mode)); + } + _ => {} + } + self.ok += 1; + return Ok(()); + } + // Not decodable: say why, as unzip's `skipping:` does, or as its test + // reports damage. + let Some(content) = content else { + let why = meta.unsupported.unwrap_or("no content"); + let unsupported = ["unsupported", "method", "codec"] + .iter() + .any(|w| why.contains(w)); + let (why, s) = match (meta.encrypted, o.passwords.is_empty()) { + (true, true) => ("unable to get password", status::NO_PASSWORD), + (true, false) => ("incorrect password", status::BAD_PASSWORD), + _ if unsupported => (why, status::UNSUPPORTED), + _ => (why, status::ERROR), + }; + match s { + // Settled in `finish`, once it is known what else came out. + status::BAD_PASSWORD => self.bad_password += 1, + _ => { + self.tested_bad = true; + self.raise(s); + } + } + match (o.mode, s) { + (Mode::Test, status::ERROR) => { + println!(" testing: {name:<22} "); + println!(" error: {why}"); + } + (Mode::Test, _) => println!(" skipping: {name:<22} {why}"), + _ => eprintln!(" skipping: {name:<22} {why}"), + } + return Ok(()); + }; + match o.mode { + Mode::Test => { + let got = self.decode(content, &mut io::sink()); + let damage = got.err().or_else(|| meta.unsupported.map(str::to_string)); + match damage { + None => { + if messages { + println!(" testing: {name:<22} OK"); + } + self.ok += 1; + } + Some(e) => { + println!(" testing: {name:<22} "); + println!(" error: {e}"); + self.tested_bad = true; + self.raise(status::ERROR); + } + } + Ok(()) + } + Mode::Pipe | Mode::Show => { + if o.mode == Mode::Show && messages { + println!(" extracting: {name:<22} "); + } + let mut out = io::stdout().lock(); + let got = self.decode(content, &mut out); + if o.mode == Mode::Show { + let _ = writeln!(out); + } + drop(out); + match got.and_then(|_| meta.unsupported.map_or(Ok(()), |u| Err(u.to_string()))) { + Ok(()) => self.ok += 1, + Err(e) => { + eprintln!(" error: {e}"); + self.raise(status::ERROR); + } + } + Ok(()) + } + _ => self.extract(root, meta, content, link), + } + } + + fn unsafe_name(&mut self, name: &str) -> Result<(), String> { + eprintln!(" skipping: {name:<22} unsafe member name"); + self.raise(status::WARNING); + Ok(()) + } + + /// Decode `content` into `w`; `Err` names what went wrong part way. + fn decode(&mut self, content: Member<'_>, w: &mut dyn Write) -> Result { + match content { + Member::Bytes(b) => w + .write_all(&b) + .map(|_| b.len() as u64) + .map_err(|e| e.to_string()), + Member::Stream(r) => copy(r, w, &mut self.buf).map_err(|e| match e { + Copy::Write(e) => e.to_string(), + Copy::Read(e) => e.to_string(), + }), + } + } + + /// Write one member to disk. + fn extract( + &mut self, + root: &Path, + meta: &MemberMeta, + content: Member<'_>, + link: bool, + ) -> Result<(), String> { + let o = self.o; + let name = meta.name.as_str(); + let rel = match o.junk { + true => name.rsplit(['/', '\\']).next().unwrap_or(name), + false => name, + }; + let Some(mut path) = target(root, rel) else { + return self.unsafe_name(name); + }; + if link { + let target = match (&meta.link, content) { + (Some(t), _) => t.clone(), + (None, c) => { + let mut t = Vec::new(); + let _ = self.decode(c, &mut t); + String::from_utf8_lossy(&t).into_owned() + } + }; + if let Ok(m) = fs::symlink_metadata(&path) { + match (o.overwrite, m.file_type().is_symlink()) { + (Some(true), _) => {} + (_, true) => { + println!("{} exists and is a symbolic link.", shown(o, &path)); + return Ok(()); + } + (_, false) => match self.replace(&path) { + Replace::Yes => {} + Replace::No => return Ok(()), + Replace::Rename(p) => path = p, + }, + } + } + let shown = shown(o, &path); + if o.quiet == 0 { + println!(" linking: {shown:<22} -> {target} "); + } + self.links.push(Deferred { + path, + shown, + target, + }); + self.ok += 1; + return Ok(()); + } + if let Err(e) = directories(root, &path, false) { + eprintln!("exav-unpack: {e}"); + self.raise(status::ERROR); + return Ok(()); + } + if fs::symlink_metadata(&path).is_ok() { + match self.replace(&path) { + Replace::Yes => {} + Replace::No => return Ok(()), + Replace::Rename(p) => path = p, + } + } + let shown = shown(o, &path); + // A link or a file in the way is removed, never written through. + if fs::symlink_metadata(&path).is_ok_and(|m| !m.is_dir()) { + let _ = fs::remove_file(&path); + } + let mut file = match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(f) => f, + Err(e) => { + eprintln!("exav-unpack: cannot create {shown}: {e}"); + self.raise(status::ERROR); + return Ok(()); + } + }; + // As unzip has it, the line is ended once the member is out, and left + // open when the error that stopped it goes to stderr. + if o.quiet == 0 { + print!("{:>11}: {shown:<22} ", verb(meta)); + let _ = io::stdout().flush(); + } + let got = match content { + Member::Bytes(b) => file.write_all(&b).map_err(Copy::Write), + Member::Stream(r) => copy(r, &mut file, &mut self.buf).map(|_| ()), + }; + let error = match got { + Ok(()) => meta.unsupported.map(str::to_string), + Err(Copy::Write(e)) => return Err(format!("cannot write {shown}: {e}")), + Err(Copy::Read(e)) => Some(e.to_string()), + }; + match error { + None => { + if o.quiet == 0 { + println!(); + } + self.ok += 1; + } + Some(e) => { + eprintln!("\n error: {e}"); + self.raise(status::ERROR); + } + } + drop(file); + set_mode(&path, meta.mode); + if o.no_times < 2 { + set_time(&path, meta.mtime); + } + Ok(()) + } + + /// What to do about a file already at `path`: `-o`, `-n`, or ask, as unzip + /// does, on stdin. + fn replace(&mut self, path: &Path) -> Replace { + if let Some(yes) = self.o.overwrite.or(self.prompt.all) { + return if yes { Replace::Yes } else { Replace::No }; + } + let shown = shown(self.o, path); + loop { + eprint!("replace {shown}? [y]es, [n]o, [A]ll, [N]one, [r]ename: "); + let Some(answer) = read_line() else { + eprintln!(" NULL\n(EOF or read error, treating as \"[N]one\" ...)"); + self.prompt.all = Some(false); + self.raise(status::WARNING); + return Replace::No; + }; + match answer.trim() { + "y" | "Y" => return Replace::Yes, + "n" => return Replace::No, + "A" => { + self.prompt.all = Some(true); + return Replace::Yes; + } + "N" => { + self.prompt.all = Some(false); + return Replace::No; + } + "r" | "R" => { + eprint!("new name: "); + match read_line() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + { + Some(new) => return Replace::Rename(PathBuf::from(new)), + None => return Replace::No, + } + } + _ => eprintln!("error: invalid response [{}]", answer.trim()), + } + } + } +} + +enum Replace { + Yes, + No, + Rename(PathBuf), +} + +/// unzip's word for writing out a member, by its ZIP method; `extracting` +/// for any other. +fn verb(meta: &MemberMeta) -> &'static str { + match meta.zip_method { + Some(8 | 9) => "inflating", + Some(12) => "bunzipping", + Some(6) => "exploding", + Some(1) => "unshrinking", + Some(2..=5) => "unreducing", + _ => "extracting", + } +} + +fn read_line() -> Option { + let mut s = String::new(); + match io::stdin().lock().read_line(&mut s) { + Ok(0) | Err(_) => None, + Ok(_) => Some(s), + } +} + +/// `path` as unzip prints it: under `-d DIR`, `DIR/name`; else the name. +fn shown(o: &Opts, path: &Path) -> String { + match &o.dir { + Some(_) => path.display().to_string(), + None => path.strip_prefix(".").unwrap_or(path).display().to_string(), + } +} + +/// A member's time as `-l` lists it, in local time. +fn when(m: Option) -> String { + match m { + Some(Mtime::Local { + year, + month, + day, + hour, + minute, + .. + }) => { + format!("{year:04}-{month:02}-{day:02} {hour:02}:{minute:02}") + } + Some(Mtime::Unix(s)) => match Local.timestamp_opt(s, 0) { + LocalResult::Single(t) | LocalResult::Ambiguous(t, _) => { + t.format("%Y-%m-%d %H:%M").to_string() + } + LocalResult::None => " ".repeat(16), + }, + None => " ".repeat(16), + } +} + +/// A member's time as an instant: a DOS time is read in the local zone, as +/// unzip reads it. +fn instant(m: Mtime) -> Option { + let secs = match m { + Mtime::Unix(s) => s, + Mtime::Local { + year, + month, + day, + hour, + minute, + second, + } => Local + .with_ymd_and_hms( + year.into(), + month.into(), + day.into(), + hour.into(), + minute.into(), + second.into(), + ) + .earliest()? + .timestamp(), + }; + match secs >= 0 { + true => SystemTime::UNIX_EPOCH.checked_add(Duration::from_secs(secs as u64)), + false => SystemTime::UNIX_EPOCH.checked_sub(Duration::from_secs(secs.unsigned_abs())), + } +} + +fn set_time(path: &Path, m: Option) { + let Some(t) = m.and_then(instant) else { return }; + // Opening a directory to set its time works on Unix only. + if let Ok(f) = File::open(path) { + let _ = f.set_modified(t); + } +} + +/// The permission bits of a Unix `mode`, without setuid, setgid or sticky: +/// unzip keeps those only with `-K`. +fn set_mode(path: &Path, mode: Option) { + #[cfg(unix)] + if let Some(m) = mode { + use std::os::unix::fs::PermissionsExt; + let _ = fs::set_permissions(path, fs::Permissions::from_mode(m & 0o777)); + } + #[cfg(not(unix))] + let _ = (path, mode); +} + +/// Whether a link at `path` to `target` leads out of `root`, by the words of +/// the target alone: a `..` past the root, or a root of its own. +fn escapes(root: &Path, path: &Path, target: &str) -> bool { + let parent = path.parent().unwrap_or(root); + let mut depth = parent + .strip_prefix(root) + .map_or(0, |r| r.components().count()) as isize; + for c in Path::new(target).components() { + match c { + Component::Normal(_) => depth += 1, + Component::ParentDir => depth -= 1, + Component::CurDir => {} + Component::RootDir | Component::Prefix(_) => return true, + } + if depth < 0 { + return true; + } + } + false +} + +/// Make the link at `path` to `target`, if the target stays under `root`. +fn make_link(root: &Path, path: &Path, target: &str) -> Result<(), String> { + if escapes(root, path, target) { + return Err("symbolic link points outside the extraction directory".to_string()); + } + directories(root, path, false)?; + if fs::symlink_metadata(path).is_ok() { + fs::remove_file(path).map_err(|e| e.to_string())?; + } + #[cfg(unix)] + return std::os::unix::fs::symlink(target, path).map_err(|e| e.to_string()); + // Where links take privileges to make, unzip writes the target as text. + #[cfg(not(unix))] + fs::write(path, target).map_err(|e| e.to_string()) +} + +/// Why a copy stopped: a failed write, or a failed read. +enum Copy { + Write(io::Error), + Read(io::Error), +} + +/// Copy `r` into `w` through `buf`, returning the bytes copied. +fn copy(r: &mut dyn Read, w: &mut dyn Write, buf: &mut [u8]) -> Result { + let mut n = 0u64; + loop { + match r.read(buf) { + Ok(0) => return Ok(n), + Ok(k) => { + w.write_all(&buf[..k]).map_err(Copy::Write)?; + n += k as u64; + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(Copy::Read(e)), + } + } +} + +/// Where member `name` goes under `root`: its plain components only, so no +/// name can climb out (`..`), restart at a root (`/`, `C:`) or be empty. +fn target(root: &Path, name: &str) -> Option { + let mut rel = PathBuf::new(); + for c in name.split(['/', '\\']) { + let drive = c.len() == 2 && c.ends_with(':'); + if c.is_empty() || c == "." || c == ".." || drive { + continue; + } + rel.push(file_name(c)); + } + (!rel.as_os_str().is_empty()).then(|| root.join(rel)) +} + +/// `c` as a file name the platform takes: no NUL or control character, and +/// on Windows none of the characters it reserves. +fn file_name(c: &str) -> String { + c.chars() + .map(|ch| match ch { + c if c.is_control() => '_', + '<' | '>' | ':' | '"' | '|' | '?' | '*' if cfg!(windows) => '_', + c => c, + }) + .collect() +} + +/// Make the directories from `root` down to `path` (`path` itself too when +/// `last`), refusing to go through a link: one already in the output +/// directory must not take a member somewhere else. +fn directories(root: &Path, path: &Path, last: bool) -> Result<(), String> { + let rel = path.strip_prefix(root).unwrap_or(path); + let parts: Vec<_> = rel.components().collect(); + let upto = if last { + parts.len() } else { - ExitCode::SUCCESS - } -} - -/// Read an archive, rejoining it with its siblings when the path names one part -/// of a **byte-split set** (`big.7z.001`, `.002`, …). -/// -/// A single part is not an archive at all: the split was made at an arbitrary -/// byte offset, so `.002` has no header, no directory and nothing to detect. -/// Pointing this tool at any one part is the natural thing for a user to do, and -/// answering "unrecognised archive format" would be technically true and -/// useless. -/// -/// Sibling names are **generated from the parsed pattern** and looked up in the -/// same directory — never taken from anything a file contains — so no input can -/// steer which paths are opened. Volumes are read in order from the first until -/// one is missing; that is where the archive ends, since nothing in the naming -/// records how many parts a set has. -fn read_archive(path: &str) -> std::io::Result> { - let p = Path::new(path); - let name = p.file_name().and_then(|n| n.to_str()).unwrap_or_default(); - let Some(vol) = exav_unpack::volume::parse(name).filter(|v| v.scheme.is_byte_split()) else { - return std::fs::read(p) - .map_err(|e| std::io::Error::other(format!("cannot read {path}: {e}"))); + parts.len().saturating_sub(1) }; - let dir = p.parent().unwrap_or(Path::new(".")); - let mut data = Vec::new(); - let mut parts = 0usize; - for i in 0.. { - let Some(part) = vol.name_at(i) else { break }; - let Ok(bytes) = std::fs::read(dir.join(&part)) else { - break; - }; - data.extend_from_slice(&bytes); - parts += 1; + let mut at = root.to_path_buf(); + for c in &parts[..upto] { + at.push(c); + match fs::symlink_metadata(&at) { + Ok(m) if m.file_type().is_symlink() => { + return Err(format!("{} is a link, not followed", at.display())) + } + Ok(m) if m.is_dir() => {} + Ok(_) => return Err(format!("{} is a file, not a directory", at.display())), + Err(_) => { + fs::create_dir(&at).map_err(|e| format!("cannot create {}: {e}", at.display()))? + } + } } - if parts == 0 { - // The set does not start where it must: everything before the given part - // is missing, so the head of the archive is simply not here. - return Err(std::io::Error::other(format!( - "{name} is one part of a split archive, and its first volume \ - ({}) is not in the same directory", - vol.name_at(0).unwrap_or_default() - ))); + Ok(()) +} + +/// How the parts of a split archive make one. +#[derive(Clone, Copy, PartialEq)] +enum Set { + /// One file. + One, + /// A finished file cut into parts (`x.7z.001`, ...): end to end. + Bytes, + /// `x.z01`, ..., `x.zip`: end to end, the directory rebased. + Zip, + /// `x.part1.rar`, ... or `x.rar`, `x.r00`, ...: each member's parts joined. + Rar, +} + +fn volume_of(p: &Path) -> Option { + p.file_name() + .and_then(|n| n.to_str()) + .and_then(exav_unpack::volume::parse) +} + +/// The files of the archive at `path`, in order, and how they make one: +/// `volumes` with it when given, else the parts its name says are next to it. +/// Siblings are named from the pattern, never from anything a file holds, and +/// read in order until one is missing: nothing in the naming records how many +/// parts a set has. +fn parts(path: &Path, volumes: &[PathBuf]) -> io::Result<(Vec, Set)> { + use exav_unpack::volume::Scheme; + let dir = path + .parent() + .filter(|d| !d.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let is_zip = |p: &Path| p.extension().is_some_and(|e| e.eq_ignore_ascii_case("zip")); + let siblings = |v: &exav_unpack::volume::VolumeName, from: usize| -> Vec { + (from..) + .map_while(|i| v.name_at(i)) + .map(|n| dir.join(n)) + .take_while(|p| p.is_file()) + .collect() + }; + if !volumes.is_empty() { + let mut all: Vec = std::iter::once(path.to_path_buf()) + .chain(volumes.iter().cloned()) + .collect(); + let schemes: Vec<_> = all.iter().map(|p| volume_of(p).map(|v| v.scheme)).collect(); + let kind = if schemes.iter().any(|s| matches!(s, Some(Scheme::ZipSplit))) { + Set::Zip + } else if schemes + .iter() + .all(|s| matches!(s, Some(Scheme::RarPart { .. } | Scheme::RarOld))) + { + Set::Rar + } else { + match File::open(path).and_then(|mut f| { + let mut head = [0u8; 4]; + f.read_exact(&mut head).map(|_| head) + }) { + Ok(h) if &h == b"Rar!" => Set::Rar, + _ => Set::Bytes, + } + }; + // In the order of their volume numbers when their names all have + // them (a ZIP's `.zip` last), else as given. + let key = |p: &PathBuf| match volume_of(p) { + Some(v) => Some(v.index), + None if kind == Set::Zip && is_zip(p) => Some(usize::MAX), + None => None, + }; + if all.iter().all(|p| key(p).is_some()) { + all.sort_by_key(key); + } + return Ok((all, kind)); } - if parts > 1 { - eprintln!("joined {parts} volumes of a split archive"); + let missing = |what: String| { + io::Error::other(format!( + "one part of a split archive, whose {what} is not next to it" + )) + }; + match volume_of(path) { + Some(v) if v.scheme.is_byte_split() => { + let all = siblings(&v, 0); + match all.is_empty() { + true => Err(missing(format!( + "first part ({})", + v.name_at(0).unwrap_or_default() + ))), + false => Ok((all, Set::Bytes)), + } + } + Some(v) if v.scheme == Scheme::ZipSplit => { + let last = dir.join(format!("{}.zip", v.stem)); + let mut all = siblings(&v, 0); + match last.is_file() && !all.is_empty() { + true => { + all.push(last); + Ok((all, Set::Zip)) + } + false => Err(missing(format!("last part ({})", last.display()))), + } + } + Some(v) if matches!(v.scheme, Scheme::RarPart { .. } | Scheme::RarOld) => { + let all = siblings(&v, 0); + match all.len() { + 0 => Err(missing(format!( + "first part ({})", + v.name_at(0).unwrap_or_default() + ))), + 1 => Ok((all, Set::One)), + _ => Ok((all, Set::Rar)), + } + } + _ if is_zip(path) => { + // `x.zip` is the last part of a set when `x.z01` is next to it. + let stem = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or_default(); + match volume_of(&dir.join(format!("{stem}.z01"))) { + Some(v) if dir.join(format!("{stem}.z01")).is_file() => { + let mut all = siblings(&v, 0); + all.push(path.to_path_buf()); + Ok((all, Set::Zip)) + } + _ => Ok((vec![path.to_path_buf()], Set::One)), + } + } + _ => Ok((vec![path.to_path_buf()], Set::One)), } - Ok(data) } -/// Reduce a member name to a safe relative path under the output directory: -/// only `Normal` components are kept (no absolute root, `..`, or drive prefix). -/// Returns `None` if nothing usable remains. -fn safe_relative_path(name: &str) -> Option { - let mut out = PathBuf::new(); - for comp in Path::new(name).components() { - if let Component::Normal(c) = comp { - out.push(c); +/// The archive at `path`, with the other parts of its set, read from disk as +/// it is needed; a RAR set is joined in memory, up to `max` bytes. +fn open(path: &Path, volumes: &[PathBuf], max: u64) -> io::Result> { + let (files, kind) = parts(path, volumes)?; + let cached = |p: &PathBuf| -> io::Result> { + Ok(Box::new(BlockCache::new(File::open(p)?)?)) + }; + match kind { + Set::One => cached(&files[0]), + Set::Bytes => { + let mut parts = Vec::new(); + for p in &files { + let f = File::open(p)?; + let len = f.metadata()?.len(); + parts.push((f, len)); + } + Ok(Box::new(BlockCache::new(Joined::new(parts))?)) + } + Set::Zip => { + let parts = files.iter().map(cached).collect::>>()?; + Ok(Box::new( + exav_unpack::span::ZipSpan::new(parts).map_err(io::Error::other)?, + )) + } + Set::Rar => { + let total: u64 = files + .iter() + .map(|p| fs::metadata(p).map_or(0, |m| m.len())) + .sum(); + if total > max { + return Err(io::Error::other(format!( + "a RAR set of {total} bytes is joined in memory, past --max-memory {max}" + ))); + } + let data = files.iter().map(fs::read).collect::>>()?; + let refs: Vec<&[u8]> = data.iter().map(Vec::as_slice).collect(); + let joined = exav_unpack::join_rar_volumes(&refs).map_err(io::Error::other)?; + Ok(Box::new(joined)) } } - if out.as_os_str().is_empty() { - None +} + +/// The format of a packed executable, which `detect` does not report: read +/// whole, so only up to `max` bytes. +fn packed(src: &dyn ByteSource, max: u64) -> Option { + if src.len() as u64 > max { + return None; + } + let data = src.window(0, src.len()); + if exav_unpack::is_upx(&data) { + Some(Format::Upx) + } else if exav_unpack::is_pepack(&data) { + Some(Format::PePacked) } else { - Some(out) + None + } +} + +/// The parts of a byte-split set, read as one file. +struct Joined { + parts: Vec<(File, u64)>, + len: u64, + pos: u64, +} + +impl Joined { + fn new(parts: Vec<(File, u64)>) -> Self { + let len = parts.iter().map(|p| p.1).sum(); + Joined { parts, len, pos: 0 } + } +} + +impl Read for Joined { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let mut base = 0; + for (f, len) in &mut self.parts { + if self.pos < base + *len { + let off = self.pos - base; + f.seek(SeekFrom::Start(off))?; + let want = buf + .len() + .min(usize::try_from(*len - off).unwrap_or(usize::MAX)); + let n = f.read(&mut buf[..want])?; + self.pos += n as u64; + return Ok(n); + } + base += *len; + } + Ok(0) + } +} + +impl Seek for Joined { + fn seek(&mut self, to: SeekFrom) -> io::Result { + let new = match to { + SeekFrom::Start(n) => Some(n), + SeekFrom::End(d) => self.len.checked_add_signed(d), + SeekFrom::Current(d) => self.pos.checked_add_signed(d), + }; + self.pos = new + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "seek before the start"))?; + Ok(self.pos) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wildcards_match_as_unzip_does() { + for (p, n, want) in [ + ("*", "a/b/c.txt", true), + ("*.txt", "dir/x.txt", true), + ("dir/*", "dir/x/y", true), + ("?.txt", "a.txt", true), + ("?.txt", "ab.txt", false), + ("[ab]*", "bcd", true), + ("[!ab]*", "bcd", false), + ("[a-c]x", "bx", true), + ("[a-c]x", "dx", false), + ("one.txt", "one.txt", true), + ("one.txt", "one.txtx", false), + ("*a*b", "xaxxb", true), + ("[", "[", true), + ] { + assert_eq!(matches(p, n, false), want, "{p} {n}"); + } + assert!(matches("ONE.TXT", "one.txt", true)); + assert!(!matches("ONE.TXT", "one.txt", false)); + } + + #[test] + fn a_link_out_of_the_root_escapes() { + let root = Path::new("/r"); + assert!(!escapes(root, Path::new("/r/a/l"), "../b")); + assert!(!escapes(root, Path::new("/r/a/l"), "x/../../b")); + assert!(escapes(root, Path::new("/r/a/l"), "../../b")); + assert!(escapes(root, Path::new("/r/l"), "../l")); + assert!(escapes(root, Path::new("/r/l"), "/etc/passwd")); } } diff --git a/crates/exav-unpack/src/span.rs b/crates/exav-unpack/src/span.rs new file mode 100644 index 00000000..37c0a6c8 --- /dev/null +++ b/crates/exav-unpack/src/span.rs @@ -0,0 +1,386 @@ +//! A ZIP split across files, read as one archive. +//! +//! `zip -s` (and PKZIP spanning) writes `x.z01`, `x.z02`, ..., `x.zip`: one +//! ZIP stream cut into parts, the central directory in the last. Joined end to +//! end the parts are the original stream, except that the central directory +//! records where each member starts as a part number and an offset within +//! that part. [`ZipSpan`] reads the parts as one stream and overlays the +//! directory with the same records rewritten to offsets in the joined stream, +//! all on one part, which any ZIP reader then reads as a single archive. + +use crate::source::ByteSource; +use std::borrow::Cow; + +/// The parts of a split ZIP, read as one archive (see the module docs). +pub struct ZipSpan { + parts: Vec>, + /// Where each part starts in the joined stream. + bases: Vec, + len: u64, + /// Rewritten ranges: `(offset, bytes)`, read in place of the parts' own. + patches: Vec<(u64, Vec)>, +} + +const EOCD: &[u8; 4] = b"PK\x05\x06"; +const EOCD64: &[u8; 4] = b"PK\x06\x06"; +const LOCATOR64: &[u8; 4] = b"PK\x06\x07"; +const CDFH: &[u8; 4] = b"PK\x01\x02"; + +fn u16_at(b: &[u8], at: usize) -> Option { + Some(u16::from_le_bytes(b.get(at..at + 2)?.try_into().ok()?)) +} + +fn u32_at(b: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(b.get(at..at + 4)?.try_into().ok()?)) +} + +fn u64_at(b: &[u8], at: usize) -> Option { + Some(u64::from_le_bytes(b.get(at..at + 8)?.try_into().ok()?)) +} + +fn put(b: &mut [u8], at: usize, v: &[u8]) { + b[at..at + v.len()].copy_from_slice(v); +} + +impl ZipSpan { + /// The parts, in order (`x.z01` first, `x.zip` last), as one archive. + pub fn new(parts: Vec>) -> Result { + let mut bases = Vec::with_capacity(parts.len()); + let mut len = 0u64; + for p in &parts { + bases.push(len); + len += p.len() as u64; + } + let mut span = ZipSpan { + parts, + bases, + len, + patches: Vec::new(), + }; + span.patches = span.rewrite()?; + Ok(span) + } + + /// Where offset `off` of part `disk` is in the joined stream. + fn absolute(&self, disk: u64, off: u64) -> Result { + let base = usize::try_from(disk) + .ok() + .and_then(|d| self.bases.get(d)) + .ok_or_else(|| { + format!( + "the directory names part {} of {}", + disk + 1, + self.parts.len() + ) + })?; + Ok(base + off) + } + + /// The end-of-directory records and the directory, rewritten. + fn rewrite(&self) -> Result)>, String> { + let last = self.parts.last().ok_or("no parts")?; + // The end record is in the last part's final 64 KiB and 22 bytes. + let from = last.len().saturating_sub(22 + 0xffff); + let tail = last.window(from, last.len() - from); + let at = tail + .windows(4) + .rposition(|w| w == EOCD) + .ok_or("no end of central directory in the last part")?; + let eocd_at = self.bases[self.parts.len() - 1] + (from + at) as u64; + let mut eocd = self.read(eocd_at, 22); + let (mut cd_disk, mut entries, mut cd_size, mut cd_off) = ( + u64::from(u16_at(&eocd, 6).ok_or("short end record")?), + u64::from(u16_at(&eocd, 10).ok_or("short end record")?), + u64::from(u32_at(&eocd, 12).ok_or("short end record")?), + u64::from(u32_at(&eocd, 16).ok_or("short end record")?), + ); + let mut patches = Vec::new(); + // Zip64: a locator just before the end record points at the Zip64 end + // record, whose fields replace the 16- and 32-bit ones. + let locator_at = eocd_at.checked_sub(20); + let locator = locator_at + .map(|a| self.read(a, 20)) + .filter(|l| l.starts_with(LOCATOR64)); + if let (Some(mut loc), Some(loc_at)) = (locator, locator_at) { + let disk = u64::from(u32_at(&loc, 4).ok_or("short Zip64 locator")?); + let at = self.absolute(disk, u64_at(&loc, 8).ok_or("short Zip64 locator")?)?; + let mut e64 = self.read(at, 56); + if !e64.starts_with(EOCD64) { + return Err("the Zip64 locator points at no Zip64 end record".to_string()); + } + cd_disk = u64::from(u32_at(&e64, 20).ok_or("short Zip64 end record")?); + entries = u64_at(&e64, 32).ok_or("short Zip64 end record")?; + cd_size = u64_at(&e64, 40).ok_or("short Zip64 end record")?; + cd_off = u64_at(&e64, 48).ok_or("short Zip64 end record")?; + let cd_at = self.absolute(cd_disk, cd_off)?; + put(&mut e64, 16, &0u32.to_le_bytes()); + put(&mut e64, 20, &0u32.to_le_bytes()); + put(&mut e64, 24, &entries.to_le_bytes()); + put(&mut e64, 48, &cd_at.to_le_bytes()); + patches.push((at, e64)); + put(&mut loc, 4, &0u32.to_le_bytes()); + put(&mut loc, 8, &at.to_le_bytes()); + put(&mut loc, 16, &1u32.to_le_bytes()); + patches.push((loc_at, loc)); + } + let cd_at = self.absolute(cd_disk, cd_off)?; + let cd_len = + usize::try_from(cd_size).map_err(|_| "central directory too large".to_string())?; + let mut cd = self.read(cd_at, cd_len); + if cd.len() != cd_len { + return Err("the central directory runs past the last part".to_string()); + } + let mut i = 0; + for _ in 0..entries { + if !cd.get(i..i + 4).is_some_and(|s| s == CDFH) { + return Err("a central directory record is not where the last one ends".to_string()); + } + let field = |at: usize| { + u16_at(&cd, i + at) + .map(usize::from) + .ok_or("short directory record") + }; + let (name, extra, comment) = (field(28)?, field(30)?, field(32)?); + self.rebase(&mut cd, i, name, extra)?; + i += 46 + name + extra + comment; + } + patches.push((cd_at, cd)); + put(&mut eocd, 4, &0u16.to_le_bytes()); + put(&mut eocd, 6, &0u16.to_le_bytes()); + let total = eocd[10..12].to_vec(); + put(&mut eocd, 8, &total); + if u32_at(&eocd, 16) != Some(u32::MAX) { + let at = u32::try_from(cd_at) + .map_err(|_| "a directory past 4 GiB with no Zip64 record".to_string())?; + put(&mut eocd, 16, &at.to_le_bytes()); + } + patches.push((eocd_at, eocd)); + Ok(patches) + } + + /// Point the directory record at `cd[i..]` at its member's offset in the + /// joined stream, on part 0. + fn rebase(&self, cd: &mut [u8], i: usize, name: usize, extra: usize) -> Result<(), String> { + let short = "short directory record"; + let (usize32, csize32) = ( + u32_at(cd, i + 24).ok_or(short)?, + u32_at(cd, i + 20).ok_or(short)?, + ); + let (disk16, off32) = ( + u16_at(cd, i + 34).ok_or(short)?, + u32_at(cd, i + 42).ok_or(short)?, + ); + // The Zip64 extra field holds, in this order, each field whose short + // form is all ones: sizes, offset, disk. + let mut z64 = None; + let mut e = i + 46 + name; + let end = e + extra; + while e + 4 <= end { + let (id, size) = ( + u16_at(cd, e).ok_or(short)?, + usize::from(u16_at(cd, e + 2).ok_or(short)?), + ); + if id == 1 { + z64 = Some((e + 4, e + 4 + size.min(end - e - 4))); + } + e += 4 + size; + } + let (mut off_at, mut disk_at) = (None, None); + if let Some((mut p, stop)) = z64 { + for (wide, width) in [(usize32 == u32::MAX, 8), (csize32 == u32::MAX, 8)] { + if wide { + p += width; + } + } + if off32 == u32::MAX && p + 8 <= stop { + off_at = Some(p); + p += 8; + } + if disk16 == u16::MAX && p + 4 <= stop { + disk_at = Some(p); + } + } + let disk = match disk_at { + Some(at) => u64::from(u32_at(cd, at).ok_or(short)?), + None => u64::from(disk16), + }; + let off = match off_at { + Some(at) => u64_at(cd, at).ok_or(short)?, + None => u64::from(off32), + }; + let abs = self.absolute(disk, off)?; + match off_at { + Some(at) => put(cd, at, &abs.to_le_bytes()), + None => { + let abs = u32::try_from(abs) + .map_err(|_| "a member past 4 GiB with no Zip64 record".to_string())?; + put(cd, i + 42, &abs.to_le_bytes()); + } + } + match disk_at { + Some(at) => put(cd, at, &0u32.to_le_bytes()), + None => put(cd, i + 34, &0u16.to_le_bytes()), + } + Ok(()) + } + + /// `len` bytes of the parts as they are, from `off` of the joined stream. + fn read(&self, off: u64, len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + let end = off.saturating_add(len as u64).min(self.len); + let mut at = off; + while at < end { + let k = self.bases.partition_point(|&b| b <= at) - 1; + let inner = (at - self.bases[k]) as usize; + let part = &self.parts[k]; + let want = ((end - at) as usize).min(part.len() - inner); + let w = part.window(inner, want); + out.extend_from_slice(&w); + if w.len() < want { + break; + } + at += want as u64; + } + out + } +} + +impl ByteSource for ZipSpan { + fn len(&self) -> usize { + usize::try_from(self.len).unwrap_or(usize::MAX) + } + + fn window(&self, off: usize, len: usize) -> Cow<'_, [u8]> { + let mut out = self.read(off as u64, len); + let (from, to) = (off as u64, off as u64 + out.len() as u64); + for (at, bytes) in &self.patches { + let (s, e) = ((*at).max(from), (*at + bytes.len() as u64).min(to)); + if s < e { + out[(s - from) as usize..(e - from) as usize] + .copy_from_slice(&bytes[(s - at) as usize..(e - at) as usize]); + } + } + Cow::Owned(out) + } + + fn read_error(&self) -> Option { + self.parts.iter().find_map(|p| p.read_error()) + } + + fn identity(&self) -> (usize, usize) { + (self as *const Self as usize, self.len()) + } +} + +#[cfg(all(test, feature = "zip"))] +mod tests { + use super::*; + + /// A ZIP of three members, and where its directory and end record are. + fn archive() -> (Vec, usize, usize) { + use std::io::Write; + let mut buf = std::io::Cursor::new(Vec::new()); + let mut zip = ::zip::ZipWriter::new(&mut buf); + let stored = ::zip::write::SimpleFileOptions::default() + .compression_method(::zip::CompressionMethod::Stored); + for (name, n) in [("a.bin", 3000), ("b.txt", 10), ("c/d.bin", 5000)] { + let opts = match name { + "b.txt" => ::zip::write::SimpleFileOptions::default(), + _ => stored, + }; + zip.start_file(name, opts).unwrap(); + zip.write_all(&(0..n).map(|i| (i * 7 % 251) as u8).collect::>()) + .unwrap(); + } + zip.finish().unwrap(); + let blob = buf.into_inner(); + let eocd = blob.windows(4).rposition(|w| w == EOCD).unwrap(); + let cd = u32_at(&blob, eocd + 16).unwrap() as usize; + (blob, cd, eocd) + } + + /// `blob` cut at `cuts` the way `zip -s` writes a set: each member's + /// directory record names its part and its offset there, and the end + /// record the directory's. + fn split(blob: &[u8], cd: usize, eocd: usize, cuts: &[usize]) -> Vec> { + let mut b = blob.to_vec(); + let bounds: Vec = std::iter::once(0).chain(cuts.iter().copied()).collect(); + let at = |abs: usize| { + let disk = bounds.iter().rposition(|&s| s <= abs).unwrap(); + (disk, abs - bounds[disk]) + }; + let mut i = cd; + while b[i..i + 4] == *CDFH { + let (disk, rel) = at(u32_at(&b, i + 42).unwrap() as usize); + put(&mut b, i + 34, &(disk as u16).to_le_bytes()); + put(&mut b, i + 42, &(rel as u32).to_le_bytes()); + i += 46 + + [28, 30, 32] + .iter() + .map(|&f| u16_at(&b, i + f).unwrap() as usize) + .sum::(); + } + let (cd_disk, cd_rel) = at(cd); + put(&mut b, eocd + 4, &(cuts.len() as u16).to_le_bytes()); + put(&mut b, eocd + 6, &(cd_disk as u16).to_le_bytes()); + put(&mut b, eocd + 16, &(cd_rel as u32).to_le_bytes()); + let ends: Vec = cuts + .iter() + .copied() + .chain(std::iter::once(b.len())) + .collect(); + bounds + .iter() + .zip(&ends) + .map(|(&s, &e)| b[s..e].to_vec()) + .collect() + } + + /// Read through the span, a split archive is the archive it was cut from, + /// byte for byte, wherever the cuts fall: inside members, inside the + /// directory, right before the end record. + #[test] + fn a_split_zip_reads_as_its_archive() { + let (blob, cd, eocd) = archive(); + for cuts in [ + vec![1], + vec![100, 4000], + vec![cd - 1, cd + 5], + vec![eocd - 1], + vec![10, 20, 30, eocd], + ] { + let parts = split(&blob, cd, eocd, &cuts); + assert_ne!( + parts.concat(), + blob, + "{cuts:?}: the split must change the directory" + ); + let boxed = parts + .into_iter() + .map(|p| Box::new(p) as Box) + .collect(); + let span = ZipSpan::new(boxed).unwrap(); + assert_eq!(span.window(0, span.len()).into_owned(), blob, "{cuts:?}"); + for (off, len) in [(0, 7), (cd - 3, 60), (eocd - 2, 30), (span.len() - 1, 9)] { + assert_eq!( + span.window(off, len).into_owned(), + blob[off..(off + len).min(blob.len())], + "{cuts:?} {off}" + ); + } + } + } + + /// A directory naming a part the set does not have is refused. + #[test] + fn a_part_too_few_is_refused() { + let (blob, cd, eocd) = archive(); + let mut parts = split(&blob, cd, eocd, &[100, 4000]); + parts.remove(1); + let boxed = parts + .into_iter() + .map(|p| Box::new(p) as Box) + .collect(); + assert!(ZipSpan::new(boxed).is_err()); + } +} diff --git a/crates/exav-unpack/src/stream.rs b/crates/exav-unpack/src/stream.rs index f4dcdb2a..72f48b4d 100644 --- a/crates/exav-unpack/src/stream.rs +++ b/crates/exav-unpack/src/stream.rs @@ -84,7 +84,10 @@ impl Read for BudgetReader<'_> { } /// Metadata for a member handed to a [`walk`] visitor. -#[derive(Debug, Clone)] +/// +/// New fields may appear in any release: build one with +/// `..MemberMeta::default()` for the fields you do not set. +#[derive(Debug, Clone, Default)] pub struct MemberMeta { pub name: String, /// Compressed size within the container (for `.cdb` matching), or the @@ -100,6 +103,36 @@ pub struct MemberMeta { /// matching password, damage part way). What was decoded, if anything, /// still comes with it. pub unsupported: Option<&'static str>, + /// When the member was last modified, as its container records it. + /// Recorded, not checked. + pub mtime: Option, + /// Its Unix mode (`st_mode`: file type and permission bits), as its + /// container records it. + pub mode: Option, + /// For a symbolic link whose container records the target apart from the + /// content (tar), that target. A ZIP link's target is its content. + pub link: Option, + /// For a ZIP member, its compression method number (0 stored, 8 + /// deflate, ...). + pub zip_method: Option, +} + +/// When a member was last modified. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mtime { + /// Seconds since the Unix epoch. + Unix(i64), + /// A wall-clock time with no zone, as a DOS timestamp records it: the + /// local time where the archive was written. Turning it into an instant + /// takes a time zone, which is the reader's to choose. + Local { + year: u16, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + }, } /// A member's content. @@ -182,6 +215,7 @@ pub fn walk( size: None, encrypted: false, unsupported: Some("format excluded by the caller's allowed_formats"), + ..MemberMeta::default() }; return Ok(visit(&meta, None, budget)); } @@ -290,9 +324,8 @@ pub(crate) fn single_meta( MemberMeta { name: name.to_string(), comp_size: src.len() as u64, - size: None, - encrypted: false, unsupported, + ..MemberMeta::default() } } @@ -352,6 +385,9 @@ pub(crate) fn emit_entry( size: e.unsupported.is_none().then_some(e.data.len() as u64), encrypted: e.encrypted, unsupported: e.unsupported, + mtime: e.mtime, + mode: e.mode, + ..MemberMeta::default() }; // An unsupported member may still carry what was decoded of it. let content = (e.unsupported.is_none() || !e.data.is_empty()).then_some(e.data); @@ -393,9 +429,7 @@ pub(crate) fn stream_single( let meta = MemberMeta { name: name.to_string(), comp_size, - size: None, - encrypted: false, - unsupported: None, + ..MemberMeta::default() }; emit_stream(&meta, &mut *dec, budget, visit) } @@ -491,10 +525,8 @@ pub(crate) fn stream_stored( Region::Unwalked(name, reason) => { let meta = MemberMeta { name, - comp_size: 0, - size: None, - encrypted: false, unsupported: Some(reason), + ..MemberMeta::default() }; if let Some(t) = visit(&meta, None, budget) { return Ok(Some(t)); @@ -510,8 +542,7 @@ pub(crate) fn stream_stored( name, comp_size: size, size: Some(size), - encrypted: false, - unsupported: None, + ..MemberMeta::default() }; if let Some(t) = emit_stream(&meta, &mut window, budget, visit)? { return Ok(Some(t)); diff --git a/crates/exav-unpack/tests/cli.rs b/crates/exav-unpack/tests/cli.rs index ecf0891f..9d06d6a4 100644 --- a/crates/exav-unpack/tests/cli.rs +++ b/crates/exav-unpack/tests/cli.rs @@ -1,4 +1,5 @@ -//! End-to-end tests for the `exav-unpack` binary. +//! End-to-end tests for the `exav-unpack` binary, whose command line is a +//! subset of `unzip`'s. //! //! This crate is a thin shell over the library, but the thin part is where the //! only actually dangerous code lives: it is the one place in the project that @@ -6,9 +7,12 @@ //! is ordinary in a hostile archive, and every extractor CVE of the last decade //! is some version of honouring it. So the containment property gets tested //! directly, not inferred from the shape of the code. +//! +//! What the command line prints and returns is checked against `unzip` itself +//! by a differential script; these tests hold the behaviour exav-unpack owns. use std::path::{Path, PathBuf}; -use std::process::{Command, Output}; +use std::process::{Command, Output, Stdio}; const BIN: &str = env!("CARGO_BIN_EXE_exav-unpack"); @@ -46,14 +50,19 @@ impl Drop for Scratch { } } -/// Minimal stored-member ZIP (local headers only — exav reads those). +/// Minimal stored-member ZIP (local headers only: exav reads those). fn zip_of(members: &[(&str, &[u8])]) -> Vec { + zip_with(0, members) +} + +/// A ZIP whose members claim compression `method`, their bytes as given. +fn zip_with(method: u16, members: &[(&str, &[u8])]) -> Vec { let mut v = Vec::new(); for (name, body) in members { v.extend_from_slice(b"PK\x03\x04"); v.extend_from_slice(&20u16.to_le_bytes()); v.extend_from_slice(&0u16.to_le_bytes()); - v.extend_from_slice(&0u16.to_le_bytes()); // stored + v.extend_from_slice(&method.to_le_bytes()); v.extend_from_slice(&0u16.to_le_bytes()); v.extend_from_slice(&0u16.to_le_bytes()); v.extend_from_slice(&0u32.to_le_bytes()); @@ -67,11 +76,57 @@ fn zip_of(members: &[(&str, &[u8])]) -> Vec { v } +/// One tar entry: `(name, type, mode, mtime, link, body)`. +type TarEntry<'a> = (&'a str, u8, u32, u64, &'a str, &'a [u8]); + +/// A ustar archive of `entries`. +fn tar_of(entries: &[TarEntry]) -> Vec { + let mut v = Vec::new(); + for (name, kind, mode, mtime, link, body) in entries { + let mut h = [0u8; 512]; + h[..name.len()].copy_from_slice(name.as_bytes()); + let octal = |h: &mut [u8; 512], at: usize, width: usize, n: u64| { + let s = format!("{n:0w$o}\0", w = width - 1); + h[at..at + width].copy_from_slice(s.as_bytes()); + }; + octal(&mut h, 100, 8, u64::from(*mode)); + octal(&mut h, 108, 8, 0); + octal(&mut h, 116, 8, 0); + octal(&mut h, 124, 12, body.len() as u64); + octal(&mut h, 136, 12, *mtime); + h[156] = *kind; + h[157..157 + link.len()].copy_from_slice(link.as_bytes()); + h[257..263].copy_from_slice(b"ustar\0"); + h[263..265].copy_from_slice(b"00"); + h[148..156].copy_from_slice(b" "); + let sum: u32 = h.iter().map(|&b| u32::from(b)).sum(); + h[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + v.extend_from_slice(&h); + v.extend_from_slice(body); + v.resize(v.len().div_ceil(512) * 512, 0); + } + v.resize(v.len() + 1024, 0); + v +} + fn run(args: &[&str]) -> Output { - Command::new(BIN) - .args(args) - .output() - .expect("spawn exav-unpack") + run_in(None, args, b"") +} + +/// Run in `dir` (or here), with `stdin` as its input. +fn run_in(dir: Option<&Path>, args: &[&str], stdin: &[u8]) -> Output { + let mut c = Command::new(BIN); + c.args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(d) = dir { + c.current_dir(d); + } + let mut child = c.spawn().expect("spawn exav-unpack"); + use std::io::Write; + child.stdin.take().unwrap().write_all(stdin).unwrap(); + child.wait_with_output().unwrap() } fn code(o: &Output) -> i32 { @@ -107,76 +162,82 @@ fn tree(root: &Path) -> Vec { out } +fn s(p: &Path) -> &str { + p.to_str().unwrap() +} + #[test] fn list_names_every_member_with_its_size() { - let s = Scratch::new("list"); - let f = s.write( + let sc = Scratch::new("list"); + let f = sc.write( "a.zip", &zip_of(&[("one.txt", b"hello"), ("two.txt", b"worldly")]), ); - let o = run(&["list", f.to_str().unwrap()]); + let o = run(&["-l", s(&f)]); assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); let out = stdout(&o); - assert!(out.contains("2 member(s):"), "{out:?}"); - assert!(out.contains("5 one.txt"), "{out:?}"); - assert!(out.contains("7 two.txt"), "{out:?}"); - // The detected format goes to stderr so `list` output stays pipeable. - assert!(stderr(&o).contains("format: Zip"), "{}", stderr(&o)); + assert!(out.contains("Length Date Time Name"), "{out:?}"); + assert!( + out.contains(" 5 ") && out.contains(" one.txt"), + "{out:?}" + ); + assert!( + out.contains(" 12 2 files"), + "{out:?}" + ); + let o = run(&["-Z1", s(&f)]); + assert_eq!(stdout(&o), "one.txt\ntwo.txt\n"); } #[test] fn list_writes_nothing_to_disk() { - // `list` is the "just tell me" mode; if it wrote files it would be a trap. - let s = Scratch::new("listclean"); - let f = s.write("a.zip", &zip_of(&[("one.txt", b"hello")])); - let before = tree(s.path()); - let o = run(&["list", f.to_str().unwrap()]); + // `-l` is the "just tell me" mode; if it wrote files it would be a trap. + let sc = Scratch::new("listclean"); + let f = sc.write("a.zip", &zip_of(&[("one.txt", b"hello")])); + let before = tree(sc.path()); + let o = run(&["-l", s(&f)]); assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); - assert_eq!(tree(s.path()), before, "list must not create files"); + assert_eq!(tree(sc.path()), before, "list must not create files"); } #[test] fn extract_writes_members_into_the_output_directory() { - let s = Scratch::new("extract"); - let f = s.write( + let sc = Scratch::new("extract"); + let f = sc.write( "a.zip", &zip_of(&[("one.txt", b"hello"), ("dir/two.txt", b"world")]), ); - let out_dir = s.subdir("out"); - let o = run(&["extract", f.to_str().unwrap(), out_dir.to_str().unwrap()]); + let out_dir = sc.subdir("out"); + let o = run(&[s(&f), "-d", s(&out_dir)]); assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); - assert_eq!( - std::fs::read(out_dir.join("one.txt")).unwrap(), - b"hello".to_vec() - ); + assert_eq!(std::fs::read(out_dir.join("one.txt")).unwrap(), b"hello"); assert_eq!( std::fs::read(out_dir.join("dir/two.txt")).unwrap(), - b"world".to_vec(), + b"world", "a nested member name creates the directory under the output root" ); + assert!(stdout(&o).contains(" extracting: "), "{}", stdout(&o)); } #[test] fn a_traversing_member_name_is_written_inside_the_output_directory() { // The zip-slip case. `../../../` components are dropped, not honoured, so // the file lands under the output root by its basename. - let s = Scratch::new("slip"); - let f = s.write( + let sc = Scratch::new("slip"); + let f = sc.write( "evil.zip", &zip_of(&[("../../../escaped.txt", b"pwned"), ("ok.txt", b"fine")]), ); - let out_dir = s.subdir("out"); - let o = run(&["extract", f.to_str().unwrap(), out_dir.to_str().unwrap()]); + let out_dir = sc.subdir("out"); + let o = run(&[s(&f), "-d", s(&out_dir)]); assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); - assert_eq!( std::fs::read(out_dir.join("escaped.txt")).unwrap(), - b"pwned".to_vec(), - "the member is still extracted — just contained" + b"pwned", + "the member is still extracted, just contained" ); - // Nothing above the output root: not in the scratch dir, not in its parent. assert!( - !s.path().join("escaped.txt").exists(), + !sc.path().join("escaped.txt").exists(), "escaped one level up" ); assert!( @@ -193,10 +254,10 @@ fn a_traversing_member_name_is_written_inside_the_output_directory() { #[test] fn an_absolute_member_name_is_rerooted_under_the_output_directory() { - let s = Scratch::new("absolute"); - let f = s.write("evil.zip", &zip_of(&[("/etc/cron.d/backdoor", b"x")])); - let out_dir = s.subdir("out"); - let o = run(&["extract", f.to_str().unwrap(), out_dir.to_str().unwrap()]); + let sc = Scratch::new("absolute"); + let f = sc.write("evil.zip", &zip_of(&[("/etc/cron.d/backdoor", b"x")])); + let out_dir = sc.subdir("out"); + let o = run(&[s(&f), "-d", s(&out_dir)]); assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); assert!( out_dir.join("etc/cron.d/backdoor").exists(), @@ -212,12 +273,12 @@ fn an_absolute_member_name_is_rerooted_under_the_output_directory() { #[test] fn a_member_name_with_nothing_usable_left_is_skipped_and_reported() { // `../..` reduces to nothing. Silently dropping it would be a member the - // user never learns about; the CLI names it and fails. - let s = Scratch::new("empty-name"); - let f = s.write("evil.zip", &zip_of(&[("../..", b"x")])); - let out_dir = s.subdir("out"); - let o = run(&["extract", f.to_str().unwrap(), out_dir.to_str().unwrap()]); - assert_ne!(code(&o), 0, "a skipped member must not report success"); + // user never learns about; the CLI names it and warns. + let sc = Scratch::new("empty-name"); + let f = sc.write("evil.zip", &zip_of(&[("../..", b"x")])); + let out_dir = sc.subdir("out"); + let o = run(&[s(&f), "-d", s(&out_dir)]); + assert_eq!(code(&o), 1, "a skipped member must not report success"); assert!( stderr(&o).contains("unsafe member name"), "the skip must be explained: {}", @@ -228,70 +289,547 @@ fn a_member_name_with_nothing_usable_left_is_skipped_and_reported() { #[test] fn extract_defaults_to_the_current_directory() { - let s = Scratch::new("cwd"); - let f = s.write("a.zip", &zip_of(&[("one.txt", b"hello")])); - let work = s.subdir("work"); - let o = Command::new(BIN) - .args(["extract", f.to_str().unwrap()]) - .current_dir(&work) - .output() - .expect("spawn"); + let sc = Scratch::new("cwd"); + let f = sc.write("a.zip", &zip_of(&[("one.txt", b"hello")])); + let work = sc.subdir("work"); + let o = run_in(Some(&work), &[s(&f)], b""); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!(std::fs::read(work.join("one.txt")).unwrap(), b"hello"); +} + +#[test] +fn no_arguments_and_help_print_usage_and_succeed() { + for args in [&[][..], &["-h"], &["--help"]] { + let o = run(args); + assert_eq!(code(&o), 0, "{args:?}"); + assert!(stdout(&o).contains("usage:"), "{args:?}: {}", stdout(&o)); + } + for args in [&["-v"][..], &["--version"]] { + let o = run(args); + assert_eq!(code(&o), 0); + assert_eq!( + stdout(&o).trim(), + format!("exav-unpack {}", env!("CARGO_PKG_VERSION")) + ); + } +} + +#[test] +fn an_unzip_option_outside_the_subset_is_refused() { + let sc = Scratch::new("refused"); + let f = sc.write("a.zip", &zip_of(&[("one.txt", b"hello")])); + for args in [ + &["-X", s(&f)][..], + &["-a", s(&f)], + &["-fo", s(&f)], + &["-v", s(&f)], + &["-Z", s(&f)], + &["-k", s(&f)], + &["-lt", s(&f)], + &["-o"], + &["--frobnicate", s(&f)], + &["--max-size", "lots", s(&f)], + ] { + let o = run(args); + assert_eq!(code(&o), 10, "{args:?}: {}", stderr(&o)); + assert!(stderr(&o).contains("usage:"), "{args:?}"); + } + let o = run(&["-X", s(&f)]); + assert!( + stderr(&o).contains("-X (owners and ACLs) is an unzip option"), + "{}", + stderr(&o) + ); +} + +#[test] +fn an_existing_file_is_kept_unless_overwrite_is_given() { + let sc = Scratch::new("overwrite"); + let f = sc.write("a.zip", &zip_of(&[("one.txt", b"new")])); + let out_dir = sc.subdir("out"); + std::fs::write(out_dir.join("one.txt"), b"mine").unwrap(); + // No answer to the prompt: "[N]one", a warning. + let o = run(&[s(&f), "-d", s(&out_dir)]); + assert_eq!(code(&o), 1, "stderr: {}", stderr(&o)); + assert!(stderr(&o).contains("replace "), "{}", stderr(&o)); + assert_eq!(std::fs::read(out_dir.join("one.txt")).unwrap(), b"mine"); + let o = run(&["-n", s(&f), "-d", s(&out_dir)]); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!(std::fs::read(out_dir.join("one.txt")).unwrap(), b"mine"); + let o = run_in(None, &[s(&f), "-d", s(&out_dir)], b"y\n"); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!(std::fs::read(out_dir.join("one.txt")).unwrap(), b"new"); + std::fs::write(out_dir.join("one.txt"), b"mine").unwrap(); + let o = run(&["-o", s(&f), "-d", s(&out_dir)]); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!(std::fs::read(out_dir.join("one.txt")).unwrap(), b"new"); +} + +/// A link already in the output directory, to a file or to a directory, never +/// takes a member outside it, even with `-o`; nor does a link in the archive. +#[cfg(unix)] +#[test] +fn nothing_is_written_through_a_link() { + let sc = Scratch::new("links"); + let outside = sc.subdir("outside"); + std::fs::write(outside.join("victim.txt"), b"keep").unwrap(); + let out_dir = sc.subdir("out"); + std::os::unix::fs::symlink(outside.join("victim.txt"), out_dir.join("file.txt")).unwrap(); + std::os::unix::fs::symlink(&outside, out_dir.join("dir")).unwrap(); + let f = sc.write( + "a.zip", + &zip_of(&[("file.txt", b"evil"), ("dir/planted.txt", b"evil")]), + ); + for extra in [&[][..], &["-o"][..]] { + let o = run(&[extra, &[s(&f), "-d", s(&out_dir)]].concat()); + assert_ne!(code(&o), 0, "{extra:?}: {}", stderr(&o)); + assert_eq!( + std::fs::read(outside.join("victim.txt")).unwrap(), + b"keep", + "{extra:?}" + ); + assert!(!outside.join("planted.txt").exists(), "{extra:?}"); + } + // A link member pointing out, then a member written "through" it. + let t = sc.write( + "links.tar", + &tar_of(&[ + ("up", b'2', 0o777, 0, "../outside", b""), + ("up/planted.txt", b'0', 0o644, 0, "", b"evil"), + ]), + ); + let fresh = sc.subdir("fresh"); + let o = run(&[s(&t), "-d", s(&fresh)]); + assert_ne!(code(&o), 0, "{}", stderr(&o)); + assert!( + stderr(&o).contains("outside the extraction directory"), + "{}", + stderr(&o) + ); + assert!(!outside.join("planted.txt").exists()); + assert!(!fresh.join("up").is_symlink()); +} + +/// tar keeps what unzip restores: the time, the permission bits, the links. +#[cfg(unix)] +#[test] +fn times_modes_and_links_are_restored() { + use std::os::unix::fs::PermissionsExt; + let sc = Scratch::new("meta"); + let t = sc.write( + "a.tar", + &tar_of(&[ + ("bin/", b'5', 0o750, 1_600_000_000, "", b""), + ("bin/run.sh", b'0', 0o755, 1_500_000_000, "", b"#!/bin/sh\n"), + ("bin/sbit", b'0', 0o4755, 1_500_000_000, "", b"x"), + ("bin/run", b'2', 0o777, 0, "run.sh", b""), + ]), + ); + let out = sc.subdir("out"); + let o = run(&[s(&t), "-d", s(&out)]); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + let m = std::fs::metadata(out.join("bin/run.sh")).unwrap(); + assert_eq!(m.permissions().mode() & 0o7777, 0o755); + let secs = |m: &std::fs::Metadata| { + m.modified() + .unwrap() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + }; + assert_eq!(secs(&m), 1_500_000_000); + let setuid = std::fs::metadata(out.join("bin/sbit")).unwrap(); + assert_eq!( + setuid.permissions().mode() & 0o7777, + 0o755, + "setuid is dropped, as unzip drops it without -K" + ); assert_eq!( - o.status.code(), - Some(0), - "stderr: {}", - String::from_utf8_lossy(&o.stderr) + std::fs::read_link(out.join("bin/run")).unwrap(), + PathBuf::from("run.sh") ); + let d = std::fs::metadata(out.join("bin")).unwrap(); assert_eq!( - std::fs::read(work.join("one.txt")).unwrap(), - b"hello".to_vec() + (d.permissions().mode() & 0o777, secs(&d)), + (0o750, 1_600_000_000) + ); + // -DD: no times at all; -D: none on directories. + let out2 = sc.subdir("out2"); + run(&["-DD", s(&t), "-d", s(&out2)]); + assert_ne!( + secs(&std::fs::metadata(out2.join("bin/run.sh")).unwrap()), + 1_500_000_000 ); } #[test] -fn no_arguments_prints_usage_and_exits_two() { - let o = run(&[]); - assert_eq!(code(&o), 2); - assert!(stderr(&o).contains("usage:"), "{}", stderr(&o)); +fn test_decodes_everything_and_writes_nothing() { + let sc = Scratch::new("test"); + let good = sc.write("good.zip", &zip_of(&[("one.txt", b"hello")])); + // Deflate-compressed by its header, but not deflate: it fails part way. + let bad = sc.write("bad.zip", &zip_with(8, &[("broken.bin", &[0xff; 64])])); + let before = tree(sc.path()); + let o = run(&["-t", s(&good)]); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert!( + stdout(&o).contains(" testing: one.txt OK"), + "{}", + stdout(&o) + ); + assert!( + stdout(&o).contains("No errors detected in compressed data of"), + "{}", + stdout(&o) + ); + let o = run(&["-t", s(&bad)]); + assert_eq!(code(&o), 2, "stdout: {} stderr: {}", stdout(&o), stderr(&o)); + assert!( + stdout(&o).contains("At least one error was detected in"), + "{}", + stdout(&o) + ); + assert_eq!(tree(sc.path()), before, "test must not create files"); } -#[test] -fn an_unknown_subcommand_is_a_usage_error() { - let s = Scratch::new("badsub"); - let f = s.write("a.zip", &zip_of(&[("one.txt", b"hello")])); - let o = run(&["explode", f.to_str().unwrap()]); - assert_eq!(code(&o), 2, "stderr: {}", stderr(&o)); - assert!(stderr(&o).contains("usage:"), "{}", stderr(&o)); +/// A ZIP of stored members, each ZipCrypto-encrypted with its password if it +/// has one. +fn zip_locked(members: &[(&str, &[u8], Option<&str>)]) -> Vec { + use std::io::Write; + use zip::unstable::write::FileOptionsExt; + let mut w = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, body, pw) in members { + let mut opts = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + if let Some(pw) = pw { + opts = opts.with_deprecated_encryption(pw.as_bytes()).unwrap(); + } + w.start_file(*name, opts).unwrap(); + w.write_all(body).unwrap(); + } + w.finish().unwrap().into_inner() } #[test] -fn too_many_arguments_to_list_is_a_usage_error() { - // `list a b` must not silently ignore `b`. - let s = Scratch::new("listargs"); - let f = s.write("a.zip", &zip_of(&[("one.txt", b"hello")])); - let o = run(&["list", f.to_str().unwrap(), "extra"]); - assert_eq!(code(&o), 2, "stderr: {}", stderr(&o)); +fn a_wrong_password_fails_the_run_only_when_nothing_else_came_out() { + let sc = Scratch::new("badpw"); + let locked = sc.write( + "locked.zip", + &zip_locked(&[("a.txt", b"hello", Some("secret"))]), + ); + let mixed = sc.write( + "mixed.zip", + &zip_locked(&[ + ("a.txt", b"hello", Some("secret")), + ("b.txt", b"world", None), + ]), + ); + let out = sc.subdir("out"); + let o = run(&["-P", "wrong", s(&locked), "-d", s(&out)]); + assert_eq!(code(&o), 82, "stderr: {}", stderr(&o)); + assert!( + stderr(&o).contains(" skipping: a.txt incorrect password"), + "{}", + stderr(&o) + ); + let o = run(&["-P", "wrong", s(&mixed), "-d", s(&out)]); + assert_eq!(code(&o), 1, "stderr: {}", stderr(&o)); + assert_eq!(std::fs::read(out.join("b.txt")).unwrap(), b"world"); + let o = run(&["-t", "-P", "wrong", s(&mixed)]); + assert_eq!(code(&o), 1, "stdout: {}", stdout(&o)); + let want = "No errors detected in {} for the 1 file tested.\n1 file skipped because of incorrect password.\n"; + assert!( + stdout(&o).ends_with(&want.replace("{}", s(&mixed))), + "{}", + stdout(&o) + ); + let o = run(&["-t", "-P", "wrong", s(&locked)]); + assert_eq!(code(&o), 82, "stdout: {}", stdout(&o)); + assert!( + stdout(&o).contains("Caution: zero files tested in"), + "{}", + stdout(&o) + ); + let o = run(&["-t", "-P", "secret", s(&mixed)]); + assert_eq!(code(&o), 0, "stdout: {}", stdout(&o)); + assert!( + stdout(&o).contains("No errors detected in compressed data of"), + "{}", + stdout(&o) + ); } #[test] -fn a_missing_file_is_reported_and_fails() { - let o = run(&["list", "/definitely/not/here.zip"]); - assert_eq!(code(&o), 1, "stderr: {}", stderr(&o)); - assert!(stderr(&o).contains("cannot read"), "{}", stderr(&o)); +fn members_are_chosen_by_pattern_and_excluded() { + let sc = Scratch::new("patterns"); + let f = sc.write( + "a.zip", + &zip_of(&[("a.txt", b"1"), ("b.log", b"2"), ("d/c.txt", b"3")]), + ); + let out = sc.path().join("out"); + let o = run(&[s(&f), "*.txt", "-x", "d/*", "-d", s(&out)]); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!(tree(&out), [PathBuf::from("a.txt")]); + let o = run(&[s(&f), "nothere", "-d", s(&out)]); + assert_eq!(code(&o), 11, "stderr: {}", stderr(&o)); + assert!( + stderr(&o).contains("caution: filename not matched: nothere"), + "{}", + stderr(&o) + ); + let o = run(&["-C", s(&f), "A.TXT", "-d", s(&sc.path().join("out2"))]); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + let o = run(&["-j", s(&f), "-d", s(&sc.path().join("out3"))]); + assert_eq!(code(&o), 0); + assert_eq!( + tree(&sc.path().join("out3")), + ["a.txt", "b.log", "c.txt"].map(PathBuf::from) + ); + let o = run(&["-p", s(&f), "b.log", "a.txt"]); + assert_eq!(stdout(&o), "12", "in archive order"); } #[test] -fn an_unrecognised_format_fails_rather_than_reporting_an_empty_archive() { - // "0 member(s)" on a file that is not an archive would read as "opened it, - // found nothing" — the one answer that must never be guessed at. - let s = Scratch::new("unknown"); - let f = s.write("notes.txt", b"just some text, definitely not an archive\n"); - let o = run(&["list", f.to_str().unwrap()]); - assert_eq!(code(&o), 1, "stderr: {}", stderr(&o)); +fn a_wildcard_names_several_archives() { + let sc = Scratch::new("several"); + sc.write("a.zip", &zip_of(&[("one.txt", b"hello")])); + sc.write("b.zip", &zip_of(&[("two.txt", b"world")])); + let out = sc.path().join("out"); + let o = run_in(Some(sc.path()), &["*.zip", "-d", s(&out)], b""); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!( + tree(&out), + [PathBuf::from("one.txt"), PathBuf::from("two.txt")] + ); assert!( - stderr(&o).contains("unrecognised archive format"), + stderr(&o).contains("2 archives were successfully processed."), "{}", stderr(&o) ); + // A second name is a member, as unzip has it. + let o = run_in(Some(sc.path()), &["a.zip", "b.zip", "-d", "out4"], b""); + assert_eq!(code(&o), 11, "stderr: {}", stderr(&o)); +} + +#[test] +fn a_split_archive_is_read_from_any_of_its_parts() { + let sc = Scratch::new("split"); + let whole = zip_of(&[("one.txt", b"hello"), ("two.txt", b"world")]); + let (a, b) = whole.split_at(whole.len() / 2); + sc.write("set.zip.001", a); + let second = sc.write("set.zip.002", b); + let o = run(&["-Z1", s(&second)]); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!(stdout(&o), "one.txt\ntwo.txt\n"); + // Parts elsewhere, or not named as a set, are given with --volume. + let elsewhere = sc.subdir("elsewhere"); + let first = elsewhere.join("first.bin"); + std::fs::write(&first, a).unwrap(); + let rest = sc.write("rest.bin", b); + let o = run(&["-Z1", s(&first), "--volume", s(&rest)]); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!(stdout(&o), "one.txt\ntwo.txt\n"); +} + +/// A ZIP written by `zip -s` reads from any of its parts, as the files it was +/// made of. Skipped where there is no `zip`. +#[test] +fn a_spanned_zip_is_read_from_any_part() { + let sc = Scratch::new("zips"); + // Incompressible, so the set has three parts. + let mut x = 0x9e37_79b9_7f4a_7c15u64; + let big: Vec = (0..300_000) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect(); + sc.write("big.bin", &big); + sc.write("small.txt", b"hello\n"); + let made = Command::new("zip") + .args(["-q", "-s", "100k", "set.zip", "big.bin", "small.txt"]) + .current_dir(sc.path()) + .status(); + if !made.is_ok_and(|s| s.success()) { + eprintln!("skipped: no zip"); + return; + } + assert!(sc.path().join("set.z02").is_file(), "zip made a set"); + for part in ["set.zip", "set.z01", "set.z02"] { + let out = sc.path().join(format!("out-{part}")); + let o = run(&["-q", s(&sc.path().join(part)), "-d", s(&out)]); + assert_eq!(code(&o), 0, "{part}: {}", stderr(&o)); + assert_eq!(std::fs::read(out.join("big.bin")).unwrap(), big, "{part}"); + assert_eq!( + std::fs::read(out.join("small.txt")).unwrap(), + b"hello\n", + "{part}" + ); + } + std::fs::remove_file(sc.path().join("set.z02")).unwrap(); + let o = run(&["-l", s(&sc.path().join("set.zip"))]); + assert_ne!(code(&o), 0, "a part missing: {}", stdout(&o)); +} + +/// Sets written by official RAR (see `tests/suites/rar_solid.rs`), new-style +/// and old-style names, extract whole from whichever part is named. +#[test] +fn a_real_rar_volume_set_extracts_from_any_part() { + use sha2::{Digest, Sha256}; + let hex = |b: &[u8]| { + Sha256::digest(b) + .iter() + .map(|x| format!("{x:02x}")) + .collect::() + }; + let fixtures = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/rar_volumes"); + for parts in [ + ["set5.part1.rar", "set5.part2.rar", "set5.part3.rar"], + ["set4.rar", "set4.r00", "set4.r01"], + ] { + let sc = Scratch::new("realrar"); + for p in parts { + sc.write(p, &std::fs::read(format!("{fixtures}/{p}")).unwrap()); + } + for p in parts { + let out = sc.subdir(&format!("out-{p}")); + let o = run(&["-q", s(&sc.path().join(p)), "-d", s(&out)]); + assert_eq!(code(&o), 0, "{p}: {}", stderr(&o)); + assert_eq!( + hex(&std::fs::read(out.join("noise.bin")).unwrap()), + "6987decf3255b53b0f8cf0da45aafacce4294578716058e11cd018a18e57035e", + "{p}" + ); + assert_eq!( + hex(&std::fs::read(out.join("text.txt")).unwrap()), + "306e49d40e25318b97ad7628011238f0ed6af09880396518cd5a8e1cbe7efe19", + "{p}" + ); + } + } +} + +/// A RAR set reads from its first part, its members split across volumes +/// joined; parts named otherwise are given with `--volume`. +#[test] +fn a_rar_volume_set_is_read_whole() { + let arc = std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/rar_solid/solid_rar4.rar" + )) + .unwrap(); + let vols = rar4_volumes(&arc); + assert!(vols.len() > 2, "the fixture split into a set"); + let sc = Scratch::new("rarset"); + for (k, v) in vols.iter().enumerate() { + sc.write(&format!("set.part{}.rar", k + 1), v); + } + let whole = sc.write("whole.rar", &arc); + let want = stdout(&run(&["-p", s(&whole)])); + assert!(!want.is_empty()); + let o = run(&["-p", s(&sc.path().join("set.part1.rar"))]); + assert_eq!(code(&o), 0, "{}", stderr(&o)); + assert_eq!(stdout(&o), want); + // Renamed parts, given one by one. + let mut named = Vec::new(); + for (k, v) in vols.iter().enumerate() { + named.push(sc.write(&format!("piece-{k}"), v)); + } + let mut args = vec!["-p".to_string(), s(&named[0]).to_string()]; + for p in &named[1..] { + args.extend(["--volume".to_string(), s(p).to_string()]); + } + let o = run(&args.iter().map(String::as_str).collect::>()); + assert_eq!(code(&o), 0, "{}", stderr(&o)); + assert_eq!(stdout(&o), want); +} + +/// `arc` (RAR4) with each member cut in two, a volume per cut, as RARLAB's +/// technote describes a set. +fn rar4_volumes(arc: &[u8]) -> Vec> { + const MAGIC: &[u8] = b"Rar!\x1a\x07\x00"; + let u16_at = |p: usize| u16::from_le_bytes([arc[p], arc[p + 1]]) as usize; + let u32_at = |p: usize| u32::from_le_bytes(arc[p..p + 4].try_into().unwrap()) as usize; + let crc = |d: &[u8]| { + let mut c = !0u32; + for &b in d { + c ^= u32::from(b); + for _ in 0..8 { + c = (c >> 1) ^ (0xEDB8_8320 & (c & 1).wrapping_neg()); + } + } + !c + }; + let (mut vols, mut main) = (vec![MAGIC.to_vec()], Vec::new()); + let mut pos = MAGIC.len(); + while pos + 7 <= arc.len() { + let (kind, flags, size) = (arc[pos + 2], u16_at(pos + 3), u16_at(pos + 5)); + let add = if flags & 0x8000 != 0 { + u32_at(pos + 7) + } else { + 0 + }; + let end = pos + size + add; + match kind { + 0x73 => { + main = arc[pos..end].to_vec(); + vols[0].extend_from_slice(&main); + } + 0x74 => { + let data = &arc[pos + size..end]; + let halves = [&data[..data.len() / 2], &data[data.len() / 2..]]; + for (j, half) in halves.iter().enumerate() { + let mut h = arc[pos..pos + size].to_vec(); + let f = (flags | if j == 0 { 0x02 } else { 0x01 }) as u16; + h[3..5].copy_from_slice(&f.to_le_bytes()); + h[7..11].copy_from_slice(&(half.len() as u32).to_le_bytes()); + if j == 0 { + h[16..20].copy_from_slice(&crc(half).to_le_bytes()); + vols.last_mut().unwrap().extend(h); + } else { + let mut v = MAGIC.to_vec(); + v.extend_from_slice(&main); + v.extend(h); + vols.push(v); + } + vols.last_mut().unwrap().extend_from_slice(half); + } + } + _ => {} + } + if end <= pos { + break; + } + pos = end; + } + vols +} + +#[test] +fn a_missing_archive_is_reported_with_unzips_status() { + let o = run(&["-l", "/definitely/not/here.zip"]); + assert_eq!(code(&o), 9, "stderr: {}", stderr(&o)); + assert!(stderr(&o).contains("cannot find or open"), "{}", stderr(&o)); +} + +#[test] +fn an_archive_name_without_its_extension_is_found() { + let sc = Scratch::new("ext"); + sc.write("a.zip", &zip_of(&[("one.txt", b"hello")])); + let o = run_in(Some(sc.path()), &["-Z1", "a"], b""); + assert_eq!(code(&o), 0, "stderr: {}", stderr(&o)); + assert_eq!(stdout(&o), "one.txt\n"); +} + +#[test] +fn an_unrecognised_format_fails_rather_than_reporting_an_empty_archive() { + // An empty listing of a file that is not an archive would read as "opened + // it, found nothing": the one answer that must never be guessed at. + let sc = Scratch::new("unknown"); + let f = sc.write("notes.txt", b"just some text, definitely not an archive\n"); + let o = run(&["-Z1", s(&f)]); + assert_eq!(code(&o), 9, "stderr: {}", stderr(&o)); + assert!(stderr(&o).contains("not an archive"), "{}", stderr(&o)); assert_eq!(stdout(&o), ""); } diff --git a/crates/exav-unpack/tests/fixtures/rar_volumes/set4.r00 b/crates/exav-unpack/tests/fixtures/rar_volumes/set4.r00 new file mode 100644 index 0000000000000000000000000000000000000000..3cd62dcc6553b6ca89211a312e314c943f7a4ce3 GIT binary patch literal 8192 zcmV+bAphS|VR9iF2LSQ=a{&Mi000000001P{d6OcE&!?@001CP000B#7AJgpZ9!cf zGYJ5+fdBw*Z)tO7E@EkJ0I>N$P*xHARNn%u6GXj>J?jZ4HYv#YHy7!Uo8$XH;(^FE ziP=~DMnMB?D#YGKJm+=iVi9(XuxqI7Zt?gykdFfGT)k~!O~DKN&6!Hx!hWtvX5+qT zMEacH+GT)!!lSi=vc}m9Emno%?%<0x7j(x zFPVuBX5vb6`rpm5B)tMA{Y(W}RTDR@mZ!D)m=&Qx$MtOA*9v@>V=o|!KBZ0t)pDA^ z;0E9bVWJkE zd^*W+-g4KB#4B&IZ@cp7?LKU)JTFO`@HG5~ly+@ErnJRn9G`5f*pQ{`Eot99gd*SP zOzpa@28d zhx^k~A8-KlYnnG(L5Zrze@pd3tiU&GSGf%J;@}7I7|%25#3BVT!_1xThd&i)#BE!7 zWFFO`O8Bg~=B0hD@u8P&*Q}(_U7W|2FYLgYzYo@Q`V%>POP_fyC-z?pu}g@$yGGOs zt(bqBXSDlGy?cCquKRFIEqUE`ht-MhS z(O^)ULnO5 zJ#l96Mu#6O8w0d>YJC^A=2%D9d2>{+o zaTX&zuWCw5zzM0d+hNIWi-3w~+^{V!2H~vup4=KMcNw|9aww?^^Amo6m9lO}u^5US zBqoZMf$}_G?bv^}QQrYyd9a^X#bAvWAH+<}LpU@V#~1tNWDj_!zYKGu3V=OBjXPQ< zRSRQmmdO6|T=r(b0Qg&ZFDLhYZywqtF>X0AR!sLDq7EQe z>@M9^2=zfpI6ICT+@GOUCvy01apKkE5@9r?TH!=M%o~jU2jVCeW_NSg=(Ukst=qHO z>fhw`@!=Gc7G!jk0g@Gs?eU*oL!u8-AMv3_=UZG9KmPTwtvN4`r`?Ka^mq_%Hpph1 zOI93FPVn`s0rGaA5uZ*E`}5BgMV%m);@<2Wk{tUf16M>A z1vXEk=cF}Qh383>mk^dl8x^3fN_g}t)vHXzrxxIxE;ZGrsd%AsRe?pb$MVC_$X68T<_@%f* z$h5Cl3C_<YDoL)vhtGr>vY5r7C$}*AgPLoaXDa91t`yMiJb~c%-ODHY* zK-$LzS^gY&Q#Y&XC04ZaVleq22x-mldc?*P+!pzI+JV%(kymo4;0pZvzCA|WKQpsj z<12C%93ps5@kC1uZN9BfN)rl#CB50~Y0Zinde3KSf1g-!z8+K;WVBG2;rJAg4W*yj zSpcUk8O3`xb7U|y^6aID<2=M-MMS@M=PK)NfiktZU!C5rBf4MD{d)X~OM{^1T(U}j zYgjg6e64{iedQT|Kt!%^5SC%K|9_1a(qs$xlUU$;D(589vh&&?-BFh1GfxQZc|sKR ze+w)1Z=%!LYhX_>VrKF1E3K7ZjofI}K=Jfr&+%LlVr4KYUE16WA9ULDyn{(t(zEV#0PKZxT zeZUCztv!uHO_M1V!ZymeA6h#->^t^UG1GNxMweavmdZvXfQ3`k;8#{4Ghnyj!lq?} zojM{8QE*UMv^jldweN{yC64vDTCRgoh17qYMGUsblS{Di2MfQfZu!!Rj#b#HtCjPS zAP|aE-UtQU&@Hb;F|k|&9amgS@A8XyKQPlDLSDumI;k1WEIX9>aA_xCLS?wJh{>L9 zprYJl0#+Va&o(@fwx;x0qIQN_qsVaSnhHzZoSQ>19yGl%M~o0Xg;JB?g3>9QgUMw5t&F62@B|t}blevNR?a$4 zv2T0)`aQw8z(@Dr_JWJR2N|t)R|gdEDVZ|bcw)*gL)cg)dQHl}L`>0Nj2pWFVvt~X zz&s-%kzNX#QRgk1(p1;c)c}0vN3S@-v|2d7OrfEVPb`cwcgUEHEhP|a)6KqJ^?!tj z<5ZyZ4G-nd@8{y%Kncj=v{iSRfr7!XWybHtnJy$vZz0C^QqRR$9Qy@9C?gI-H14i( z0H6Z(>!lh??wD$3mk5Go!5L&A;unfJ;ZX80Uoy-n8u|dkp0QRi+N5-M2HEkLm|5n# zSo3FOEUv$kQ|(tnDFG;4Z$1T&W99cavUhLnrrW*y_N8-*`n)W8HUFhQldjx}_h(NN zpLS&5gW)?;!wn&b(Zz?QxW1iJja@ZOwwZ$}X)e^hvv!-V2sqXj2g3LmNvVK@eWBe)F{F`#d3=;13Ka&Eu{yj?7K=rLZ>14_QYep7xK8`*fd`Z8c$Y~D z@`?dwYgs{4W+-CHi!5TS*MFMI{pu7B zBNCy1VMRW?YAWiaSKTVVVhq2fN=oBqflyVqa<^;Ld6Dc4&?f ztto`=C$dP`=|>Nz)%pE=gKXJ203T%FxJPAgRI&EEvT6dpA=Q5EM#7L%@@O!#YKobd zWq~(|u=ZTtp+0T+&F4()H^gpP|7~+`>$Nk>ycM_ik(D?}&+DbSg~l3JK|cn9p&1w@ zKfhKsmymj|_KN64WnV1}4$YiY9@reN+xWn@Hmnw=k;b_wgHPse|6Dh=ijP#&vTPWg z8PKcPqpZ20sPD4v z2K@nK2oghH>L_oN8s-KWjunaGFT{c1l;K&Iwi!vKMpU{tR0Rqflb1pHO}_1Z=bk5! z)fn3UWMlPa+j7*J4suHCyueIKG=wm0FB)nDw{*w;v~$}IPf_LQ*tzpCa361TV55f2 zm-IcTbse>!Ti$F3z$?L+W-ipB;NFDR*X}X-9=?_}?m}IUOpFuESy)E9~M6Jj+ z=C%_9I78ncu$`!Bww>+*0GX#b@{+Y;>0}f2-p1I?y_$W&I30F7$i)pOed!7=9qKyd z3L2s*fyO)yv?!W(fDB?aNu+#+WpOtJ&RrgKNqxto7KdrY@bFA0wrQi{W@#Mhj77uA*E-Qmt7w@sp#Li11-erpKLXzKC1Vk!=qHOsJ&aN%a3M z>XkSyPL8{7&vYrfXD^^~tdsdsKOCJMI4XC(%fdR}`_ZA=p~rmIFAEJL`040l%GLp4 zH4X!sTJiS4&ve6Ash4d0ZCf`rMZ$IX*tTEBG)P1d`Kl_KM0`u7>4x+iUt^OVIIDU) zrcJD_V$=o8&rZ?@s>}rVs?JvxepY@bjs%PFEljTcr^u6t1}kJP;C7YJrc2{~Ci%(t z4dn(#$u%mEdED(L4soirD&UdyL1u+M)Z=A+DwGz8%z;{kn}u}H*%Bpx;#fsYJJEO} zjpNPO{r`a6;4(uW$53#J-#=Y?cDtTWVLomP^oYU#W>2><{W1u%qu;T9_Dg^LG=-gR zcS`F+FzWM?AC#0VMP@okZ;MpPp#voOufArHH+nH=-oWmLC#u@$-)_PAnl21l!yi(< zN8h!C-j(o}HQnv4KPalll%o)Sb&k{B(<#7`qv;sFOwGnUCr*ocz}DsxUpWu1-kUEB z72A(jAk!4}aE(9by^bPNj@8_T?Z=T3cgm%;dU$Nl+-9T6-ux=0K723Zd~uWm*`3a} z`7ys5WWOi4N?-neWcoprRz#ZRFoh_PDWLOy|M|g|hL8}MU@r^B!{e++VdTE#I=t!) zIhv}OQ#sAq2=<~%0blI|b6G608jB2&O*rui-OGwAx>5=^k+YzmR4FJK^8}@~&{oUDVWK5KtOi+U!jqoL(jMI&amko% z=7VYcU`^e_0)ggSi2)isE~L`z0EV5D*vtYA=Q{pudxH8`1_iz*^kYFA@=xOY>?qIi zx#&JIjazg9m$+tjGl3VcTrG#IOmA%rG8Q*bPlE4j zlF6H%zd1^NFp8oVzo(@$&f0`2$_SC?MWIpwxVgeB>etlh;VU7ku?k(Upjfq+skzQ{+%$73e)Zdv)s zpj9r?VME#;3q*Br;z^r@n3omwYAs$ZcHk^2lkW7iWRwVf#2X)Aifdo*;O>vUd=+cA z#M0i#0h-=}8n13Te1lqe89LXb0qG=bCfco1dyA}2EY_3lpwa|a0XQyo8JGN_zh5P} zl;-*Ue_)V10&>JbTIH(`8x6Dve9Iw>>WKT77m_NM8g~{iD8!4tRVn;Aa-JSv9Rb>WGLcc?^ZFp7I+!F9thPVa2u=|LKg$s|4iNc*ql*&j7=F6M7Dtabd%*qu0;%r zByowLPq>;Sh8(OV4)eTN35^eb>DUiC2A_Uuzz8~qJvYg~QFDM6h@w(x;JujmYmF{r zFA!v;_dXYOXQ^A|j>2S`=@!AzJMqIj1_G&~{=dI{PZcD=^5zMY@xjgdpAkQcS0%X> zvjOjlp~wxPkLrmnB8Ke5Pa-J*7Q!@A<`&HtXCA?PmN0E%2|>520i1$3QUY`_ayk7Ak@t1y$V}<4$ClKnwYmu2;^yCZas*$ZEVG1>5(N@K8O0ArZ zj7S3&0EO(zcII2%XSas38#1p_&ue& zW6bJ66PGixJk}cuwO>M&75OEqyGA^VR3Lg{TD)?7W9$_hXkame+^pg2#@;iNjPTpx z`CLwAS83RpE-?2yF=OX}J4-9GG}y6@d8=!S&aE|Xc9=2Px1|#qY0<+;kKP#1iBOmR zUUE2MGK0l2%#KJfv{!cqK}1U#D4y|K#Lj@S=U+waLhd)i|45y5;-_Cg5S@d34HBKF zEB@ER|BHbsRyyyuy%nfPIx}~vrh?441p>LN%w-dJhdNeQXYdg7M%(d%pcJCq*Kan- zUG?Ulj~D4$VFujn=!HNSr;s@(i$Dl`f6>MWM)*dkw{9X}nPnX*^ zy<3)m--hwP1|c@uGgK$Qa;P*CpTWKJOcgGXYYVA`EHG8)db1IXlSS=l| z!;2$+%|QKzZaAxc%!IJ$M(2_wN{Y#TIS%fbJRZfR%~5Ed-mPbxP?`|@fieVU&{RcD z&=sEn?iF7tav?R<|0T>0-6GXv6ylaFIp=9Sd1jT1J`tOi01(xkEtxMvLMjG48LcOg z=Y1qU!6^1q*1n4XIkBnfkcc^l`1}Kowd7w)4NsqgV;RqT-c~p_2H^| z84hU&CDgrf>k@Q&lod5%fIbkqcemxizYkoA)f6I+zYrAey30U$cfM!a@a=$BJWjI^ z0GI@$_&JT0{^1E9PJj!bRqM`$2wLIRxsABdGR@Awg4?ru_#OIchVYdd(Vyw04jELU z#LILlaoc(GgJh&M95x>o1XLN2!8aaEHPL{|Qf9TZ8H^$EA@F!F4r@q5W~&`Dslmqk z6R1EtvN6 zJ1VKnQO8Lp=%6+#0a*up56xS+X#NlS-U!7}s|ZXl!P&=swe#E+p0=@9AwtXO1E0~z z{&R0rLG@w1OvOA+mF$p3lTjhfCuof;wJO%*Z>4XmXq#u1tqeR#rNQ!CxIjAwCj{9C z@ErY;u8?pCiWGo+UFoCcFcp+%$mRko{0{Z@JyiBWMhtqI!byQ(e*9uG0i3HJmVIQ& zt?4uB6GzJUXEH*lVezG-PKkHFXzH3`M%lwg%Y>^q z5T`q2`#0LKupZa(ptSn#CNTet>;BeJlg`u-`qUa#LQffD=7_fG`CsSG?HuOoWDY}s+i($dp zuSX|41@|F3q*i?B^;j&ElR!n-ETBjU>m(h(4^y|UH@ForGUmuVUs8Nj8P9_!4P@|V?nWBr8JMvd{JX`HT@N3EmN-Sr``ZEu z0x69)csxryA2jVh-+r+T>0|HrDuy;b)dT#w+dQ2Gw-%K{S^ON{~P z$w^kE&Uh=3n?F5&(7U%jF3|!_bnO_Qik5C6!U`Sa@|W+W7f-g z>x^C2^8xTnPpxO&eR+wB-isgib;sZyl`;I`Scw*}>jH898Zc{3+F$w4&zrX;X;IERQ-#+9xVTg6Y zr>}7)W3o!kBObq9MDoSZg}#bXcA3?MHDn3E1l_wqA!c~dCk}Qa=zKCNM1_qK0*$$2V!_fx ztXn+^ol%YKp|-(Wlzy=Lkr#YP`WY0<=v)r1yhC+-HzA@kpA9*i4ejQF2Tww|G9qF> zNr!%*0VK2XVuu;q%8?ACPw{cM?nk1uAVdr#0Q(c=XrQejl2e-JCr1u#g&&x@Tt3^$ zDsCR4mYA#m!-wYjiR#6=991*~GHy8D$rF&n5y6zOt8y;;av6`-A7U`8ig}RxWq)Gi z-b_d|K3~>BN>)!-PaRtJuk-FG$i!0e@XH>8 zO@;;+fs)^EI5#fJEW+xIBX@@WHLvNQPTgM*#@adPXNnMUWqpZhq+dRViPc<@^X# z?;eo8F|Gj2k)r~Oh@NVWL^2~lomzWgx&k3kVVmyNMJsxCK!%!Kw`*k&`*-oaNMje{ zT(g1qa*Jj(Ei`LBVZm7ohf#FyeE{ytpIDkG7zA7ZtQ6E3p-v+lW}$kdf#Ck>QK8ca50ws*>lvcZ{?l}e0$AD zF;oK1^O)~h0FIV~A8HbPg2HrXV0!OF!Rp`SXz|;7+cghj)Q#wwf`KP`xQ_9_FmY~m zsn~w2M*k-7diJ4kvXQZIEfKpg>Y~TqZ40QXO1=j&`3VsHm(Ht(pCFQe+)Mm%)`IeY z7jpKZwqC!=%!~6A)KkW-UP^9`t#Q<}+V6fua7N$P+iz+R$!q6iu^yd8W1Qi16YFUhLgxcR4L$^e@cW` zf>$dI7K{57I?8$E?C7M?30k!W#5q1%l? z;tv6CxbUPgT$348x6WvNu{>1H0E=!4q}pAD$_F?w!5vJGKE#iT`vnlu&{MLJ-A00N z+hBZ4V6W3?-q0cPX*EZmXr(_`_W8sTbF$ns!iJ14&$n<;$JrYDZV;mp1X!pLLW7UC zlQMPTf1vx3+4wqYP2N!J3|3iOZd2tqz8?O*G1#n#YXK1Yy~=k|JMC(A}cz~nF`T5+r-B7qO7knoh{>I zU?&H!pLo7~rc7p-&H8s;m6SH<>9BDjgq~O9DhpO*n>zF_ry_i>L#^b!RJmn)6X*J? zX6{TGV(`yr-t!zkN{1BGbKS(*`6gV2x2E24kp=SN_Ca*ZAt+N8G4A4rOL3DYwe5iX z#xAS{asytEC3to_d?GYg#*t`^@v{g8+2bbI;BB)Kj}2>1NPDxgAk-$=@7X2zzk3Hr z*Iw##8w7d@-2W%h&z~Z;dFjUA9#4%lChoP2u*V!HZ3yyo84}rEQDl zUNaz~_uD8ND>Mng#C_@86A8?VFn+=vuuE`!PJ6EUS%XTh`{ISe5H4K>S^{NiK-{+u zokDjG7sM&p=F}|DhuC-BeIrSEtKZoO)=*%>ZfrR_js2=-rAMph98)@su?(^SL04)^ z0P2G|L1H-wFXslM8)By>^1un0Nh1Hl1DFK#l0a`4$N1O>_vUEecRgnKvZ^a3_x8@5 z3wSHvHOT6{)_)dd&S88b!VRl7oxI52w5S|40B+Xjm-Hxal&` zXn98CHu{d>xrb@(I_)NF`w2LT>vFj}$o>TF^>dG+g5Zea?5%OX^QIKP(VZnqeB&Y0 z>&((pNOI^?h~EPEqikfiww-Mlhr06v@edc3etbsP>l@aAWEldgOzwH538{XFdT0Us zeVNoTu(a$nxlqF%w2|VKJ;er3|^lz8q&Bcj)KMah6fH&b9aIl3>t(G z8r>xujEkU1EBXM?@6nXo-xrjj1N^;?sXh7rtY~65Gh*%f#?%XX#jD($yP^gH$4BJ1 zD(pfZUD+zjopthwaBzNEO`F8ZhLD*&9*O~^(idWT3>H(mr&<)3S;EkC72N<*-px&} zH#foUe@ODb2)dmT>T2V0*6f7bAm_ms;+Jg+EF!fKyfFWL=mRyjb) zp^g9i2#1^l(wQ$&hP^QdU9W1-ZCY$z!Rg8vES5+YihBxkaRu0i0K<-K2y)}zploc2 zdKCvVl_nAzBgE~x5h25a>&WcJy7T1)>-P5s*0vhy1C)K1=6<``dsIk$nw zcx>|Psc`EqN=lSUU#qO}&OaMn9=+6p#v|a-}$~uRqtXQF=19h_hEUD8*3K#E2H>FH8X5P(efpU zpx#`VG|Wt={Tz9{V@@G@U!s-{rtL`oDO!H0=(#q`ANyQb7JH0i78>HQ4`C27oh^l9 zR|~Zw;^F0iLq3J(sTr$J;} zlu}=Jg|ijbQGc`GCxj)(FT%R1wf}GY^QUA#91ugmU5Q?ZTW}XS5mWU}P#$e!U{SFF zar*r$uQMMXe%d9E>MTq*VM|oHdvg3o>?f-KQd|*jzVnHPnL(6 zROfzrQW>{9+Yf{4M3D`0f9Vxtr}&a8Qm@P8`{j@bA5`_-5KfHL`H5){6V6sIhb;*L z?9oO^Ocl3e`jRp1$RWK22 z$PTO#uL*_!6Jb}DFgE!CW|cZjq4@HSo*NH%MIlH1t^QA?tvyOuWD2@Ua$Ddx6<9c< zy>jlCQacMBXn*IM1vy#5!4hYzmnHHwr;cB73Q?e8SqgkT_Aa3Etj&4Y!cTq-rG5n# zK9HXBlT*)zZvG#c4%9exIU1(gmufmrcWD&1Rs&)r?Dx}h_JV4Gz>=FVx+mKGr zz@RH1ECpXAEWOz}=M5;ZMywR}b&NG{=8W~&Z!guG9yrr>{h2w0G6WCcYKSn9{#FT@ zHcG4-QxvwdUnbNe*;EvDM@%Zf32HFQ+W_3vjLUm2o-5p?em9# z5js@4expzma*L2@6+I!}iOMHHWkPmSRtxF?Kvwt+h5g^|oPwJXBRittY6`A0#&-LTb(^)Im%4nNZ<26wLE**>b{0@Qq_+wPK=ab_-mJYQ*go5 z>zwM*5* zS>#40C*{j(Ss!*`<5}Jx_$)-P?DjjZNAMBDLkON(zqW-BBvC2oi{^XG`m%^H7ck{Z zha;>ld9Bl;-Cw|)90TQR=sc0rk|7s%@5qb{83_R9{A<~;RCU19_8l%`G|JKz*60wG ztbEjN&OHhS#5Ea-Kd&b7v}E%uXW(w>uo2L1EbU1)aO`(twy>362|O(szT^+$&aU2l(q)71G(9i7k3+32Td~NYL{_rHpxU02ThUo_$8iBT4OXwh1Ve z5R0OtMFO4jx~a_WHGo<3I9BIdeu$xu&5F_4*!LGQrq~kw`DNhDk9pi#__?qt$(3$p zE_4Wth{6pKNKm7QRkaznI}0oAyZ6BmlXz!`XX?~eaSn)K`%8PYasBnHG_xe>?!BU> zvt4eP*7oI>A8Sq+0Ik!jeq%5G{&-sD@8v`(QnJv>=o=|kD8l?^6@?iYnV+&O3NKBesX!3vf(NXZuq^#T69raKp$d)HKymJ+TpVF=g=e0sT@x3_F;|eN$xZPWd1q~@)%!_ zGzb)V)o(q;O#%T7a*cw#IJU)2i^`&&DH%o6&27%Lk zUr`)CWv^ReNEp0jtA$4d$dS)VO!BTP@|EQTge`B}?GW6AZ%$(tp8i#%jK9f>*mGEA zrhY}0YZ|z*dP;RpDWjk^@RM!#_9K=${~GpCQP#H2JQz^|wc3pTi`o!bx}&-dGh znCLq{H}}5;Z*j==7J&+np-BW~*$F$9W9NvYFOJQwX@dJtKhfZJyeNAy3A^OEss!`S z=V{LS%y96Ti(U&l*~PWd^r-y=c!9)BfbZ^5os}L~v*MYi3QO&> z-TfG#$sGuJ4jWqPY>*!PG`nq&!??E5vgv)(c^X%aT}4*cI$M!Y|GK*{m_*7(xCYN{1pi~TY2%eLIJh9ruuO`$;-Bs7KaAl$6rTvrkgc8O4u7Ngw{4QyR)KE^D zuN&5WiDL0-6-2nER@43XN`m-2H4>F|@_tv@HpknMu);A)zUj*n(Zr%G{zM5BHPk7p zPIb`TlGZ-EB#U=lJqPy(MzYzm@NJ8KWgMc(6_%NoYeY|p?F!IjIE6|JTv}sIvwf>b z$h2bTTV@Pjl2wIUxsXoO-8leEz9+8zQ|U{gOi{$rd#13t?_sS4 zURMG=&P=u(gF@p3N{tyONqkB>!^F*Tht11?HHkw129C<>IQSb+$ig{<*{dCnU(mJ! zlGm8__Zooxg`G8zKs7udT(?3=+YnImD2bngxgJc%7-Z?B$lJx0@80dDzvN*5M0)vu z5tHf4Wj0opwd=i=y;m-K^@R&u4&K6n$0S)vz?c13?jJlFj8sx%b~-8W zc9(GJiDK!ZqB|o%tjarTct3`6%O@5Jpg3X`-}n8L5dWkkKpka2kN31b%^F>A0Yq2Y zEOuvL`thA`x3nN~(DRrAE_4Rk?4HOR2Aw|=&*MX9ZAbSw^xrccap1~vg2;9?etpS+tvW%5g2N` zq!%j$2|aJO+5s}Y>0KKZhbbu&r@>%FGvb69zHcIbK{5{cl02`dIHAHj<`eYEhIgHl z@JIFkyV1>nnYw%?<&CZo1O0G8E$>8i(V*ohBv8B~9G9W8SI{+e-c~oNVRJ>rp^{M&TJrC7HVQQq4c#Yj7mg{ogaN*&1Np_6d-L=7aqUZqAy8qPn6Wzab z!0TSV?-F_B-?&l@l}{iyCHx#4`J}sjt$>hmYRo8wLc9opNP}C z{%p~-%W~b__j-IZt7(VAG)&?pCO9Cz6(y+SvsvB9N3jJgl081)bx|2VfU01_W0p9s zAi40{TIp3=2Yb8>0g>=(k(CVyUF~NVpI-BDBBN+^1WV#p)DlFEyokqb;b(_z)vgGZ zs!yj|r2l-)l_DodQEGqITNuKU%Sud`qP<7`z^@ zTBNzdNY=ylGzy7^n_e@hB}e)LB-yH#SCWl79c6B~$6X$|oWZT^4NfR0)_?k1y!$Vt z8`z62e#6bK>|=(TL2!;HRZ5>sw@)t_PO09FDPvwD(qO_Q*5jgC(e~NIwJa6@WwjKm+Z*G-}s$4JocMrAJ5XBx5f=1CP~t zQiHP!E;DrPW3#hlWXfQ`_Vq-Hta0Y<560sQF8nS98f_~uGADUUk__KLZA)vfR;`fY z-M_NVW1y<7E0;JLQNHC%W2*Pmze=A(WgOAKgc=q0rsNAcN=Ivhz zEE%W{*P0V6lxo)RL(Rc@;dA8frMrRI$PvvlQ36g%_dXXd_&)4BVpLw=lnSPA6wXT% zCQJnIj4LMr-;DKBGP3SW$Bgw4YopTIkG51Fb?lY>pVGSC{(z3M{qNZ)_i7ao#6em1d0XeNUm)_ zc!jz@wL}IujI~QL)Ws0EUzwTDf$HU?&3BHHYE7NstKbvh9qT0#Cxf#%Gw6kef?COn z9}Gs7FEHHoDP>%vNoEouHJWZp{c(3o=wwffhR(+=`26_;YiG^NnsyWUpo8afa=ORZ zb4CI;f`B6LTZNK;n|A3Mv$4xKC)hppe~G~<-RLn0eS!DQhf41Q>;WC+m=SbCLxEAek0w<3NE zQ5W)?JzmtGQC7EO)6k^H#*+;XwGak!5frx<5^3 z*b%I&cCl$BX_i<&I`LWq5Qhba-?Cupw+t4Vg&QRZ#qt6+PvO<%DgFY;w*>q*9cMsYs+!U&y9j z9YtE`#(1_G=ZRqUx9B068PkjpR#fP~3>najq|eOw{b2vM<@x_FwLgE`{Qpn?e}DD) ze;@1i{vYG?|9_v``Tn2t?Q80m&+hx8SD=ZCwR-isInRQ&{GA4CR_WKT&{e*1^vBns zieY>Ye#b*T(HTwqGAXg|C;JW1y=mla?z$RF(%)YvSsv>rwB6t9SrKrBv);(~s+6p8 z`5zy(8~;CLUPVIt;=7haU1c#&wpYW}Ibmm`hU*{i@;-mt^XjjpP8`v_8}$qO1LQku zyW4bARCqbGkDExUpGb9kt=LDAW{> zopfe<>yMy51U_G+ULQfw%I`Ma9|15w8!Zc!YTDk)`YEqS2o}G|Tuq%RW0S4f&acc6 zMrq^k;GvpVh>L<>Jszc7Cbr0%+w{L^Ki^2aS6x<{-IdPsF2@36y87PVqn|c4pO1rW z<3k(@(JJtJeV;GhC~Ung8uNW|^$7J{we*W4=)DaoT~G@5S+pCopEM)9RRx#Uk2LBQ zJlj`P1LS3TnJY^1Dww}p!|$VRKc+VvIJG_E_qD8uzWei4#>n~0WpTjP&ESeO8jRJQ zjtd?I3V^Am&n=xUsPR>aV>Y>MYn!!0TtT@BuXl)i)xf*gibn8{o!eK9NDLXzvZUO!Xp- zU~*t;!&no0g@3tVdP2_}EaB_%Jw)e7YbhtaQix7fQyCJ_m1d_Uge2NN3>pwa8vRHa&?IZtg-oIxGq$OLOGNu>zc|@nk$lyJ0GlFPQO1Ma z>XyO>zzxu~%52rM@O8GDGk{K-(P9FMU0&dn} zQ#c8-P%VS|lAo^fzJk zbQ~wXJ?~yzCmt+e(Pe$!Mxw!Xz>GFEjx&b6;3Wtp$SZhmC%ZDd`OPvL^Gow~)Juhy zIF1I|bP|aMZ<755EzOQ5+8haGAT7Mk8k2s~;=Ht*HNjDAsbwfQ6u`H-E4jZOQ&JLoO81s zf0Rng*m{xBNc_wqq+oL>m)?;ixjTfEOC}QScjIOf)-{Vlrd8~ym zni9r7B@*@fJ3j~X$o%R_3B(|pxc%nd6HQDXCT?Le=-mA75>YTbC(U1yiI|l&YvIsK zTzcQ*wKB;A@~8BkD+iJ`)3RZCsQjcG*K zbd0QMt&~l-J-qjXuxDAmqm2Zb;Q#XBLuz$&P8GHT=I1e4%^`5MqU zZf6g6$zE-y6K-o7qNNjIGrt74>=q5+-gsCeq0Y9BHfk!-VHRiHLmLdBK5WDfcGGN! zH(Sgg;X21z=<#Vp*@_aRJA(-5y|bD#V2&?QhGl~9i}WBDZS8Di4BtOI+>} zX?nGU0Ua8692bGc3$_b+jYRVVY!`T{w4!bM?sRoXSnm%pz>u|92q3#9PJ-F@uXGFL z!e!ZF3a^QQv-mhkYmMJaLZl*CTY@^%3AknJ$Sf+tB)C)J-zZxsn-k4N9nedV`*~IC zwoR_7rVuUnzDR*3DXk;tCVke_Sj77*&em8|%bZcKdd`ATHjJ}pBCkgOhi>s>%vnrw z3%sx=&OD9$tR4MCCi0QOBASqB(+i%A!yVu;s8h@wUNN~Pc`r$ zQ0*$lL<#a5TH*wax{u_Z2*@!q_0VveiC!y28E?In|}thxZqgvJm8JMQxqr4Jdq zLJgBsCDw!qpX{rm%y_vR!#V(Pe6X?eV3IHQyfT22t|0UMm=icxiYSy#m@*9(>j=xX zt{}Jab=yXSw^-vrI0^m@_!4=}5)cFAa#FszaGaOek2FX<>uSs@LZd&LDx4&#$k7mz ziLAqxoMgxsWE4M3cqa<_9X>LkpP#Gwk|h&i*vDU-Xe7f#dcV*v$p^%I;63jCtSUh> z!w>7_2=Dx~I7zl`t26AlO{0qB8QI{IMczV)NGrW092#C+gA_zK^R2+)o)t3Zjbs}y(-2V84908Wf;S`{Gp z3CGtwxLW@kd`iifOt$H3iF!aRl2J7ixhe5d4N2m5evln)n6v(En?$aB)zAV&jF*lz zCHc$OO}hI~Lvl^CX`2>>du2^u*A(0Cj*MK08=8le9RMd#&w;z9m)1ISURpW{bBFFO zlt7VYRiRQ5DW;Ihg4Ls6e7r{o?CzI1gTIb1M>v|6@zVQ7w)w;=K{`dB(b9n_*%@|p z0TCkV!UsR8SneRZv(#L`MzfaGGN6)C5C1L_NlRRCM6MJZ3Al1rf||IeU)JX6t>Dh` zN+yy?HLnWrC(bb)ED^}!40%AkAzP#}vhGSaiXw{>Yu3DAzm!c&^FXprCY$~W_Q}w5 z@`|AoB;X$tY}dst5ihnvhe|pCOt-L^6xGD5{uJ^{>q0b0%qnGl*3SRU`0HAh+|r<~ zVjVRsac93icl|(PPIMxWE&#?A001CP000AE-=INxZ9!cfGYJ5+fdBw*Z)tO7E@EkJ z0I>N$P!RnQ5fKmo%;c1*HL6b!L*hHPHt#v22?lgPSOz~wi!G*N$M23@gz9@N;+F{k zo-SO^=*Stb&)E>Fu3;af*pg7zeR!_B7?!4UsWLSHYbM?Ey+xF+;jZq++)Aj1Y6f4z z#?xduVybtsK!P=UtnLJx63N;?C-x=CXcHyzDA@2WDG>NYnDnU|6)YtWFTm0z&({R?)HM4lQ>iE}j}*7)H8-bwS=A*X z@TkdMl&SPNo$uOYA^gX;O}9{Je%b&H09h4DxZ@0ZAYW%%*$UR-FB3;~Zr24;=M! z-jF8`&^znI0KFn58*c;qF?w#Os^ z1Q28>ATlXws~Hk^vPl0*Ss0`qCQj6iULXA|0f`?Cr=EoW=E^C(ifxMa-3IeYh9CV35# zTTHy~8l-A(f|4$+%`FerYfW3M?k^U7Hz;kxjvBEAG&Itx zBJxtG7U46X<=QkUPDF4HBISnk8Hnt5tZF{gbn1K&9JDqxxrv2ngCyH=nF5X_2Z_N) zC8j?Bu}TFX)6EKY(2I7|zHN^lWJITR(^mri24pA2`@hwkptN_43riLA5rRht`Lh53 zDXf;e#qUT>|9IaZ6|s4l(NsVeQ}CQJFIbX=Cl>#ahGU>Ors#J}*o*?Qeatr#k?Gys zH<3f^@s7a@-YAG}Lscv&snw^_`iQP6*OdzLL`8D6ZZ+Xb88Fm}gaj?AW_5p#X)s2} zd~`kCN7BB<4WtnFUeOv8qklr;Gg)Y7^x$UFZE=@9>Eor-Y@D;cBT{;sQov{bq1CL! zAmsM;A%~REwAYz!&fkeQGa4Huz69}Jub2dQiB7oJlFJdLRQglQIWt9}n2_CVK1jB>!9}X- zneDEH$>{g%{~R>GaWX<@@UfyMnTL+~xl|pHi57zje@sYk(u(ojD&j>cKHVyyKSV7< zyg-fQupy&y2h)XL$Yj$Iy`Fvpm+&95Jen_zG+wX<25(^|7QB!YdQ38#GgQc$O~lt!`5!&=9JFHUbrpHj{xQ&5{m)VNK`g4iK>gaEe+g9?uIT>IsS{Q8 z*hn+(Yx_65Rd?YjMs^%ReAw5J>BauEK1*UX-a-CV=9c=7kJ7X>u|2d}9cNK)L1p20 zDcJMac0MAaPm8Z+^=vTgb?t>h7T_`dqy6A1Hk52tD(b>DX(dsYkGt%$K6&Wy>Q9(x z!U(S*36u9ed*2?F{Cz#<%j&{_Ks;68sjY4+9Tx!Pu+<8J1(n|VTh&Srx@pc1ilv2<+mgY;vMLqSl#wIOd1E5+!0>Hy2XP}t@&PKH@y3=_Cz1MtO` zJC9&pQBkLD4q@&e)w!1hB=9B#?th?TUd8VVteJn{q_22lahBQW32!I*%AX|VNa1El zMPJE#PN#)ZN~1L*e83!L4y_!ofX2AsD$ri37vc|Qbs6ev2rTH>&fAt&3Uc&dP(vZ1-9A?YLqUL$-=EBc*e<%+Ph)1K~rkm67v~tyaC#zE{|-4&w`AYu3V0?LS`ctUuc_7(`kPXik2%#>2D-`c15XhLUP`f!940o1yP)LykyK;=K%+A zdG1*ZcoF_)M^g4jgm~D07=Zznf4Gl!=`HZD4B000C+N*G&d$c}!!havU-k0*b8j9; z{w%_~)tEWNni`QTHi^l_i?Ur$W#3#s{+nECO)zFu>nJE#wrsOyWV+<~VX}r^{|dhB zzBb4|znS5{2)eP>*g^79q*NHanx!QhZe#r&9;jg#cwo9(D7-#zrZM0Zr?`ww#mrhr zv(JdYj1rWU15>HFY>&_fZIRTU11Fua|8DQm@sAyz96r4bAxoX|sf>Il(~P~}0Z#bV z&)YN&x6Br_?g&Cf3`%BD$DcYN2VdtStuCM4%j>;cK|i z4Pz#*I#7YCHl8u1g$UkN1V)fPhQeF!-H(8vf|cEs_)glz=FZ0pU4G(jEO1+xl_r!- z*R@4RX`^-kt3Fz+Dz7QW2eyDEPTmOQ6DqoXxF;cFU%h7oj+)S)OenrzZ^`WK z(d|!?w1#!;C%FZ~1SSP?{;(}x=>KWOOoby9q{x)z7o%z)kv56{V1Pzwu%hSAb6N&4 zuWC3PECV^B+sGvkgzbIaC7*9Zzq_MLE`0MQSO;dO&T8gM0q5=`K>}8sMcxok zUS1}zYzj=0lSGE~{$2~%HwHW;@u1mMc7H=GtI3seYyZK|evfSaPFA|A>lOHnFIg}Y zc28SeQv1#xi}wmi#M+ZhFD`B5Z{1b6oUYo4R9)CNXM}!~CHxEX4B%Zsn$XzV--{;2 z;I8WenQGwydXU0Ws}Ez&(1mRJ1{-sUf**Dqt_Q3~)GX#bxNiQZ&OBA!WeI9G}S#tQ_)ho8ckjZ!FXkj*9-JbiI zyT~Y&)3SBIZFw#6K8g>We68z1krsvm9NpcuzZiyfzkIDbX)eQ+d9Qd8CGhbg80$rp z-B{zl5@=AhB`y7P&GzZq$qMztjy=O_-)W;VljBhl3YGoe$C+h!pY=Z7tG1Le@(;h0bL| zs3FS1d^VbXc8Jo~+z`kjfS!^TF1Dlt44N4$Q(Z%S7a705=oX`m`#-u;JeG zJRVb20#wB(lKrl94naj`b8_yRwV_5+4v%4@h3!$n2c~;oV>T3Xc8#g{9 zn-K_kcA|_MPQ+8vv~gIr?A?dq(0|2aNlR|Wv{djNFO`t7h@!`WQ=e9b$Ie&Dy5QA~ zN{A{!L;RyiCyhWrFkJvlWI^WivB_edbcCo28(vBqDQ9hSQeI!QwDP8T9ey}^q6~Gt z7sDm|lPfi*a{}$!mkT{N>`}tuy2sofXq{Z^Spz!~^z)o4 zeDY^%H86ln!$B%(p@RfH8UyB&{Q9k)k64X=u|^|w6P2$H-_Rrk<3j5_klR*hy1@nY zU5*s@vozT~lPjAA+%@d}(@>k0;;u^s$C)f}*#K;p@Kg`Nph^o=hRveJhrDOQE?cUB^$A zAdh#f?&UtTbqXG&22cfMmQZ8>`c5C88u#evhal)MK%e`uMbpzAKaLO06=w~m&2aN} zbNmuTdZOGsN{fo-Y$tDy?8)-DXdd zmz!H(fG&SKOx?;Qs3$b-X2XWEHmF{W?mswku4CB`zJo#BOMXjGQ(~FRtN03ZHT;Xs zM5y-H1d3!)F?Y4QycKG{Qbez1#qG%t)JjuZCOEEyM%3V8t#>ek8ZL_SFx{*Eeqdv1 zD2T+prb}`fh3u5S`mYA6PNS|pGcO|vIUW$cDS;zMF0Zw)huCu=A402_Z9X@Pg!(yv zqzMDh>&jRHMymIw5iU*!S01Ke4AjT-_5JLE*SPZJ{O(CUc(`^e-_TMi%?Q^n2;QPL zIrHWyXFY8$7w=2Ir7iRP52V=C_T2^hUpG4wfcGoInHjyuKC#~tjHw)dycXSrcfv5@ z`~yRzvU&cDp26zA4Z=&ho|A&KYx>iHa`6@SoPY*O74oMBq!Vga9^w;vk2v2k|HS86 zWbOS@RD4uQNcYM*251yFjX1)qdf3K;gZ+{j!h)M|Njp@+ms=$c9I1Y7*Mc-=fJ2%4 z5OJWybtsi)RbKV*v!g+Q^WO6=G@^mcH+i=Hvawm>hp-#$Fxpuek_{n+T4T((rhQ) zi|+?VoHQyVouw=f_yS`t(Gs-N`?Ty*6>4{)Wuo~@y&e|bzJ@3|^w>wqr-O=}{ znNq6vAU9Z&&b!mWst5Ew#yW~s5$!htOy%ZAHU4(>Ocq%7&A@FI$QE21Hc2t zLyFKSkQls_NOYLn_l^q#P)&>!a`IsDY5Q-H_ooy+j(&Q$RiyA%?`x>C5sxMVr49du zUZWn^g_2V%KtCB@gA{a-rk9EAV=LIAplfa{SQL|!gh3PxsM`MZo2}pe>@Ms7P{4y; zGva44ll?N?087brxb6Y6gmcd*Qy+=8%U#d0;_u~UDc<=+j$Snfj?mqd~VX0>T6NkUuTe$(oATs9;U~1mFvb%^^`mquc z4!s3K&VWMUYrpRVY-0ZMYKhX9085KE{__u4XKK(MuLoHKPGQh3)YqQ%M^JZqxXFf2 zAC232)pG}mc=ditJNRB%^;>Ri`eqr0rtrbd^M2n)!)%O%rX+0^Tp_mO&uXf|ia=wX zM?;j}2djy7uYxKhz^@Phw{+U1qC}rE(@s57Ya$U*9?`OHMY>RNT_E_?0lyjAVF~DS zg+07|K7uESkV+}uzfP}UWZzIt{w$)_*3hE%H$H53kf6Z66?(0~L$bk81Qixm#0eT) zTRK_sgw)uVnihCx@mk%>ud&%~iJQSGeJ78uOOy>|MHE-MXJq~HYP*K%+T9dt_Yn); z5%+1r?QL$h>`!hXJ5~U_13ve8ve}|e_we4d_c8w6VS`GmDg7Ztk}@|=S79U*Rp{i0 ze9Pd!uYUwcr8}4Tn-;j<>b9p^Q39rDiIznz-1a@q4s&-qA8`BE0Ck*B`+*K;(Q48k zn6;S0JPwbfo7Ux?&y;bs3LbMCR)Nk31<^Z3uEU-locFs-{c!Oxj19+HPw717_Ml2Q z-|Dlh{l2#@b|QdGJk04}ABPp-BLHmzc(PKgWLSVY!oGpvLet#&V1)!9M)HgoA3s?u zxNG+`qVtWPLIpqY9j&KJa|ot}&*&CC=c1j@-6WyG^fNSo>!PHC)gRtBW z4{!tbijKo~5QuweM^05{e8!|*^dIR;l!xDhSb{|DB+@~tk}Pjl?2>;-uKa|$S~Hxf zz*6cZ!Z(?LE!7E4DOEgC>z;!)Z)*If2q2fT`roady&TyL5a z?pAR`Y4mMk%H4iY$~1?hff@=xzjtkdiLx?jDft?hxsQ0q*y3?+>_V zoyYTh)?QoH#N{Ie8Ui4A+`b$O0R;^K6#*fq8v;Njj&fo`M8axJ0U!@v0UAOOzZkC? zLXZ&9-7LJ^INZG4Fp&s7WV>wnMGL6@aMof7xY23Kx+;;NfZ@|d4RrR#!PhPaqM9H&f+b6J%{@g7g)W-2|rzU@DEtG{(G-o3r8 zKfUH|-oKq+yaXQJz6S0-26R6^zU9u>Wa?4hzV>NtMb7)YocMhYCcoW#5MF6k>I@97 zx>#2`xv=F)Y`ga3^2mVME&aeA_K~@G!&(T#t(xp-_R(tx*YPh(HTq26R_5z`X+1Hw z*HF+}36L4`h-NBZKCvn*PbY4d`A1MW2A6T^{?RVnUCm&9p z5tW<%xHml%+bjw;#APv+=cqXpXu4b|b}qM9wN0HC@S-Wk)zoQ*G;zkAjT&%wge_Xm z6;7#sb|3wB@AiZ!iJPA9p%5cAzkb?9PyYR3*;v0qYobYW8u&v_yz^y#U>bG_R%(_d z@4AqnntcxJm0B_E_}0t*Gbc(jDz1*P>5UF9c%AQ2-CD)p#u{h&5scjJeqGf)jHc)- zlZ(1Ce3gwk&AKSs6{TcIW8ck|aqX#8DjR+JGc2jWFr4UjB#JkA17%@INlF#5gga~e zU5=z#u_itsRp^dU;i^HQ(CBno_-HJl+|ItS)HB!0PDe85O-N&CWci_0-~o0uyhw7SZ-GD_{Z{l zwpO`^hf{Ftd>av+#3vv2M0w)Q?nmq+k5F+&3G^J=2a@W#n~jhG?wpBxp6ek8l+|BV z9gD$YYSMFI9}IT7Q!!t1&OK_ybY!o}SpP!X^%Ys+yjmkSaa9}|V#B3D{K5>>aGQFy zhRGE~R7DEY1lrpmn2EyDx!XjUwWrIS2y%&sl$ZyB0u1|#iJ3BGDrILU_57aia!XY} z;L~6ON>b|w^gdx&@q|(6Gae}A3dapY=(Zy&wLhZWHkR2q*KhH%7`7H;Spz{HC0M@8 z#Qmh+gh!8`N#Y68Z4&WW1#$01>1nDF*0*_#3|eqYZcP*XMy_X24;jTy!z@nlYYF1D z<0xx4U$VA7k}hVQB$mEgOT*;O5_7jcrx*{0J}Oit*yL5QlE^b~Cp8D3k^~erOaX0V zAq>vc9emXNxporr?Lc4HZGif7<*4HLly!5ncVFX4QYuHOQ5AAar7ZYi{F z5kM!=WQ+j}A?GZ4Z%ZXy4OYtF{LoaK zi5aKh9cZ1o$u1|olOmO4dSzb>-ut50qqiePuR6$_I*}`yTZzDW1+7ZxDKkI&LJ0Pt zOSppScp3hJ&^ua8D-EsJY4BQ%RT8MNr7Je>f-=THY0XJITkQE^#3*#GLMA03Py5WN zH0Dd@^qR5dhwfQA*Rnc-4MgkRtEP*e*2edi2fQ4>$#vsoXat{Sg#5IAt~)S^A_V0LA+EsF5a!+DUeoiJ9&`2|<3n_F05g!w8c^xPaqPzBfk;sedBS zVMCb9z*X0=^=yWaFMqC2`3T*I#m<(cSll6bS7gms7c&q?&Xg7N>EQu#dI*Np3NCxc z@?M>muTAV~^bk`SaLLhgXDpzXJDb|KulGHUdvn8bUd6WyoviO{PCvxKX;c2Ipyc;| zoT!k@cRvL5?#w`IOC97+kkGd}n;m19=oIHvpvK(}oEK5Zqy7XZzukEA3 zk`HN-@1L;{EW0?AxGn%}e!sg`jk8FfH(_~<^8-d-mwQF~IwYiv?aijF)3s%*ms^4t zNAGRO2-f|JgRKeSHCv$TG$jCrG^N7hdY!EipLc=8!RJbE0%Gbe>vH=nwWQ#W47@s? zY&Ux&q-;cY;F!ylbb$F7qhhk_-2L=Mn5k@yYZ}yw&_eXEVZq)89t|jC0iU znpC>ntEpbb(WdH`QRhsVA0w?x;?PG__rp={ET!=P(wi>3!wY$%$W`xuTydE?6>l9c}k>TRh||h_JP|v$MI;j%C+S$9V|j z-H8dj1N2SIv{)kLu+wY*ICilo?z9D01m$~isIe@uMxM;c-vKHlQxu~U>8+tZ9Z|?3 zd{4U`W1AQ44KS$Nm~GmZ^j`#{47@6xqe^}B6j?hS=jb?11M+X5L4|g?#1I5V8O3Li zeAE6(`n*5W@3j>36;C^Fs%bUS+riJig(0Mfs0SbST@Fhw(FQ8n&_!HaE;j7j9jX9HoP1>Al|Ig1q{6vP2Po1t9Hg;9oY(cTQV`DFBFBTzl}+M4UY9J`s7pT?|_ zo-Uht`;1oHje*`c;O~G)G^_t z;ygFJx4Z(r!(LXxbaeUnbI0Eg_<_p-E#EWI{HjQPD{b~+X*1mS=0ek&NqG^)>e9#455T?1thfLOGqzpD@gs)jz> zvt0hcRsZVu8pF$F=j_vMC9c5|t+>tFHSM*0H;MbOkq_CSpnl=7f#2c)^X?5NB|Z(Ob(_c6?>)Rk z+J#YF;6UCmJjr)9=g1yCgw4#9pM^!!`=x&BFek@NAJa%63$ie=mHR~haV>GB5*=@N zl*Q1g>Kz3x1zFMYA=T8%6I{w^2vjO&h`EnYBra5zx>+?$O zA;YcQ@-L~kTiy9k7W`a1;s`mA7!u;le)b+DY5O&H3xFB(Z4y80iZ}O6c8S=}(IeD? z24x>x;C!x|D|8w8^_6w0_AYYKIqZ#IU6#ANnBM$p z_c2c0ivD@!M-&gwsI4q3py++6kA^A%<6ghQ7REFjT{5XYX9FA7+4hsCtVWi;DY`#c8CBn-AT3 zn^1ptG4@{@)slLaN|;g;pX{SX6VnonL2d0-H)Bs z@r0!S>ONrk*KBFW9#oRR?~IJ&c|_6rk`+?sOZEBUX#JDsrndxcI(eYELkkQ}FcRB< z%TpLb)E4 zeyB2DaKARZyI_|2*m7vg$9l}=P1UG1Kq7Y z%_NG8SX`8K1_-l%!rSYar${C)I>)rz*QJJeZ%}0wCE=HTjRlvQbMGWqIvbd%l=xG9KjYjVEg|9u zFcQ5c{5Er(f1~Uu3r3K40t{S>_HPlK>3YwDbyo8G)(SEw3sI|#C>n}1eKar6$FcS#wYT?i>MN_Zu1=ih`+86d0tPiA-jMfU;jB^xlHD6};f)xj z{{Z9u7YF{rAGQVvafPgd;j$Mo0^;TJlAaN@9&p6DLCp1%NxV&27qu~WYbBQuqKCc zKfL~1(MLr<7=kcdiHr-_VwXtlG91p?Ufu=dfVXBuoO~DfivPK^4WxT74%(x& z>Q>BocLdGicaV$E6*g%I!;<{UN9)qMgcRrCa?yr2&)@S+=#%0m$QsT^u5?sIbX4@& z9@{D>JrlZfrh^I3GQ9}B{Ue+lwO2q?$VIC&*08zJJY?YD(5uKb&qWF`V4SzAf3bkhxM=L362-gLixm!rvIe0 zS!g*Nn#(Lq^vo4lkJoN!E7!VhJI+k|Ps7I?h=x!5RTbMCpELcY-S_#S*RMkd;&-TSH^QprO_P<9*8p`YA&;VA;O4=p(ev5 zx6svpR~;iMC^4|?T0k+PWav?dqUj`N0m8=bszgtq z4a5y^-R%5`B!$M_$$pjF95;UfhM*nLtgB_Lk%;Sr z2+ZxmhcRk-ytyb#kM#OYp-~GIif5_?^lS^LgeU8DO|4`L6+|A7|CmNvK);;2s?hy( z;c(T0`q){cba0aC;?CjlF0s|C1fg>57UQRwOzY^YGO~ zuWRN_9TOTYQwGB?SEw0Q?=Cn^4U~YyPHzt>f3V_F^VKlEll}UIILCJY=EBBsT*m@HS9eP)J)>)L8UQ40V(QK z(6IBan7lQIr{xGS#$G7BOIRntUXYr4A0CWc>+}Ei6aDY$9;2i)vs? zG+j{LF}v81PWr4J0~tD+n2`UCpm!F_$-fL*=a(XYD!G4EZ7io(oAY}cE4A`CH~8cI z^jl*(ZQXOZNm6tEhYeMs#3e3gY}a0Q8M_d(iZ1(At-khu3PmRaD1SpRqW}3%`UVr< zFlkrgSju&g)jb$q8YNyY3w(qNZF+O3H_llcQ_rCI#~2|~y;J`oBfpBcUw0x!cHPqW zq}G*_e~uc?`SQ*?A+8vkt~>Rd5*)%vy}muL+}_l|QNn4wlb(P{D$zv$fc#_~(8P0p zDk-h2Emz;%Oe#{KODe{5%f9yvPGfwf3C%znh%7wSOld5f6p7jno|}94jO#^tSYd8Y zs^BdEI_W(4gPHIrRGKA2pxdx$fojmDndWzR_L)EPjTpow_iZNCcIeNa=~oYv&r19c z>Xc3^vKVycJr19SV^vXxPWhbr3XkU&ZOB=@L8t;cjT}7@ii~(q(d183ih+q{G|{v^ z)?O=5zu8nHL;pDZ;2;Xx?Dv}ElUSo!bMnKy7-QLu$U4P?Iyf_X?>ax0T1?v=EPo>U zNI-o`fDp0E+JHhsWo4<|X=S@0{MF;aGAxG9A;Oey6x+reI>DG914-Q0U!456m0y5= z4p1s1(#hM19GzuZBF(8fPWo-|!5C>}Xu-+Mns93whVcF}7Y?dxPCSn7_>rx_2&ApT9FPGAPZ%7MboiL7s8Y~kN+(C|5fKt^Aup(N z1nRLB3*Kw%Ce}9I)YcV;AAaM%!_qSn^2}enPH*EHw%hiTbXu0|xAhd~IBkX8h*)Ht zZk>~x9S}JyN`6VDMaSZop-QR>G2miD+vT2~SpCs3%>8Yw8$a}EPVQ#Q;H|>n<&R?@9 zbnDU@GtIpHlt@YY?gs!fS3huUXx;=0BTDBGuz}I&B9`|R}ah7!my;7 zOZ%9S1ihs@rP1NeSIT^$#T56kaG}c*w`nX7flr{elbd4vi~HP-vZchrZwtZWfa->E z>|koe!P|Y~H)?`|59sKGD;a7YccUu%VYjk+gLaLEBAdAgs42$ft@uR04xrcY_{QL> zB7v|_5fa{c+F(-M{RsZm8w#=O76hBPp~cuLT{3UI9|wm=OUpZ0`BU?PZZY;4lcIWc zyVVJmq(%NoQ}3%XAHH1{%y<)Rr!Ek!l)9Y)5OZ;Z=x^27M4S8Rvq->3=gyaC`n3xw z?5nQ+Zr|>S(w4>&Sqll9g~%L?CTK?DesxUA9;6&LeBB`4jQL`Q!lajfnMsbq9iXV+)jm)?-;(g=wLsrJmTb>uJee}j zIokU>hrZS3kV2LI zs=nxiUCH60#RIb$Y6pJ#_YpDDQKe3Ba3ZTLWd_l;KAooM<9r0;iam~*8x#ehJBIq< zXpKL$2!ah;>+&XgdLY7`yPvPl2(yi2KegYt&?4MsV?3C_`(}>bVMa6A0+DDCcrt9F z^=w~PJy|fDL}qwr{EIgev7EM~sWTs2&vOe-yJm403LounwbR$^=NfB2cqw3r@hv1^ z|1|VRO;nxE`{x~LE?L9yC#a`nS-!BpIYdkUXT`nKF?@a^)%MXUm+p676{Pm*xq$VC zlzin>nOGh#D^34#IInba1f5v4o@~8X7|#yhuKh!@WbZ)A5ts+}eH)!?dXie;yOm-C>4;t&Pz#e5VO7QjOeHFq`2!? zM()B_YJ&~RotSOL{$;34%kRt2;G`N6w?!B=l8P`J5fP_dHquOM@ndWncUXIS=mP^w z@va})*gbO@#ab*03d8;mo(vMJKu?#|Kd%upQjaHs-cTdHU2C4ySKbW(5a!W(j!+-w zHpQ$?9o;-`xJH$iYS#0!v+_M_L27{$Fp-H5LuuR?|Jk%h&xu_%sEV7kcpFdgC|Ko% z5ptnFfavU>Y}fP=QFmUNYc#LBh?H>xa&@?-gtR}0g?`;$bCM}M_(i2Hx%GeItY<^N4Pp2r?-7oP-DZ1zN0jz+9RUGO7mD;`kkpQS&jqrB*_+D= z#w;lWi1j7VH9_m}L^+o;LMIfwqg<__P3=BBb^k1=fC|ogxB8@U9TDyA{tD~x+E{U4 zJv;R|$V4gmE(vyhGa8XP#SSh0b(*$TAYiU^!YYL+zV~5xfY>mi62pTFk)=q2GZE5! ztH(e)NL{Zs(K&NckTjjE%5yd(GwQ&$+@*} zG&K=WMW>~uLg&%k_3s}4qlDOxXAX%0UWutt; z9-VWHY>Bd$QrzlsqBx7e)Fq+A{ebT$BB+1$3{p=#lVUcKJ^xBMoSqBVPoCwsFF-m& zzIs0K>e5bneK+FwtP3>QK~iKb_s<_BsB4~B-2RoaU}J&i4>S?yOhiwD-T!uNs=5-> zIGyCD4Fjp9RAK51*~2MvtQ_yUO}#iEC#_VT!5@_AH0ir*PwG&;a&kvoI2606%gp|! zc3L*Ghh)cN72LlwL=)+3bGoPSH8tJN$i``;QGEhC^b%c(YaFilzMQSw4?;4YDAdHe zk?_ zk%36vao->dSR&|O7Xr+;)6Z5}Tt%EV|LzW;l2x_XZg?6eIEOr7#`oVRq5B>WPs|^@ z$kK1P<=Ko8qirGUAbxWwQte|Qs61bEAb5*B7a3^FbY1%kP-C$~6j<)%AeN7_`Qew$ zy3zl+rb6UU&Uo=!nEG_%#ef{B!UTeKf+CKF!iMTx+yOouX|(~ISiEt#$=(S&D}%x_ zB@HY3&J-jTcI+T&A*9q1ru;r}FJ4OA{Xp}`!xr$CJQLXtU0e#TmahC;?YWEo-AbDI zV7ReBLR(5FHS{%Jc-_9kQ4}b3iD~4BP@?W1wcRB}bcAP0ETZdHP)SpKEk?qZWL0MN zt$38|N5#1czbL0Az)m#089ehVR_X%(dV74Iy?Uy6EBhl6&wEr0d%O26I7q0?0Kk8R g01yyWl&T9=vv>d%tQ`MOjVr6dh=hWS0QkTB52?EIng9R* literal 0 HcmV?d00001 diff --git a/crates/exav-unpack/tests/fixtures/rar_volumes/set5.part2.rar b/crates/exav-unpack/tests/fixtures/rar_volumes/set5.part2.rar new file mode 100644 index 0000000000000000000000000000000000000000..344e4d8c8c0f81632cf022f42da3f9032e6bfebc GIT binary patch literal 8192 zcmb7}V^<{%!-cb5lWp5JPBtgIrju*3&B;x+b+Vi6$vhd8Z9Vsw_XoUt?GM*K*y~zb z!_1X}5FQdj_9{pY1rin>8WIi?GPf@XLi2#|o&kyod8-!!rXUBhITT7l%JO#*G$f)E z(9X?@)!fbr2^!l|uGc=oOrI>$>&{6y?30^yV6J#@T!WPTUaOP-j-!M207cFp8Mw)_ z@Q&E;{tX9HnAx;)Qpo>|jeNtfIA5Vb_gcMMKQr+lHi79jb+whuXB;{(`$Bb-hLK#bjkU;tLr+~P! zo%MSD1q}cfj6!qhET%QXlcW$0`g1BC4$mZ=Wr?RJ5XpsOW1Cc<(U?A4lqRvc>Y z9Iydo`f5NKtvIftFEU2AW92Kg3oyPWgMr zH1gaGyK@zBC-DeEbD@Fn5HhiTHvxGmQe)pr&)Bw|&TMkY zX9ZPuR^R4dlX#zC;ky<%;qz@ww`EYc%L#k|kve?rNpDpII+3rf5>0|RTGi-Xa$aUz zXNg1Kx>}SD8xl1Kp!O4Qkp3h2Ugp(OWQfP8<}@m|4m0OZ9osi((HCOtF$Mh!DD;ti zm(qKmR9lp!=9XMIC>fpSB8E3w$WJ$utXvG$n515;yuaQY!1^B&pagTfG5ecL%)=SK zxOF+S3h2oFnl@H=<;-*QHX=EqZI!xvxYLEb`;KiY)qhzw5%-L-j-D>3xToMs#(??9 zUhHRcR?^hVw~6bh-BCo&<49hC&>k9_n}=9SDI}#@oz82!2RLSrh4ROgFj!&n?26mx z7qiR<=B+0e|7#^4aH_x-`s+~s@$Dz(43EJQqK}xh`$dW7-K4@jN(YO{BtTL<96VQ* zujkFnom}_@bC7Y4A;S2Wd$eQjGqAmnYFaIhdV2V&sT?(i`K}`cMhZqqRtmb~i*1}; zvX>yTzO=&|_AZ(c5{>Qv`o?*CoXNZiNYi;q92y~u)AlXG)~cj#0V$s@4GmDs=ikk5 zMSFK!u(~`cQKc?cD3NHki5)R~%M=ra_8wie`6wwaPw`X6p-(iV@3G%H;l^-TXaz?f zco#k8Y*L5@b|8MSI2`-8>cni=o3*;|p_A$}%t?W~);edH1@IwiQQUYuEa0(zYIb?W ze&X}foXaCSu_3rT%UOxW68JBq8w%86Al=ancgiXDAmKuDmA;%jJbj$+!6bw*duVTj z@;O8`(KY=o$($=?HrKn-_02tmCwRws!N713wg#I4^yRxw^osES7j@k z3KNje7-7ILYB-B)o2^vBj7Y675nGvZD}QqVg`K=2o%0PzB>UYPZEmbt^+t;9g;4Mb ziDO}Qm)r{L#5x_6lqX?Zc#SL+UC?3T>~=UCIFc;bLyC+^TYpN0#@(A~&^+>?Vlug( z=etVa0QA1oh+yAG&7Spt1%PI-W}#g~`?BW?FoNs!lVP);UKFh?@oYUT$%_RWQlQc4^;U4MS4hcLq@eL6}Y zB>;b=+dr15i2B7o{&53qO;MENrMK3R_eB2E0B)N~rcbr4GeE6Fxg6}CcP%bD z`Y)8Ahn??#nh0)Vvc0{i#RlX6O*$9Pn`hIY8RiKWwU&vb-~v8M7`!Px=EcXWjx{&N zK8rqJ4|sI$?regCmHY1rJ2W#F)&806pF}0v8o4t-#&T~zy>CRqMECiKY}Z`bfWQ4>FH; z)d>I4H!yUD)5$P2P4P);)%1sT@~_sSx*E<_8n>k4X&}mnb&wYyRU+0cp5Ho zEZ5ZP9h6c zgxWOxEK5dmo6ps2w&jzCR%4sg59FSfqtRA+4s@$;JHci)N`Kut+ZQhT7(+Mx#pvQe zJe2`Zn{kAjW>0UVb?zzQKtM>Jg~FQV+qu3iep3z!!g3@=xBh_#LSodJ)$NB^D7)5zo4Av7=bK9LjSV<}t??bK!qo1vel)#U1o3)I5sZhkSqtdz&{5k-49`$>%NfcD5gRQzP)@e65)!Yv8;|c*_%c}2LLPBf zU3+SGBYg0DDXs<+#isw-L;6+gP-_JJ=U1)KE&X`FH-6np-Ik7}N)nTLPiO3v!(^*8 zEDy*-7*k#`yiYQsql3)sp=?IMYP6SoN4Vj%mOqPxov#Am>S z38Q-SDSCS_nGQWC{VC^RG5@IkN{PehKT-HVML5ALmJ>}!aW4IF8H{AcH{zHx=?KRR zzxE|%4jj6nKY+RzWIlLy7F|)7nqf{8zE~Z#(_|s^dj?88ebiFb*hZo=?ZJKEM72umhM#DIXgtH`751KTNmX}P81PFi?3vbuO0$0%jK z^tr5N74Fnclw2$!6z!e(Zu35m%`;hv4G`Bx9KgoWcZx|8XEa;DOyqmWA2j?D9 z>{(qQn;<4o5FM^%Cpyr{@kZHO$pqqBRL&zNqoT8GT%o;o zYUbn6@gu`5n`$H$CVH&7)+dR4y2ulA6+CS+SCHj0?Qxo$?5(ljRDY)YqNSmX&wk}s z5@UD4%sIo!hb;0J4Dk2c_!lgYvw~Fb*3hup8rNa`ldDVyvCgu?1hD>-DKvkVzBJ5S z{+Lupr64uPMC5^L{8KYtS<@_On!mUfhIr6rw7V*AB=%?>7TTmg^IoS~_V*H6!meR2 zC?!@z_bl(?2W+~Y>**(x?O{P_Z2Uh7PR9>1oAglF^joBR>9(WlT5%|!=-xOMnpczJcWK%4L@W689kIcgQZvbo)aW??U@oen zV6Tf6r3AiRkt-v#lRKK#HSx1BR_H->YEhEVVz|~sYTN|IRcDDsOCa}c`P#Y9eF*sQ z`r-XrVaPnync4OqoudZA`2gOafga6RFO=G0(%DG!Gd;e26U07I?E^{19n2`Z$ zKs~Ak6wjga7R+O`;deVq9>K4`*@pNoIdbUpZaxN-8KhR@r}Q7Dx(5ccA7jt1i~&Dr zdE;H&G@1X(Vd{vUbuG$WYzu%9d&==^&$Q-P6G+lWvNDR=$gng<84O5LSDaV(6%phi z{<=$1!HnS}9o=EEd6M4v)7MPF4g{|)R26SfrM>0z^T9n7E#B%q$*{)|U!0}O;oi6+ zb*BF)#vT;p!^VqFkZgo)->$byR|~aqFg+N<<~`HbLy(^4YA;x#qT-G-j0@Rv5^YXv z)8Y}u_3z*AXD)=2kNvP>#VR)18RXb&G_uBTFyic1)WqFq&>A~!vq35-lyXf$Fi}|f znEk*$#1b4NGX3kAH}#g_12UVj^Ur8!sI+b_h zgoH8_Xr3j`R%x~M5ok1$1_;ig4Yrh5|62{PByd&C!w`G3MFX*EI zVi=%?lk~o8CLNVc#rJbvh%dc*oT)NdgTZFbxe2l&6_D|i6fg-s*{y|SlrJ$ZJ$Q8g z?pN{zFY{iCeT6E9G2Ni|hmubRF+~LjOP7G9^;vhxR47CiLbEd8&t4M3NXiux%vK0O9w-0|f4jjF@)CqRRiZ%tC{I*ryh-=$@>vdmN` z@jEhN#BhSUqbJ#h1gdLr_1qC&Nhu7U-ujG>L)YH~sF;0UHHr>LxSElYd^2RrXY9hter5as>R&P4H98$A1c~pP}w| zCo}2eH^pb_Aa9VWU^w3=4xaNTm8~SJ?Lr49Q&Zk#C_6shZWMOTDx~$j>H-2@&Bk62 z8R1(*@?Gw&g19Gx)Lqm2B|cM6;WP}e9McAIiY3a!X5P!g!dOu$D7gkq@ZzGz^b?0B z0>(XkmI&Q>b-2pib3mAvGHFo#8wkfB0}EbU0*c?R``@}Var8V&ah$1bCB-T=)wz!m zm7EIeT4YP{wgP31qbeP{+5JoC(!IJ(-*c->?R#X=(<_k8~9c%z!Ndqt~7_25|f2hs>9Eb2kd92 z9er-D0;2Gh0H6X+$L>uq1iYc`B3et(cAk!R2|rUZg0tV#yD8E<3YThFfyXkAXvHbQ zao&b@x%blgAd+7A<`C8MB<D$Wr+(F12{cF=}Zik4a3HSCKTLcIK3t-iki z@w+yu@>mOjQL(ml&b17NkXhh$*3c-dcs_ ziSF@7z!~>Kha1{%ez|87Qw53^o8b1E&FMkOGoYX7qfro6_V9@zZlcgKX-jz&#X?iL zpqsfaR9c`#!IJWa!sqlAgm-|Q~x!ECW>B{_gZ6j-w938B38D@^0(3IGe?>?*by2O9EN~94{j!TW$=0&6U2n zxM8RznM=$&^bR9l0j{#-AI$HY*L3p{J!iU?5U-M8>Tg5JWG2oX^QY$yt)WLlA9zlG z*5l0g`b#Q+P@HIT;BT6uO$E0zfj++(YDL4memy>AGi_a#(qB6H}6lSlLcln1JF!Je+f>w zQLPli$^7b&QL8YT{xz7OlLGH!ToX15-|yiVnH#Dm)?yk)GMGL$0_H2r{VW}ElZoHP_tTm7qm?DKeHzEA)pJ<&lyCz3d} z16#d7JW0_1469BpI|u&|mOdBiS!6S()sI|gU9%_ypb*>{?=3og2F_{ z{aevjrq}rMu!n)I>e*rGCO z>eGB*-Uugcizxtkoxr$SdD!4;H)c|J;al#$ad?%Qzsrxjp8>bLfa!Z@eaOr7Mqa9V z10*=_Iu7AW8`rZ3=C~ETF7WCtH$?8?GQoOH7QD!}N7&=^^{2NuedZjj@u>#}ibMD- zr3#peI5FT)BNzSPy*1iMYQ1cSw?XnftEt7r_Djn0Abrr)KCfAu5vACW(|WSoY(Ik2 zZZB=6(_JfB9OMCVA%mwO`U;q5%+^3NV|=7&YL$kvWAZ$lE3@3d(<=&t1Kau?v#0>) z&*oF2&Pf0+TwlFthvu5I&vZ{Pt)KxOyI1DNGjXK@9XI`b;JBD%1jQVm#UsO6){g^9 z+5cox-qtaYjc`6Taol(o)!LRYHor&%Wpp~1m)tE>^G*Vv>OIE~giaf{Yb9tIcdY0eiCG@e4UO0T^SWa| zt@>@j3PzH7kJPXu=I)651(sjXX9>fP!~H98BPD@#oT`&wn=OHz;Uc^&>Gmsbd-P@l zqb91>lO2*BQ4hk6Nf>pWQl%CHPp$g6mXN$xAr0TgGGa zRiH(xGD9v6Yomh`K7@#0L`^ysNVSy`g-ff1TQ@-#X(PEKEma<6#@+lktoi73J|_t0 zjJ@%b5xgDj~euI&SMb3HdEHw+OYck-yFeGEpu} z-@EQuwY}8K^|ED6CMB{-6!Fq{DE%}?mtuDmCu`wyi&)Gur&M;HV19jm`>krUK`6RR zDQd}LHWn_i&$c@Pj*ogAO+&5U{wa`P$@V*9LtFQre);Tqu91_XmH)s>CAEWI--Dhi zj^~Ri+(G?5t(Xa;5O-jUmGj|WVV{~#n!PyU&c613C+?Ww zwK>|skjK<==^B3wv@5!`AEo-%8NR4BCa$c|+h^q4eh7NZwGo*L4w?1d#(J~0AqykrF&(H^eCOrdKokJQf47Lc z0;R}8o@jh8w2dik0y7mXiomOI3lVjr+&iM9X#a0)=)H8Y_<1ofFbGJv*pQqwZ;y%z zG|a44lWH!q-I| zHx0AKbk}MYn0#xV%AI6D^$u@&Thh{hjMK zBP-K;f!8epmZlslyG!X%BmRU4JQ-E`c?cH(MgEUGdWWp>FRXP67Kv6LKkuU*4ak(* z$%morx|3TDmtP}3esW@)y2d=KdyN$ozkkkrs^uh*C*ah^TVw_9XHG@5zY)Y+8ZIX8 zb3>D{zS=8PzQMIBL8l}XHUtE=?tQJG+e+Hfh)N`e&;>#@kGFG=Lu4+R1NcWDml=t_ zP!v_~d&kX3Zl3EtfjgMF`%&!o$cj%k8kt~HBuo>G)){`STE`E+7jT;!yzFUesxluR zPRBJh_(FoRS6Uru&9(|1jyqBTU%9IrPllrfl8bc#mUwsw;XK-p~| zi8<^X0_Xlbs%IH6-7r??`b3@FXSv^#T-iY!O%IL1LuC_{na%^unuNM=O;T>@m8dE& zU&7{+d)s8qQF>G^WVUz>qeG_|`M^6oAP@4$ zW-MIi)EepjYp!FOup! zK_Eh8L7Xg*z10D*q0)6pO_JH}L8f!Higw@lkz5s>_#;grO3C~bjOnt6IN%arD}kaF z(wDKMLN(-)-qys>NiVFSjsZ*hIFlg{>N%eiBeRWc`6M{1#qRtq%0R;yG0tNvA)U$v9ISdduDFO=%m9qc@pqv4;g986% zTw;I&0SRtzX>(;RVrgy)0~2;fzG{YHSswx2c+>0}SR0XtmO|KGSOWHD++NezEb*C! zYJ!r0)w9u6+1<86o}*>=(2fz4+J-oqaH;t)CkXQ^^&zj|^=(VLqoR#^Md<&=Xo`me zLQ?ciVOn}dk`EZT9I-5>p^!7vc#?Zi8Nsc`sATN8Ee>NMv$%(Xpx3L1LF?@F(k}hY zQ;7`xGs6{9)Ep>rXOH@)MhOYzYZP5n)_n|Wi9i!m?NpBRO6g^$)q`e>T~k1zp%|Ve zFIySO%K%DA&ZO1B|?NcJs zv%6XCAIpQx#XMeXW(RQ8nN5}-k@u8H0UcF=cyBH^J9Vr)LF6slA{dGpHH;CMC%G1BdwL|+?DlXWTqZI4tiEUtkg~)a1V`e75;G_-UvQXDy;YM z7_09h<2;5^dS8lsP}Pau`tlAO|0iQ{#u#EGj~NCGP;vU!YEr;2_#bvZLkm$&zs@0L zj?iIj=dv?R0SY-JvJNktV-H1MWKSvq8dN1DjR%q3-K;OBL1xqn%vFnxgBun2g!Q#P z?7(|mtl^~bgsW6v{Nb<0M`>G?BU~q4dYa&tSC{Bjr@n4(1mgGg@fp#!%Z}9>!BFkA zw20pyI1VH*mCXKULu}5KX<&!;=24gGcfOgdnJ%$pe*sr+-^+zAU>WZH;AM&HQskVF zeY%}NCe4z#_TA7jAi!Q=$}XE`B@AT9J>F95a5852zp@{i$Jc_eLT%ILu@K2|hDeP1 z)GZUhHe&+4K4#q>4Y*`e{AL4YXM#C7kiI-) zVmgv%57|Vw32zUI(01TVF=|%eeNnk%1xljNx<>J=K($?ksHSHAZ-Ku^fhrjpR|2iRuz-{{Af=Fy%q4A<#u1}K#Vi!F~1S= z%%eJ;PpO3BV%=7#+t{5061)nJ{shVKC6A#QCDkLS4Xx2{%%nh7l#h|4#K!Cv9ajHu$4(4-W zcSdWuJkWO%J`6xY&8p3)Ra-^RP=3}yI{;K^SuM}G>NC-4dPn3p`;Xzeh-&UT?k8;g z2|0`GbGkgqAPL;-=pRM}!V$*WTjYS~O)7w-J4==N$3&>unx&CWL8zCUL+rg2yu1&r_v4jiNB@B}XyHV7d$yGuA37eSF&^#P*bq$;?- zFe*a_`g|QzeDwZV(Zz9R#@zOfsTTH(Sh_lRMGOUxkjigX*@Zv6vsRcp?B){T;r+9m zIEt1HA~ShD6az}7FU9v5EvR-+wJI;PhN0;!y8x!Xo19&5aD?0c5@r68^}422R!3wm zS&6v9Py#STuUeAW#p;(vTlkqK3c42xpy)Bj*#Ux~3bDN>Xgn2(bR(=;}>X>UDS zM9&skmMev|%f|}7hS-7q<#TXcw)<0Njjk6Edo>&^T3%!!0-EJcOXi*9{Bt70+#1BfBq>! z%<6a`;3g~hvJRQCtRj4|y~vKXjC_{T{Un>3wkv*_c z%ZF3eZztr^I8|IJxF06|_MzRYHaoGp42pLZoLozG{+Zjyl9ArJsntMIcGvG*$ zH(e8B-A1tb(f9i|&4$1Ost34MIN_Jr)GUPuhfvfZGOkLguslOr3hSvrT5yy?lVliS z-PKz&JO27qGN29!B4DpYFT}1mi=7Cndnc%mH!-lOSpd0x{}xyp50Ag?lSp=iyU`nq zRni|_h71Kjn|oRZ)51Jbu?nZOjC#0AsbTrpyKx%su$_O*aOj>^rBJD2d(+ed2Y0O|7A$p8F^lLf%rVa?rXMqbSGG`B86jr1 zT5Q4j60W{GFPLRPNFB}|Z^g}BYIuYy+G>(~05>&wSi`-O4!3e!D?Nyu>~pe~~J2z+qV`oL&}A!1B!PnfT&Qo-DYB^eAv(8mpp_q&uhw-Cz(pb^37ae&S(rWB0Br9ZtvirD?dyne?&~(`CA7qNYG}? zH5QeuRgea)75I-&^}9Y;^OqgDS)@`VFaT<(P_Z6%DY;f^%sO*4*3*AR^h4QnG?iy; zS8w7EbV~wkc7J{PF;Ab(yJmBlFhC1JLt%OZV!CyLAIKaUd>U%L_xEykJkNg4Lz*1ij106wvV! zR%zY>h>pnR_BBwUw{-r1RBZgj8K9t2?{4s3BC!{{qxO`6$=|xDf=OqHIAgC%9BPKr z=KAEh-)kSlAK-~8uqK(pF>Ct`>I{e~q`n<9PZM~2=D-L-MwK7Lh7Ph=T|{jspj*km zb}8V9=yDcM_QOvOq6e{%h=KuP zHIH|}-GWb$jw&cs0^SY8IEgZuUsP{H?sw&+QGmIZI&4WTackXvttR^p6zK3DJ8x0s zo~)q=%g2sXaA3$-7f17M-IBAu7O%kRgDJLVpti?QguQe2vxb4_U^X$R+)&KPQ4R(3 zF*fryRVi9g3mDkO;QS>_37$%7XNG6%)K+m2if8>yd$V!=_A4~AC28+HqbD<5?;6#U$}T=toiGAhrB?n$-+sMt zm5bl%XhPLxs|v^q88#@hEM^sjDEWDR$}|E!BycR1;@ma-$isk}tQ&W@y~$}ZrRkyd?e|ht4#gRO^q8a3ZVAyquf28O$Cej{wOng zPKqtQD<5ur>O|6+)YkWI5dPGz>p(V7^QDKJ&ifG9C~QN@zKeQ!)Dk%5dj;(km}ylC zG+fH*o3`XJs?FaArDA{p@y>3zg9{y)dOb1W#wCkuib9Sr#vF$I`oza_9j*Wlz+_sO z{^_6~s_gTW&Gq1Rj*x zSBj!d1@R}h(GRNr(1Ar z30Um?>)61Fw6$sB!G{~J)NB4;(1Fd>C8?1=*mIrmi(_PvP}6+9yfV~DS45JOVYV&7 z+df;nGQ03VXuT`jJXN1~V*rTTfX;!r*Gl_;N#UisPf!kiZ=qY2>H&NTIuXiUj4LtjgO=_ z_Yv~w{!Rw;REHO#Hlya&tV9@frE~vy5aqu`32s=A=I$gG9=<|PPi#*A2 zT%Y_E4Lw_b(%T&V+TBlkGN9z;`zaX=C}N0&QPWYZuW0|X&g;az;zuStey54Qz!o71 zd)Czta=KB1gbPY-f9Y1qGt~o+FUR#wl0jTDu4wN^jb^HkBdh8J#eZH*JJ4$xa^VkNR=>lew7SoU0cZTMC6-PXU$M=yIe@Ui<~6;>m`7=URa4LnPZ%dH;IcNP zRDDWACm^md3RvR6x@j^t@#TE?$OAgIZiy@Dm+U#jucx<*#cDCe||fQNm~>rYyr$?)@bvl%mg<-$ZxL z@qfc2Kc7aeRo-Vuk{P&)j1Hd4E0`8OjUw~o3`BU0GD5Mtc;38l9Gti z;jSc?@0|q3I%WW@PlABZm${1duxOKC%yzN zwcxJG6fC6_ndgLavm{1J+19Shx~BC^9(bs%fw}P&Hl^07tDSz{&o8Zm*!xgQ8lpiv zjOEZu-CkSf6f`FBM%k-dwzxl3ROl*KTU6fCtEb+m4Qt2E@^4&=?-EH*vbqr9>Ma5$ zdjOVx7-m!1T4G=_U&Fdxx1 z)?bAUsS_}3x>Q$JRs{eSLZ&t8j2%&Rf_W&F`LEAwk$HIUJx*@v@{!3^FRf_!P<#gx z&QzY2R$-E?hG85&2jkLscfIO*YCT^S3?$O7Gj8+o!$#XJ!*_e-?eWpArk{$@HHVUx z;ens0Fi0Vq&hADvh%tPJi2sGvO2TseRLzFJEP-@6kEG*T3-drd)#xTVa zl2Yb~&r!dq;K2~x&9I-^s2s@B1V_``1xfWXvThA>@P0wIO>^AkcR=;3Huo}Id6N)d z_o=SuX{QmvsHwD##zm`B^{aG0Odx1NAnY3?iD?Lis7#qQjBG$-L8g{zwB$U2RDPWT zoGzV}9@_VZJ(%mhMmaD( z%)`Hpzao^L0RMB59LM-S~6yCjKNp&AltDmtmUKdD`f?{R+LWjV`qvlMbl3MtutHfGw@DY?pZa9mO zi0=HhgMO}>f0NesUe{RSXcN|KML6=*F!wel7qS+5XILL%51hOCFFy`A)=Xoj`=Niv zT3q>`gnPWJzAC5OV$%uVEu4=XUrEbb2~EVp=2#9q?F=^!kDlxw2a4P;)!e`5A7PXy zizX&V6-2`>We>-GvqH<;VDH#tfWd;*8I^D#<*I%|09*fa)d!W#J!^ZV=G@6}$I~^b zY~Xr!2!l+7IK#5-+53m=50(}&LNPy}5~gn)zDpA(OZ?cZ8y5l}y!BI3;~recjP)Oj zqte=!#!M%9?4kLe+PdESf{?Q1Ayj*4biB3;PYI$T3ii99vIe5wE>G?B+_7BVl2cfd z`RKGzLwump(%u{E*Q>JDe8#Gs@dV~+Gpo{3Jo`3aN$F%bb~|s!=*S;eIC5gsu%XWaBRiLw);7DIEE&uT`6qtZG=ucRw@~$+j%ksf z3`fQOHw3qLx0$+)BpL@xlBm~lNp2%5cnU#NAvdA_a*(>XGZvJuSmH17x_i%F%Zeb< zZitTdmSHXtMrm9j`Z!nJ-3jNL9V^(2(jky9RLH7sRCC_DXnvEN4r{~DVIm85#k|!9 z!tX6AX*WTNFMxP-Q;5hMoD@7vPda@To(2FR`Lw zCqhpZ29wepDJgZHeaRC2WxsY)|I4(RYG5{8KEU27i@i01u$fbAWliss7N z-|)1GG*z&85u`r$LilV!n<5SoPOqhWIpf~sRl{g1`b%rC;+6iSM_9TNy;C@CcIFiF zp#1lXd}^xsaIglGZNQQ#oR#_4ezuqD{Qq@A+?(q6|3Q-$H=gRO>2R5&t~Qqlo8FT5dUKCeTt*fQ2a`D(PW(?!%UQ+zV=CRGQc)^xjtAPbarHIkXn&w_ zZBtW$fFenU}ULq)d^l}SqasDA_?!O-%4`OmtfrpXmyFjtlSfM6(jf8a_VqbUOfgJ%)n zY8WU01mFhSl@uIpIrs;K_Jm!xm`UBA|Ep_8bmMeT?<@ya1jF_0^LvJ_T1ewQMBell zl?TIbZz!fx9tZj5ocEnl66v5qrL|aTCf$KMyl|^$8J#pC=}%@YD}0q5jFTT7Ze0M~ zbTXe|q1ELezhQe+5#dMR-YP)*K~x}YYaA*L0|vJM0Ji`DfB*pkQBMiT8X__P!!s^9 z3=lXe0t*Y3vj7C3oB_0h0{>=QVt@ky32tv`b7d}KX>JMw6Lv?wYKCE1A02mAQ3C}8 E00+YImjD0& literal 0 HcmV?d00001 diff --git a/crates/exav-unpack/tests/suites/mod.rs b/crates/exav-unpack/tests/suites/mod.rs index ac87e7b7..3b67a918 100644 --- a/crates/exav-unpack/tests/suites/mod.rs +++ b/crates/exav-unpack/tests/suites/mod.rs @@ -142,6 +142,8 @@ pub(crate) fn extract_each( comp_size: meta.comp_size, encrypted: meta.encrypted, unsupported: meta.unsupported, + mtime: meta.mtime, + mode: meta.mode, }; visit(entry, budget) }) diff --git a/crates/exav-unpack/tests/suites/rar_solid.rs b/crates/exav-unpack/tests/suites/rar_solid.rs index ad249883..e5951fdd 100644 --- a/crates/exav-unpack/tests/suites/rar_solid.rs +++ b/crates/exav-unpack/tests/suites/rar_solid.rs @@ -179,3 +179,77 @@ fn a_member_that_decodes_to_the_wrong_bytes_is_not_passed_off_as_content() { "corrupting the stream must leave something reported unreadable" ); } + +/// `sha256sum` of the inputs of the `rar_volumes` sets. +const TEXT: &str = "306e49d40e25318b97ad7628011238f0ed6af09880396518cd5a8e1cbe7efe19"; +const NOISE: &str = "6987decf3255b53b0f8cf0da45aafacce4294578716058e11cd018a18e57035e"; + +/// Sets written by official RAR, each with `noise.bin` split across all three +/// of its volumes, joined and decoded. The inputs are made with: +/// ```python +/// r = random.Random(20261001) +/// open("text.txt", "w").write("".join(f"line {i}: the quick brown fox {r.randrange(10**6)}\n" for i in range(400))) +/// open("noise.bin", "wb").write(bytes(r.randrange(256) for _ in range(20000))) +/// ``` +/// and the sets with RAR 7.23 (RAR5, which ends every volume with a +/// quick-open record) and RAR 6.12 (RAR4, old-style names): +/// ```sh +/// rar a -ma5 -m3 -v8k -ep set5.rar text.txt noise.bin +/// rar a -ma4 -m3 -vn -v8k -ep set4.rar text.txt noise.bin +/// ``` +#[test] +fn a_real_volume_set_joins_into_its_archive() { + for parts in [ + ["set5.part1.rar", "set5.part2.rar", "set5.part3.rar"], + ["set4.rar", "set4.r00", "set4.r01"], + ] { + let vols: Vec> = parts + .iter() + .map(|p| fixture(&format!("../rar_volumes/{p}"))) + .collect(); + let refs: Vec<&[u8]> = vols.iter().map(Vec::as_slice).collect(); + let joined = + exav_unpack::join_rar_volumes(&refs).unwrap_or_else(|e| panic!("{}: {e}", parts[0])); + // Each RAR5 volume's quick-open record (a service header named `QO`, + // its name length before it) indexes that volume alone, and is left + // out of the join. + let qo = |b: &[u8]| b.windows(3).filter(|w| w == b"\x02QO").count(); + if parts[0].starts_with("set5") { + assert!(vols.iter().all(|v| qo(v) == 1)); + assert_eq!(qo(&joined), 0); + } + let mut got: Vec<(String, String)> = members(&joined) + .iter() + .map(|x| { + assert!( + x.unsupported.is_none(), + "{}: {} {:?}", + parts[0], + x.name, + x.unsupported + ); + (x.name.clone(), sha256_hex(&x.data)) + }) + .collect(); + got.sort(); + assert_eq!( + got, + [ + ("noise.bin".to_string(), NOISE.to_string()), + ("text.txt".to_string(), TEXT.to_string()) + ], + "{}", + parts[0] + ); + // Each volume alone reports the split member, never passes it over. + for v in &vols { + assert!( + members(v) + .iter() + .any(|x| x.name == "noise.bin" && x.unsupported.is_some()), + "{}: a lone volume", + parts[0] + ); + } + } +} diff --git a/crates/exav-unpack/tests/suites/zip_disguised_members.rs b/crates/exav-unpack/tests/suites/zip_disguised_members.rs index 158a9b35..59abd848 100644 --- a/crates/exav-unpack/tests/suites/zip_disguised_members.rs +++ b/crates/exav-unpack/tests/suites/zip_disguised_members.rs @@ -123,7 +123,7 @@ const PAYLOAD: &[u8] = b"payload-that-must-be-scanned"; /// /// From a live JAR: `kingDavid/9.class/` held a real compressed Java class. The /// JVM loads it by name; `unzip`, python's `extractall` and the `zip` crate all -/// call it a directory and discard it — so the malware inside was never scanned. +/// call it a directory and discard it, so the malware inside was never scanned. #[test] fn slash_named_member_with_content_is_extracted() { let blob = zip_stored(&[("pkg/9.class/", PAYLOAD, 0, crc32(PAYLOAD))]); @@ -170,12 +170,12 @@ fn genuine_directory_entry_is_still_skipped() { ); } -/// A member flagged encrypted whose bytes are in fact cleartext — proven by the -/// CRC-32 in its own header — must be scanned, not reported password-protected. +/// A member flagged encrypted whose bytes are in fact cleartext (proven by the +/// CRC-32 in its own header) must be scanned, not reported password-protected. /// /// From live APKs: the packer sets bit 0 on *every* member (Android's ZIP reader /// ignores it), so scanners decline an archive the platform installs happily. -/// Reporting it is not good enough here — the report is exactly what the packer +/// Reporting it is not good enough here: the report is exactly what the packer /// is buying. #[test] fn lying_encryption_flag_is_seen_through() { @@ -197,13 +197,65 @@ fn lying_encryption_flag_is_seen_through() { ); } +/// Typing a ZIP that is not held in memory, then walking it without reading its +/// members, reads its start, directory and headers, not the members' bytes: a +/// listing over a network or browser reader must not fetch the whole archive. +#[test] +fn listing_a_zip_through_a_reader_skips_its_members() { + use exav_unpack::source::BlockCache; + use std::cell::Cell; + use std::io::{Cursor, Read, Seek, SeekFrom}; + use std::rc::Rc; + + struct Counting(Cursor>, Rc>); + impl Read for Counting { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.0.read(buf)?; + self.1.set(self.1.get() + n); + Ok(n) + } + } + impl Seek for Counting { + fn seek(&mut self, to: SeekFrom) -> std::io::Result { + self.0.seek(to) + } + } + + let (one, two) = (vec![1u8; 1 << 20], vec![2u8; 1 << 20]); + let blob = zip_stored(&[ + ("one.bin", &one, 0, crc32(&one)), + ("two.bin", &two, 0, crc32(&two)), + ]); + let read = Rc::new(Cell::new(0)); + let cache = BlockCache::new(Counting(Cursor::new(blob.clone()), read.clone())).unwrap(); + assert_eq!(exav_unpack::detect(&cache), Some(Format::Zip)); + let mut names = Vec::new(); + let walked = walk( + Format::Zip, + &cache, + &mut Budget::new(Limits::default()), + &mut |meta, _, _| { + names.push(meta.name.clone()); + None::<()> + }, + ); + assert!(walked.is_ok(), "{walked:?}"); + assert_eq!(names, ["one.bin", "two.bin"]); + assert!( + read.get() < blob.len() / 2, + "listing read {} of the archive's {} bytes", + read.get(), + blob.len() + ); +} + /// The guard must not swing the other way: a member whose contents do NOT match /// the declared CRC is truly encrypted (or corrupt), and is still reported /// rather than handed over as though it were cleartext. #[test] fn real_encryption_is_still_reported() { // Plausible ciphertext: bytes that are not the payload, with the CRC of the - // plaintext — exactly what a real encrypted member looks like from outside. + // plaintext: exactly what a real encrypted member looks like from outside. let cipher = b"\x9f\x2a\x71\xc3\x04\xde\x88\x10\x55\xab\xcd\xef\x01\x23\x45\x67"; let blob = zip_stored(&[("secret.bin", cipher, 0x0001, crc32(PAYLOAD))]); diff --git a/crates/exav-update/Cargo.toml b/crates/exav-update/Cargo.toml index a7ffa767..45e58284 100644 --- a/crates/exav-update/Cargo.toml +++ b/crates/exav-update/Cargo.toml @@ -15,13 +15,15 @@ description = "Standalone signature-database updater for exav: fetch ClamAV-form # Deliberately isolated: this is the ONE place a TLS stack (ureq → rustls → ring) # enters the tree, kept out of exav-core so the scanning engine stays pure-Rust. -# ureq 2.12 bundles webpki-roots, so HTTPS works on a scratch/distroless image -# with no OS certificate store. +# ureq bundles webpki-roots, so HTTPS works on a scratch/distroless image with +# no OS certificate store. [dependencies] -ureq = "2" -# Verify a downloaded `.exavdb`'s embedded SHA-256 before installing it, so an +ureq = "3" +# Verify a downloaded `.exavdb`'s embedded CRC-32 before installing it, so an # interrupted/corrupt download is rejected rather than clobbering a good cache. -# Same hash exav-core stamps into the cache; RustCrypto, no C/asm. +# Same checksum exav-core stamps into the cache. +crc32fast = "1" +# The tag that keeps two sources' same-named files apart. sha2 = "0.11" # docs.rs builds with every feature on, so the parts gated behind a feature diff --git a/crates/exav-update/NOTICE b/crates/exav-update/NOTICE index 97811f32..bfb76b76 100644 --- a/crates/exav-update/NOTICE +++ b/crates/exav-update/NOTICE @@ -1,483 +1,7 @@ -exav — THIRD-PARTY NOTICES -========================== +exav-update: third-party notices -exav itself is licensed under the MIT License (see the LICENSE file). +This crate is part of exav and contains no third-party source. The notices +for the components exav links are in the NOTICE file at the root of the exav +repository, which also ships in every release archive: -The exav binary statically links a number of third-party Rust crates. Those -crates retain their own licenses; the binary distribution must carry their -notices. This file enumerates the major components and reproduces the notice -texts that the BSD-3-Clause and Apache-2.0-WITH-LLVM-exception licenses require. - -This is a hand-maintained summary of the principal components, not a byte-exact -dump of every transitive crate's license. To regenerate a complete, per-crate -attribution list from the resolved dependency graph, install one of: - - cargo install cargo-about # then: cargo about generate about.hbs - cargo install cargo-bundle-licenses - -and run it against this workspace. `cargo deny check licenses` (run in CI) -confirms every dependency resolves to a permissive license on the allowlist in -deny.toml. - -Versions below reflect Cargo.lock at the time of writing (last verified -2026-09-09). - - -------------------------------------------------------------------------------- -1. yara-x — BSD-3-Clause (source derivative, and a development dependency) -------------------------------------------------------------------------------- - -exav's YARA engine is native, in `crates/exav-core/src/yara/`. Parts of it are -derived from yara-x's source: the module implementations (`pe`, `elf`, `dotnet`, -`math`, `hash`, `string`) and the compiler's base64 handling, including field -semantics and the RVA-to-offset and import-hash routines. Each such file carries -a per-file attribution header, and the license text is reproduced below and in -`crates/exav-core/LICENSE-YARA-X`. - -`yara-x-parser` is a regular OPTIONAL DEPENDENCY, enabled by the default-on -`yara` feature of `crates/exav-core`, so a default build does link it. No other -yara-x crate is a dependency of any kind. - -The differential test that compares exav's engine against yara-x -(`crates/exav-core/tests/yara_difftest.rs`) invokes yara-x as a PROGRAM — the -`yr` binary from `PATH` — rather than linking it, and skips when that binary is -absent. - -Components: yara-x-parser 1.18. -Upstream: https://github.com/VirusTotal/yara-x -Copyright (c) VirusTotal and contributors. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other BSD-3-Clause components linked by exav (same 3-clause terms above, with -their respective copyright holders): subtle, tlsh2. - - -------------------------------------------------------------------------------- -1b. libarchive RAR5 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR5 (unpack50) decompressor in -crates/exav-unpack/src/formats/rar5_unpack.rs is a Rust port of the -decompression core of libarchive's archive_read_support_format_rar5.c. As a -source derivative, BSD-2-Clause requires retaining the upstream copyright and -licence text reproduced below. - -Upstream: https://github.com/libarchive/libarchive -Copyright (c) 2018 Grzegorz Antoniak (http://antoniak.org) -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1c. nwaples/rardecode RAR3 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR3/RAR4 (unpack29) decompressor in -crates/exav-unpack/src/formats/rar3_unpack.rs is a Rust port of the -decompression core of the Go package nwaples/rardecode. As a source -derivative, BSD-2-Clause requires retaining the upstream copyright and licence -text reproduced below. - -Upstream: https://github.com/nwaples/rardecode -Copyright (c) 2015, Nicholas Waples -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1d. ppmd-rust PPMd7 (var.H) model — CC0-1.0 OR MIT-0 (source derivative) -------------------------------------------------------------------------------- - -The PPMd variant H decoder in crates/exav-unpack/src/formats/ppmd7/ is vendored -and adapted from the `ppmd-rust` crate (version 1.4.0), generalised over a range -decoder trait so it can be driven by either the 7-Zip or the RAR range coder. -`ppmd-rust` is dual-licensed CC0-1.0 OR MIT-0 (public-domain-equivalent) and is -itself a Rust port of Igor Pavlov's 7-Zip PPMd7, which implements -Dmitry Shkarin's public-domain PPMd var.H (2001). It does NOT derive from -UnRAR/RARLAB. - -Upstream: https://crates.io/crates/ppmd-rust -License: CC0-1.0 OR MIT-0 (https://spdx.org/licenses/CC0-1.0, - https://spdx.org/licenses/MIT-0). Both place the work in (or treat it - as) the public domain; no attribution is legally required, and this - entry is recorded for completeness. - -The RAR-specific framing wired into crates/exav-unpack/src/formats/rar3_unpack.rs -(the PPMd block header: order/sub-allocator-memory/escape bits; the RAR -Subbotin carry-less range decoder in ppmd7/rangedec.rs; and the escape-symbol -handling for the PPMd↔LZSS conversion) is reproduced from the public format -description in libarchive's BSD-2-Clause archive_read_support_format_rar.c / -archive_ppmd7.c (already attributed in section 1b above), NOT from UnRAR. - - -------------------------------------------------------------------------------- -1e. dmg-core UDIF reader — Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The UDIF/DMG block reader in crates/exav-unpack/src/formats/udif.rs is vendored -and adapted from the `dmg-core` crate (version 0.1.2), changed to use -`lzma-rust2` instead of `lzma-rs` for XZ decompression. `dmg-core` is licensed -Apache-2.0; the Apache License requires retaining the attribution notices and -reproducing any upstream NOTICE text. - -Upstream: https://crates.io/crates/dmg-core -Copyright (c) Albert Hui / SecurityRonin. -Licensed under the Apache License, Version 2.0. You may obtain a copy at -http://www.apache.org/licenses/LICENSE-2.0 (see also the Apache-2.0 terms -referenced in section 2 below). Upstream ships no separate NOTICE file. - - -------------------------------------------------------------------------------- -1f. rust-cab CAB reader — MIT (source derivative) -------------------------------------------------------------------------------- - -The CAB (Microsoft Cabinet) reader in crates/exav-unpack/src/formats/cab_parse/ -is vendored and adapted from the `rust-cab` crate — write support, the datetime -module, unused public types, and tests removed; bounds checks added for -folder data-block and short-read cases. Licensed MIT; the MIT License requires -retaining the copyright and permission notice (reproduced in the exav LICENSE -file) in distributions. - -Upstream: https://github.com/mdsteele/rust-cab -Copyright (c) Matthew D. Steele . - - -------------------------------------------------------------------------------- -1g. unarc-rs ARJ reader — MIT OR Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The ARJ reader in crates/exav-unpack/src/formats/arj_parse/ is vendored and -adapted from the `unarc-rs` crate — multi-volume support, volume providers, and -external error types removed; wired to `delharc` for LH6 decompression. -Dual-licensed MIT OR Apache-2.0; the MIT or Apache terms apply at the -distributor's choice (both reproduced/referenced elsewhere in this file). - -The ZOO structures in crates/exav-unpack/src/formats/zoo_parse.rs are vendored -and adapted from the same crate, for the same reason the ARJ reader was: linking -it brings in a second `zip` with default features, and Cargo's feature -unification then enables the `zip` crate's own LZMA/bzip2/deflate64 decoders -inside exav, bypassing exav's per-member budget. Adapted to exav's error -handling, with the variable-length part of a directory entry (long filename, -directory name) actually read. - -The ZOO test archives in crates/exav-unpack/tests/fixtures/zoo/ (`store.zoo`, -`default.zoo`, `high_per.zoo`) are that project's own test corpus, redistributed -under the same terms. They hold one identical member under the three -compression methods, which is what lets the codecs be checked against stored -bytes rather than against themselves. - -Upstream: https://github.com/mkrueger/unarc-rs -Copyright (c) Mike Krüger. - - -------------------------------------------------------------------------------- -1h1. unshield / explode — MIT (linked dependency + test fixtures) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/ishield_z.rs reads InstallShield `.z` archives -through the `unshield` crate, whose only real dependency is `explode` — a leaf -decoder for the PKWARE Data Compression Library "implode" codec those archives -use. Both MIT. - -The fixtures in crates/exav-unpack/tests/fixtures/ishieldz/ are that project's -own examples, redistributed under the same terms. `undhr.z` is shipped alongside -`undhr.md`, the uncompressed original, which is what lets the decoder be checked -against the real file rather than against itself. - -This is a DIFFERENT format from the InstallShield `ISc(` InstallScript cabinet, -whose only implementation is the LGPL `unshield` C project — never read, and the -reason that format stays recognised-but-unopened. - -Upstream: https://github.com/agrif/unshield , https://github.com/agrif/explode -Copyright (c) Aaron Griffith. - - -------------------------------------------------------------------------------- -1h2. iced-x86 — MIT (generated opcode tables) -------------------------------------------------------------------------------- - -crates/exav-x86/src/generated.rs holds the SIMD, system and VEX regions of the -x86 opcode maps. It is not hand-written and not transcribed from a manual: it is -emitted by crates/exav-x86/tools/gen-tables, which decodes a probe for every -(prefix, opcode, /digit, mod) combination with `iced-x86` and records what came -back. The tables are therefore derived from `iced-x86`'s data, and the MIT -License requires retaining its copyright and permission notice (the MIT text is -in the exav LICENSE file). - -`iced-x86` is also the oracle that exav-x86's differential tests and the -`x86_decode` fuzz target compare against — including the decoding that -exav-pe-emu drives. It is a dev-dependency only: the decoder exav-pe-emu uses at -run time is exav-x86, and no iced code reaches a shipped artifact. - -Upstream: https://github.com/icedland/iced -Copyright (C) 2018-present iced project and contributors. - - -------------------------------------------------------------------------------- -1h. oletools test sample — BSD-2-Clause (test fixture) -------------------------------------------------------------------------------- - -The Excel 4.0 macro test fixture -crates/exav-unpack/tests/fixtures/xlm_excel4_sample.xls is a benign sample -redistributed from the `oletools` project's test corpus, used to verify the XLM -macro-sheet surfacing in formats/xlm.rs. Licensed BSD-2-Clause. - -Upstream: https://github.com/decalage2/oletools -Copyright (c) Philippe Lagadec. - - -------------------------------------------------------------------------------- -1i. aPLib reference (depack) — public domain (freeware) -------------------------------------------------------------------------------- - -The aPLib depacker in crates/exav-unpack/src/formats/aplib.rs is a clean-room -implementation of the published aPLib on-disk format. The bit/gamma reader -follows the algorithm published in the freely-distributable aPLib package's -reference `depack.c` by Jørgen Ibsen, which the author released for unrestricted -use ("free to use"). No attribution is legally required; this entry is recorded -for completeness. It does not derive from any GPL source. - -Upstream: https://ibsensoftware.com/products_aPLib.html - - -------------------------------------------------------------------------------- -1j. Microsoft Script Encoder decode tables — public domain -------------------------------------------------------------------------------- - -The substitution/decode tables in crates/exav-unpack/src/formats/screnc.rs are -the published Microsoft Script Encoder (`#@~^`) constants, as documented in -Didier Stevens' public-domain `decode-vbe.py`. Recorded here for completeness; -no attribution is legally required and it does not derive from any GPL source. - -Upstream: https://blog.didierstevens.com/programs/ - - -------------------------------------------------------------------------------- -1k. AutoIt-Ripper — MIT (reference implementation) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/autoit.rs implements the compiled-AutoIt3 script -layout — the EA05 record structure, the Mersenne-Twister keystream and its seeds -— from the MIT-licensed AutoIt-Ripper reference together with the public format -description. - -Upstream: https://github.com/nazywam/AutoIt-Ripper -Copyright (c) nazywam and contributors. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - - -------------------------------------------------------------------------------- -1l. NSIS format headers — zlib -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/nsis.rs implements the installer layout — the -firstheader and block-table structures — from NSIS's own zlib-licensed format -header `Source/exehead/fileform.h`, together with public reverse-engineering. - -Upstream: https://github.com/kichik/nsis -Copyright (c) Nullsoft and contributors. - - This software is provided 'as-is', without any express or implied warranty. - In no event will the authors be held liable for any damages arising from the - use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it freely, - subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a - product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - -------------------------------------------------------------------------------- -1m. sevenz-rust2 — Apache-2.0 (design reference) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/sevenz/ takes its reader structure from -sevenz-rust2, re-implemented natively so the crate can keep -`#![forbid(unsafe_code)]` and use exav's own PPMd7 model. - -Upstream: https://github.com/hasenbanck/sevenz-rust2 -Copyright (c) the sevenz-rust2 authors. - -Licensed under the Apache License, Version 2.0. A copy is available at -http://www.apache.org/licenses/LICENSE-2.0. The license requires retaining the -copyright, patent, trademark and attribution notices from the source, and -reproducing any NOTICE text the upstream distributes. - - -------------------------------------------------------------------------------- -2. ISC-licensed components -------------------------------------------------------------------------------- - -ring 0.17.14 (Apache-2.0 AND ISC), rustls 0.23.41, rustls-webpki, untrusted. -These carry ISC permission-notice terms; reproduce the copyright notice and the -permission/disclaimer notice in distributions. - - -------------------------------------------------------------------------------- -3. MIT / Apache-2.0 dual-licensed components (the bulk of the graph) -------------------------------------------------------------------------------- - -The majority of exav's dependencies are dual-licensed "MIT OR Apache-2.0" (or a -superset including Unlicense / 0BSD / Zlib), so the MIT or Apache terms apply at -the distributor's choice. Notable examples: - - aho-corasick, memchr, daachorse pattern matching / Aho-Corasick - serde, rmp-serde serialization - clap CLI parsing - sha1, sha2, md-5, digest (RustCrypto) hashing - flate2, miniz_oxide, bzip2, zip decompression (sevenz-rust2 is dev-only: - it builds 7z test fixtures; the shipped - 7z reader is native, §1m) - lzfse_rust, delharc, lzxd, DMG/LHA/CAB codecs - bitstream-io, stringprep - cfb OLE/CFB compound-file reader - fatfs, ext4-view FAT and ext2/3/4 filesystem readers - walkdir directory traversal - regex path-filter regexes, and the search - engine behind exav-grep - libc syscall bindings (daemon) - ureq, rustls, webpki-roots HTTPS for the signature updater - wasm-bindgen, js-sys, web-sys, WebAssembly/JS bindings — these reach - wasm-bindgen-futures the published npm artifact - -MIT-only components (the MIT copyright and permission notice must be retained; -the MIT text is in the exav LICENSE file): cfb, hfsplus, apfs (the two latter -via DMG/HFS+/APFS support). - -RustCrypto crates are MIT OR Apache-2.0. Zlib-licensed components present: -miniz_oxide, tinyvec. Unicode-3.0: the ICU / unicode-ident -crates. Other permissive licenses in the graph: CC0-1.0 and MIT-0 (ppmd-rust via -sevenz-rust2), CDLA-Permissive-2.0 (webpki-roots, TLS root store), bzip2-1.0.6 -(libbz2-rs-sys). - -Format documentation consulted ------------------------------ - -Some decoders were written from format descriptions rather than from any code. -Recorded here because a decoder's provenance is part of what makes it auditable: - - EGG ESTsoft Corp., "EGG Format Specification, Version 1.0" — the - vendor's own published document. No ESTsoft code was used. - - AZO ESTsoft's own compression algorithm, absent from the EGG spec. - `crates/exav-unpack/src/formats/azo.rs` and `azo_tables.rs` are a - Rust port of the AZO decoder in `EggDotNet` - (https://github.com/akolman/EggDotNet), MIT, Copyright (c) Andrew - Kolman. ESTsoft's own UnEgg library was NOT used: its licence - forbids commercial use without approval and forbids using it to - develop a compression algorithm. - - HWP3 Preamble offsets taken from `java-hwp` - (https://github.com/ddoleye/java-hwp), Apache-2.0. - Copyright the java-hwp authors. Validated against - `testHWP_3.0.hwp` from the Apache Tika test corpus - (https://github.com/apache/tika), Apache-2.0, Copyright the - Apache Software Foundation — vendored as - `crates/exav-unpack/tests/fixtures/hwp3/tika_testHWP_3.0.hwp`. - - ALZ Structure cross-checked against `unalz` (zlib license) and - `unar`, used as external oracles rather than as sources. - -Full MIT and Apache-2.0 license texts are widely available and are reproduced in -each crate's source distribution; the exav LICENSE file contains the MIT text. - - -------------------------------------------------------------------------------- -Build-only tooling -------------------------------------------------------------------------------- - -Everything above concerns code that reaches a shipped artifact. The -documentation site under `www/` builds with an npm toolchain whose lockfile -includes packages under stronger terms, LGPL-3.0-or-later among them. None of it -is linked into, bundled with, or distributed alongside any exav binary, crate or -WASM module: it runs at build time to produce static HTML and is listed here so -that a licence scan of the repository has an answer rather than a finding. +https://github.com/sylvinus/exav/blob/main/NOTICE diff --git a/crates/exav-update/README.md b/crates/exav-update/README.md index 00a845c3..00d13dbc 100644 --- a/crates/exav-update/README.md +++ b/crates/exav-update/README.md @@ -31,8 +31,9 @@ changed. The body is validated before it can overwrite a good file, and the install is atomic (write to a temporary file, then rename), so an interrupted update leaves the previous database intact rather than a truncated one. -A downloaded `.exavdb` carries an embedded SHA-256 which is verified before -installation, for the same reason. +A downloaded `.exavdb` carries a CRC-32 of its contents, and its format +version, which are checked before installation, for the same reason. The +CRC-32 catches a damaged download, not a forged one. ## What it does not do diff --git a/crates/exav-update/src/lib.rs b/crates/exav-update/src/lib.rs index 657b994f..744f4bd8 100644 --- a/crates/exav-update/src/lib.rs +++ b/crates/exav-update/src/lib.rs @@ -2,7 +2,7 @@ //! //! exav bundles **no URLs**. The caller supplies them (`--sig-sources` and //! `--db-url`, or the variables behind them), so nothing here points at any -//! vendor by default — exav treats Cisco's "official" CVDs as **just three more +//! vendor by default: exav treats Cisco's "official" CVDs as **just three more //! URLs**, no different from any other feed. //! //! This crate is isolated on purpose: it is the single place a TLS stack @@ -14,7 +14,7 @@ //! validator (or, failing that, a byte-compare against the on-disk copy) decides //! whether anything changed; the body is validated before it can overwrite a good //! file; and it is installed atomically (temp + rename). There is **no** -//! digital-signature verification and **no** rsync — for GPG-signed or rsync-only +//! digital-signature verification and **no** rsync. For GPG-signed or rsync-only //! feeds (e.g. Sanesecurity), run `clamav-unofficial-sigs` into the signature //! directory and let exav read it. #![forbid(unsafe_code)] @@ -32,18 +32,20 @@ use sha2::{Digest, Sha256}; /// `.exavdb`). Rejects a server streaming an unbounded/oversized body. const MAX_FETCH: u64 = 4 * 1024 * 1024 * 1024; -/// Framing of an exav prebuilt database: `MAGIC(8) | VERSION(4) | payload | SHA-256(32)`. +/// Framing of an exav prebuilt database: `MAGIC(8) | VERSION(4) | payload | CRC-32(4)`. /// Mirrored from `exav-core::database` so this crate needs no `exav-core` dependency. const EXAV_DB_MAGIC: &[u8; 8] = b"EXAVDB\x00\x01"; +/// The format version `exav-core` reads and writes (its `database::VERSION`). +const EXAV_DB_VERSION: u32 = 4; const EXAV_DB_HEADER_LEN: usize = 12; // 8-byte magic + 4-byte LE version -const EXAV_DB_DIGEST_LEN: usize = 32; // trailing SHA-256 of the payload +const EXAV_DB_DIGEST_LEN: usize = 4; // trailing CRC-32 of the payload, LE /// Outcome of a conditional fetch. Both variants carry the current validator so /// the caller can persist it and short-circuit the next poll. #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub enum Fetch { - /// Nothing was written — the remote matched what we already have. + /// Nothing was written: the remote matched what we already have. Unchanged { /// The remote's current validator (`ETag`/`Last-Modified`), if it sent /// one. Persist it and pass it back to short-circuit the next poll. @@ -70,7 +72,7 @@ impl Fetch { } } -/// Standard base64 (with padding) — just enough to encode Basic-auth credentials +/// Standard base64 (with padding), just enough to encode Basic-auth credentials /// without pulling a base64 crate into this thin updater. fn base64(input: &[u8]) -> String { const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -160,19 +162,20 @@ fn split_basic_auth(url: &str) -> (String, Option) { /// A cache validator and the request header that may send it back. /// /// The two are not interchangeable. A `Last-Modified` date returned in -/// `If-None-Match` is not a well-formed entity-tag — unquoted, with spaces and -/// commas — so it can never match, and some CDNs answer a malformed +/// `If-None-Match` is not a well-formed entity-tag (unquoted, with spaces and +/// commas), so it can never match, and some CDNs answer a malformed /// `If-None-Match` with `400`. The effect is a full re-download on every poll /// against any origin that sends no `ETag`, or a hard failure. -fn validator_of(resp: &ureq::Response) -> Option { - resp.header("ETag") - .or_else(|| resp.header("Last-Modified")) +fn validator_of(resp: &ureq::http::Response) -> Option { + let header = |name| resp.headers().get(name).and_then(|v| v.to_str().ok()); + header("ETag") + .or_else(|| header("Last-Modified")) .map(|s| s.to_string()) } /// The conditional-request header a stored validator belongs in. /// -/// An entity-tag is quoted — `"abc"` or `W/"abc"` — and an HTTP-date is not, so +/// An entity-tag is quoted (`"abc"` or `W/"abc"`) and an HTTP-date is not, so /// the value says which it is and the stored form stays what the caller already /// persists. Sending a date in `If-None-Match` is the failure this avoids: it is /// not a well-formed entity-tag, so it can never match, and the poll re-downloads @@ -221,23 +224,24 @@ fn fetch_into( // `.exavdb` by a digest the same response supplied. The operator would see // "updated" and nothing else. // - // A source configured as plain `http://` still works — `https_only` refuses - // the DOWNGRADE, not the scheme — so an air-gapped mirror is unaffected. + // A source configured as plain `http://` still works: `https_only` refuses + // the DOWNGRADE, not the scheme, so an air-gapped mirror is unaffected. // `redirects` is pinned rather than inherited so a dependency's default // cannot quietly change how far this follows. let scheme = scheme_of(url); - let agent = ureq::AgentBuilder::new() + let agent: ureq::Agent = ureq::Agent::config_builder() .user_agent(concat!("exav-update/", env!("CARGO_PKG_VERSION"))) - .timeout(std::time::Duration::from_secs(300)) + .timeout_global(Some(std::time::Duration::from_secs(300))) .https_only(scheme == "https") - .redirects(5) - .build(); + .max_redirects(5) + .build() + .into(); let (url, auth) = split_basic_auth(url); // Credentials over cleartext are refused rather than sent. `https_only` // above covers the downgrade an attacker causes; this covers the one the // operator wrote, which is the easier mistake to make and the one nothing - // else here would catch — a `Basic` header is the password in base64, and a + // else here would catch: a `Basic` header is the password in base64, and a // fetch that succeeded looks identical either way. A plain `http://` source // with no userinfo still works, so an air-gapped mirror is unaffected. if auth.is_some() && scheme == "http" { @@ -248,9 +252,10 @@ fn fetch_into( )); } let url = url.as_str(); - let with_auth = |mut r: ureq::Request| -> ureq::Request { + type Request = ureq::RequestBuilder; + let with_auth = |mut r: Request| -> Request { if let Some(a) = &auth { - r = r.set("Authorization", a); + r = r.header("Authorization", a); } r }; @@ -258,7 +263,7 @@ fn fetch_into( // Only trust the "unchanged" fast-paths (the HEAD/ETag shortcut and the // conditional-GET `304`) while we still have the file they'd let us skip // re-downloading. If `dest` was deleted or truncated out from under us, a - // matching validator must NOT report Unchanged — otherwise the on-disk copy + // matching validator must NOT report Unchanged, or the on-disk copy // stays missing until the process restarts (which is what resets `prev`). So // gate both shortcuts on the destination actually existing. let have_dest = dest.exists(); @@ -279,18 +284,10 @@ fn fetch_into( let mut req = with_auth(agent.get(url)); if have_dest { if let Some(p) = prev { - req = req.set(validator_header(p), p); + req = req.header(validator_header(p), p); } } - let resp = match req.call() { - Ok(r) => r, - Err(ureq::Error::Status(304, _)) => { - return Ok(Fetch::Unchanged { - validator: prev.map(String::from), - }) - } - Err(e) => return Err(io::Error::other(e.to_string())), - }; + let resp = req.call().map_err(|e| io::Error::other(e.to_string()))?; // ureq surfaces a 304 as `Ok`; treat it as unchanged. if resp.status() == 304 { return Ok(Fetch::Unchanged { @@ -299,7 +296,9 @@ fn fetch_into( } let validator = validator_of(&resp); let expected: Option = resp - .header("Content-Length") + .headers() + .get("Content-Length") + .and_then(|v| v.to_str().ok()) .and_then(|v| v.trim().parse().ok()); // Refuse before allocating, not after. `Content-Length` was already parsed @@ -313,7 +312,8 @@ fn fetch_into( } } let mut body = Vec::new(); - resp.into_reader() + resp.into_body() + .into_reader() .take(MAX_FETCH + 1) .read_to_end(&mut body)?; if body.len() as u64 > MAX_FETCH { @@ -343,9 +343,9 @@ fn fetch_into( .and_then(|n| n.to_str()) .unwrap_or("download"); // The temp name carries this process's pid. A fixed name collides when two - // updaters share a data directory — a daemon plus a cron `exav --update`, or - // two daemons on one volume, both of which the README describes as supported - // — and `fs::write` truncates, so the loser's `rename` publishes whatever + // updaters share a data directory (a daemon plus a cron `exav --update`, or + // two daemons on one volume, both of which the README describes as + // supported), and `fs::write` truncates, so the loser's `rename` publishes whatever // mixture of the two bodies was on disk at that moment. Each body passed // validation on its own; the blend did not. let tmp = match dest.parent().filter(|p| !p.as_os_str().is_empty()) { @@ -357,7 +357,7 @@ fn fetch_into( }; // Written, flushed to the device, and only then renamed. `rename` is atomic // for the directory entry, but without the sync a crash can leave the entry - // pointing at blocks that were never written — a zero-length or partly-zero + // pointing at blocks that were never written: a zero-length or partly-zero // database where the README promises the previous one intact. Any failure // takes the temp file with it rather than leaving it for the next run to // trip over. @@ -373,9 +373,10 @@ fn fetch_into( Ok(Fetch::Updated { validator }) } -/// Verify a body is a complete, uncorrupted exav database: correct magic and a -/// trailing SHA-256 matching its payload (the stamp `exav-core` writes and checks -/// at load). Catches an interrupted/corrupt download before it is installed. +/// Verify a body is a complete, uncorrupted exav database this build can load: +/// correct magic and version, and a trailing CRC-32 matching its payload (the +/// stamp `exav-core` writes and checks at load). Catches an interrupted/corrupt +/// download, or one built by another version, before it replaces a good one. fn verify_exav_db(body: &[u8]) -> Result<(), String> { if body.len() < EXAV_DB_HEADER_LEN + EXAV_DB_DIGEST_LEN { return Err("truncated download (shorter than an empty database header)".into()); @@ -383,11 +384,22 @@ fn verify_exav_db(body: &[u8]) -> Result<(), String> { if &body[..EXAV_DB_MAGIC.len()] != EXAV_DB_MAGIC { return Err("not an exav database (.exavdb): wrong magic".into()); } + let version = u32::from_le_bytes( + body[EXAV_DB_MAGIC.len()..EXAV_DB_HEADER_LEN] + .try_into() + .unwrap(), + ); + if version != EXAV_DB_VERSION { + return Err(format!( + "unsupported database version {version} (this build expects {EXAV_DB_VERSION}): \ + rebuild it with a matching exav" + )); + } let (head_payload, trailer) = body.split_at(body.len() - EXAV_DB_DIGEST_LEN); let payload = &head_payload[EXAV_DB_HEADER_LEN..]; - if Sha256::digest(payload).as_slice() != trailer { + if crc32fast::hash(payload).to_le_bytes() != trailer { return Err( - "integrity check failed (SHA-256 mismatch): download corrupt or interrupted".into(), + "integrity check failed (checksum mismatch): download corrupt or interrupted".into(), ); } Ok(()) @@ -395,8 +407,8 @@ fn verify_exav_db(body: &[u8]) -> Result<(), String> { /// Pull a prebuilt exav database (`.exavdb`) from `url` and install it at `dest` /// only when it has changed. The body must carry the exav database magic and a -/// matching embedded SHA-256, so a corrupt/interrupted download is rejected and -/// the good on-disk database is kept. No signature verification — trust the +/// matching CRC-32 trailer, so a corrupt/interrupted download is rejected and +/// the good on-disk database is kept. No signature verification: trust the /// builder, over HTTPS. pub fn fetch_db_if_changed(url: &str, dest: &Path, prev: Option<&str>) -> io::Result { fetch_into(url, dest, prev, verify_exav_db) @@ -465,14 +477,14 @@ pub fn sig_dest(sigdir: &Path, url: &str) -> io::Result { // Sanitising is lossy and the query is dropped entirely, so two different // sources on one host can land on one file: `get.php?db=daily` and // `get.php?db=main` both reduce to `get.php`, and `a+b/` and `a%20b/` both - // to `a_b`. Sharing a destination is worse than a name collision — each poll + // to `a_b`. Sharing a destination is worse than a name collision: each poll // finds content that does not match its own stored validator, re-downloads, // and overwrites the other, so the two feeds flip-flop forever and the // loaded signature set depends on which finished last. // // A short digest of the WHOLE url, query included, separates them. It goes // into the final component so the directory layout still reads as the origin - // it came from — and it goes in front of the extension, never after it. The + // it came from, and it goes in front of the extension, never after it. The // loader dispatches on extension alone: `daily.cvd-473e5c` parses as an // extension of `cvd-473e5c`, matches no arm, and is skipped in silence, so a // digest appended at the end would fetch every source successfully and load @@ -497,10 +509,10 @@ pub fn sig_dest(sigdir: &Path, url: &str) -> io::Result { /// Pull one **signature source** from `url` into the signature directory `sigdir` /// (at [`sig_dest`], i.e. `/env//…`) only when it has changed. This -/// is how exav treats every source — a ClamAV `.cvd` and a third-party `.ndb` go +/// is how exav treats every source: a ClamAV `.cvd` and a third-party `.ndb` go /// through the exact same path. Validation before install: a `.cvd`/`.cld` must /// carry the `ClamAV-VDB:` header; any other file must be non-empty and not look -/// like an HTML error page. (No signature/GPG verification — trust the source, +/// like an HTML error page. (No signature/GPG verification: trust the source, /// over HTTPS.) pub fn fetch_signature_if_changed( url: &str, @@ -520,7 +532,7 @@ pub fn fetch_signature_if_changed( } } else { // Leading whitespace is ordinary in a templated error page, so the - // check has to look past it — otherwise "\n" installs + // check has to look past it, or "\n" installs // itself as a signature file and the loader gets to cope with the // markup. A JSON error body is the same class of answer. let head = b @@ -538,7 +550,7 @@ pub fn fetch_signature_if_changed( /// Garbage-collect exav's `env/` subtree: delete every file under `/env/` /// that no longer maps to one of `active_urls` (a source dropped from /// `--sig-sources`), then remove the directories left empty. -/// exav owns only this subtree — files elsewhere in `sigdir` (freshclam- or +/// exav owns only this subtree: files elsewhere in `sigdir` (freshclam- or /// hand-managed) are never touched. Returns the paths removed. Best-effort: an /// unreadable entry is skipped rather than fatal. pub fn prune_env_sources(sigdir: &Path, active_urls: &[&str]) -> io::Result> { @@ -553,7 +565,7 @@ pub fn prune_env_sources(sigdir: &Path, active_urls: &[&str]) -> io::Result Vec { let mut v = EXAV_DB_MAGIC.to_vec(); - v.extend_from_slice(&1u32.to_le_bytes()); + v.extend_from_slice(&EXAV_DB_VERSION.to_le_bytes()); v.extend_from_slice(payload); - v.extend_from_slice(Sha256::digest(payload).as_slice()); + v.extend_from_slice(&crc32fast::hash(payload).to_le_bytes()); v }; assert!(verify_exav_db(&framed(b"the database payload")).is_ok()); + let mut older = framed(b"the database payload"); + older[EXAV_DB_MAGIC.len()] = 3; + let err = verify_exav_db(&older).unwrap_err(); + assert!(err.contains("version 3"), "{err}"); assert!(verify_exav_db(&framed(b"")).is_ok()); assert!(verify_exav_db(b"404").is_err()); assert!(verify_exav_db(b"EXAVDB\x00\x01short").is_err()); diff --git a/crates/exav-update/tests/http_fetch.rs b/crates/exav-update/tests/http_fetch.rs index 214d5406..a9ab15e0 100644 --- a/crates/exav-update/tests/http_fetch.rs +++ b/crates/exav-update/tests/http_fetch.rs @@ -106,14 +106,16 @@ fn spawn(state: Arc>) -> String { format!("http://{addr}") } -/// A valid `.exavdb`: magic + version + payload + trailing SHA-256. +/// The `.exavdb` format version this build loads. +const EXAVDB_VERSION: u32 = 4; + +/// A valid `.exavdb`: magic + version + payload + trailing CRC-32. fn exavdb_body() -> Vec { - use sha2::{Digest, Sha256}; let payload = b"exav test database payload"; let mut v = b"EXAVDB\x00\x01".to_vec(); - v.extend_from_slice(&1u32.to_le_bytes()); + v.extend_from_slice(&EXAVDB_VERSION.to_le_bytes()); v.extend_from_slice(payload); - v.extend_from_slice(Sha256::digest(payload).as_slice()); + v.extend_from_slice(&crc32fast::hash(payload).to_le_bytes()); v } @@ -169,7 +171,7 @@ fn an_oversized_declaration_is_refused_before_the_body_is_read() { /// Redirects were untested entirely, which matters because the scheme they may /// redirect TO is a security property: an HTTPS source must never be downgraded /// to cleartext. That half needs a TLS server to exercise; this pins the part -/// that can be tested here — that following works at all, and that a completed +/// that can be tested here: that following works at all, and that a completed /// fetch leaves no `.tmp` behind for the next run to trip over. #[test] fn a_redirect_is_followed_and_leaves_no_temp_file() { @@ -251,12 +253,11 @@ fn head_etag_short_circuits_and_downloads_on_change() { // New ETag AND different bytes -> installed. { - use sha2::{Digest, Sha256}; let payload = b"a different database payload"; let mut b = b"EXAVDB\x00\x01".to_vec(); - b.extend_from_slice(&1u32.to_le_bytes()); + b.extend_from_slice(&EXAVDB_VERSION.to_le_bytes()); b.extend_from_slice(payload); - b.extend_from_slice(Sha256::digest(payload).as_slice()); + b.extend_from_slice(&crc32fast::hash(payload).to_le_bytes()); let mut st = state.lock().unwrap(); st.etag = "\"v3\"".into(); st.body = b; @@ -344,7 +345,7 @@ fn fetch_signature_installs_under_env_namespaced_by_origin() { let dest = sig_dest(&dir, &url).unwrap(); assert!(dest.starts_with(dir.join("env")), "must live under env/"); // Named for the source, plus a digest of the full URL so two sources that - // sanitise to the same name stay apart — the digest sitting in front of the + // sanitise to the same name stay apart, the digest sitting in front of the // extension, which the loader needs intact to classify the file at all. let name = dest.file_name().unwrap().to_string_lossy().into_owned(); assert!( @@ -401,7 +402,7 @@ fn sig_dest_namespaces_by_origin_and_resists_traversal() { // The extension survives the digest, for every source and every suffix. The // loader routes a database file by extension and nothing else, so a name it - // cannot classify is skipped without a word — a fetch that reports success + // cannot classify is skipped without a word: a fetch that reports success // and a database that never loads. Whatever the digest does to the name, it // must not touch the part the loader reads. for (url, want) in [ @@ -508,7 +509,7 @@ fn prune_env_sources_drops_deconfigured_and_keeps_the_rest() { #[test] fn redownloads_when_dest_deleted_despite_matching_validator() { // The validator ("v1") is unchanged, but the on-disk file was deleted out from - // under us. The HEAD/ETag shortcut must NOT report Unchanged — it must notice + // under us. The HEAD/ETag shortcut must NOT report Unchanged: it must notice // the file is gone and re-download, so a deleted DB self-heals without a restart. let state = Arc::new(Mutex::new(State { etag: "\"v1\"".into(), diff --git a/crates/exav-x86/NOTICE b/crates/exav-x86/NOTICE index 97811f32..9c9d1516 100644 --- a/crates/exav-x86/NOTICE +++ b/crates/exav-x86/NOTICE @@ -1,483 +1,39 @@ -exav — THIRD-PARTY NOTICES -========================== +exav-x86: third-party notices +============================= -exav itself is licensed under the MIT License (see the LICENSE file). +exav-x86 is licensed under the MIT License (see LICENSE). This file gives the +notices for the third-party data in this crate's source. The notices for the +components the exav binaries link are in the NOTICE file at the root of the +exav repository: https://github.com/sylvinus/exav/blob/main/NOTICE -The exav binary statically links a number of third-party Rust crates. Those -crates retain their own licenses; the binary distribution must carry their -notices. This file enumerates the major components and reproduces the notice -texts that the BSD-3-Clause and Apache-2.0-WITH-LLVM-exception licenses require. -This is a hand-maintained summary of the principal components, not a byte-exact -dump of every transitive crate's license. To regenerate a complete, per-crate -attribution list from the resolved dependency graph, install one of: +iced-x86, MIT (generated tables) +-------------------------------- - cargo install cargo-about # then: cargo about generate about.hbs - cargo install cargo-bundle-licenses - -and run it against this workspace. `cargo deny check licenses` (run in CI) -confirms every dependency resolves to a permissive license on the allowlist in -deny.toml. - -Versions below reflect Cargo.lock at the time of writing (last verified -2026-09-09). - - -------------------------------------------------------------------------------- -1. yara-x — BSD-3-Clause (source derivative, and a development dependency) -------------------------------------------------------------------------------- - -exav's YARA engine is native, in `crates/exav-core/src/yara/`. Parts of it are -derived from yara-x's source: the module implementations (`pe`, `elf`, `dotnet`, -`math`, `hash`, `string`) and the compiler's base64 handling, including field -semantics and the RVA-to-offset and import-hash routines. Each such file carries -a per-file attribution header, and the license text is reproduced below and in -`crates/exav-core/LICENSE-YARA-X`. - -`yara-x-parser` is a regular OPTIONAL DEPENDENCY, enabled by the default-on -`yara` feature of `crates/exav-core`, so a default build does link it. No other -yara-x crate is a dependency of any kind. - -The differential test that compares exav's engine against yara-x -(`crates/exav-core/tests/yara_difftest.rs`) invokes yara-x as a PROGRAM — the -`yr` binary from `PATH` — rather than linking it, and skips when that binary is -absent. - -Components: yara-x-parser 1.18. -Upstream: https://github.com/VirusTotal/yara-x -Copyright (c) VirusTotal and contributors. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other BSD-3-Clause components linked by exav (same 3-clause terms above, with -their respective copyright holders): subtle, tlsh2. - - -------------------------------------------------------------------------------- -1b. libarchive RAR5 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR5 (unpack50) decompressor in -crates/exav-unpack/src/formats/rar5_unpack.rs is a Rust port of the -decompression core of libarchive's archive_read_support_format_rar5.c. As a -source derivative, BSD-2-Clause requires retaining the upstream copyright and -licence text reproduced below. - -Upstream: https://github.com/libarchive/libarchive -Copyright (c) 2018 Grzegorz Antoniak (http://antoniak.org) -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1c. nwaples/rardecode RAR3 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR3/RAR4 (unpack29) decompressor in -crates/exav-unpack/src/formats/rar3_unpack.rs is a Rust port of the -decompression core of the Go package nwaples/rardecode. As a source -derivative, BSD-2-Clause requires retaining the upstream copyright and licence -text reproduced below. - -Upstream: https://github.com/nwaples/rardecode -Copyright (c) 2015, Nicholas Waples -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1d. ppmd-rust PPMd7 (var.H) model — CC0-1.0 OR MIT-0 (source derivative) -------------------------------------------------------------------------------- - -The PPMd variant H decoder in crates/exav-unpack/src/formats/ppmd7/ is vendored -and adapted from the `ppmd-rust` crate (version 1.4.0), generalised over a range -decoder trait so it can be driven by either the 7-Zip or the RAR range coder. -`ppmd-rust` is dual-licensed CC0-1.0 OR MIT-0 (public-domain-equivalent) and is -itself a Rust port of Igor Pavlov's 7-Zip PPMd7, which implements -Dmitry Shkarin's public-domain PPMd var.H (2001). It does NOT derive from -UnRAR/RARLAB. - -Upstream: https://crates.io/crates/ppmd-rust -License: CC0-1.0 OR MIT-0 (https://spdx.org/licenses/CC0-1.0, - https://spdx.org/licenses/MIT-0). Both place the work in (or treat it - as) the public domain; no attribution is legally required, and this - entry is recorded for completeness. - -The RAR-specific framing wired into crates/exav-unpack/src/formats/rar3_unpack.rs -(the PPMd block header: order/sub-allocator-memory/escape bits; the RAR -Subbotin carry-less range decoder in ppmd7/rangedec.rs; and the escape-symbol -handling for the PPMd↔LZSS conversion) is reproduced from the public format -description in libarchive's BSD-2-Clause archive_read_support_format_rar.c / -archive_ppmd7.c (already attributed in section 1b above), NOT from UnRAR. - - -------------------------------------------------------------------------------- -1e. dmg-core UDIF reader — Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The UDIF/DMG block reader in crates/exav-unpack/src/formats/udif.rs is vendored -and adapted from the `dmg-core` crate (version 0.1.2), changed to use -`lzma-rust2` instead of `lzma-rs` for XZ decompression. `dmg-core` is licensed -Apache-2.0; the Apache License requires retaining the attribution notices and -reproducing any upstream NOTICE text. - -Upstream: https://crates.io/crates/dmg-core -Copyright (c) Albert Hui / SecurityRonin. -Licensed under the Apache License, Version 2.0. You may obtain a copy at -http://www.apache.org/licenses/LICENSE-2.0 (see also the Apache-2.0 terms -referenced in section 2 below). Upstream ships no separate NOTICE file. - - -------------------------------------------------------------------------------- -1f. rust-cab CAB reader — MIT (source derivative) -------------------------------------------------------------------------------- - -The CAB (Microsoft Cabinet) reader in crates/exav-unpack/src/formats/cab_parse/ -is vendored and adapted from the `rust-cab` crate — write support, the datetime -module, unused public types, and tests removed; bounds checks added for -folder data-block and short-read cases. Licensed MIT; the MIT License requires -retaining the copyright and permission notice (reproduced in the exav LICENSE -file) in distributions. - -Upstream: https://github.com/mdsteele/rust-cab -Copyright (c) Matthew D. Steele . - - -------------------------------------------------------------------------------- -1g. unarc-rs ARJ reader — MIT OR Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The ARJ reader in crates/exav-unpack/src/formats/arj_parse/ is vendored and -adapted from the `unarc-rs` crate — multi-volume support, volume providers, and -external error types removed; wired to `delharc` for LH6 decompression. -Dual-licensed MIT OR Apache-2.0; the MIT or Apache terms apply at the -distributor's choice (both reproduced/referenced elsewhere in this file). - -The ZOO structures in crates/exav-unpack/src/formats/zoo_parse.rs are vendored -and adapted from the same crate, for the same reason the ARJ reader was: linking -it brings in a second `zip` with default features, and Cargo's feature -unification then enables the `zip` crate's own LZMA/bzip2/deflate64 decoders -inside exav, bypassing exav's per-member budget. Adapted to exav's error -handling, with the variable-length part of a directory entry (long filename, -directory name) actually read. - -The ZOO test archives in crates/exav-unpack/tests/fixtures/zoo/ (`store.zoo`, -`default.zoo`, `high_per.zoo`) are that project's own test corpus, redistributed -under the same terms. They hold one identical member under the three -compression methods, which is what lets the codecs be checked against stored -bytes rather than against themselves. - -Upstream: https://github.com/mkrueger/unarc-rs -Copyright (c) Mike Krüger. - - -------------------------------------------------------------------------------- -1h1. unshield / explode — MIT (linked dependency + test fixtures) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/ishield_z.rs reads InstallShield `.z` archives -through the `unshield` crate, whose only real dependency is `explode` — a leaf -decoder for the PKWARE Data Compression Library "implode" codec those archives -use. Both MIT. - -The fixtures in crates/exav-unpack/tests/fixtures/ishieldz/ are that project's -own examples, redistributed under the same terms. `undhr.z` is shipped alongside -`undhr.md`, the uncompressed original, which is what lets the decoder be checked -against the real file rather than against itself. - -This is a DIFFERENT format from the InstallShield `ISc(` InstallScript cabinet, -whose only implementation is the LGPL `unshield` C project — never read, and the -reason that format stays recognised-but-unopened. - -Upstream: https://github.com/agrif/unshield , https://github.com/agrif/explode -Copyright (c) Aaron Griffith. - - -------------------------------------------------------------------------------- -1h2. iced-x86 — MIT (generated opcode tables) -------------------------------------------------------------------------------- - -crates/exav-x86/src/generated.rs holds the SIMD, system and VEX regions of the -x86 opcode maps. It is not hand-written and not transcribed from a manual: it is -emitted by crates/exav-x86/tools/gen-tables, which decodes a probe for every -(prefix, opcode, /digit, mod) combination with `iced-x86` and records what came -back. The tables are therefore derived from `iced-x86`'s data, and the MIT -License requires retaining its copyright and permission notice (the MIT text is -in the exav LICENSE file). - -`iced-x86` is also the oracle that exav-x86's differential tests and the -`x86_decode` fuzz target compare against — including the decoding that -exav-pe-emu drives. It is a dev-dependency only: the decoder exav-pe-emu uses at -run time is exav-x86, and no iced code reaches a shipped artifact. +src/generated.rs holds the SIMD, system and VEX regions of the x86 opcode +maps. A generator in the exav repository (crates/exav-x86/tools/gen-tables, +not part of this crate) emits it by decoding a probe for every prefix, opcode, +/digit and mod combination with iced-x86 and recording the result, so the +tables are derived from iced-x86's data. Upstream: https://github.com/icedland/iced -Copyright (C) 2018-present iced project and contributors. - - -------------------------------------------------------------------------------- -1h. oletools test sample — BSD-2-Clause (test fixture) -------------------------------------------------------------------------------- - -The Excel 4.0 macro test fixture -crates/exav-unpack/tests/fixtures/xlm_excel4_sample.xls is a benign sample -redistributed from the `oletools` project's test corpus, used to verify the XLM -macro-sheet surfacing in formats/xlm.rs. Licensed BSD-2-Clause. - -Upstream: https://github.com/decalage2/oletools -Copyright (c) Philippe Lagadec. - - -------------------------------------------------------------------------------- -1i. aPLib reference (depack) — public domain (freeware) -------------------------------------------------------------------------------- - -The aPLib depacker in crates/exav-unpack/src/formats/aplib.rs is a clean-room -implementation of the published aPLib on-disk format. The bit/gamma reader -follows the algorithm published in the freely-distributable aPLib package's -reference `depack.c` by Jørgen Ibsen, which the author released for unrestricted -use ("free to use"). No attribution is legally required; this entry is recorded -for completeness. It does not derive from any GPL source. - -Upstream: https://ibsensoftware.com/products_aPLib.html - - -------------------------------------------------------------------------------- -1j. Microsoft Script Encoder decode tables — public domain -------------------------------------------------------------------------------- - -The substitution/decode tables in crates/exav-unpack/src/formats/screnc.rs are -the published Microsoft Script Encoder (`#@~^`) constants, as documented in -Didier Stevens' public-domain `decode-vbe.py`. Recorded here for completeness; -no attribution is legally required and it does not derive from any GPL source. - -Upstream: https://blog.didierstevens.com/programs/ - - -------------------------------------------------------------------------------- -1k. AutoIt-Ripper — MIT (reference implementation) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/autoit.rs implements the compiled-AutoIt3 script -layout — the EA05 record structure, the Mersenne-Twister keystream and its seeds -— from the MIT-licensed AutoIt-Ripper reference together with the public format -description. - -Upstream: https://github.com/nazywam/AutoIt-Ripper -Copyright (c) nazywam and contributors. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - - -------------------------------------------------------------------------------- -1l. NSIS format headers — zlib -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/nsis.rs implements the installer layout — the -firstheader and block-table structures — from NSIS's own zlib-licensed format -header `Source/exehead/fileform.h`, together with public reverse-engineering. - -Upstream: https://github.com/kichik/nsis -Copyright (c) Nullsoft and contributors. - - This software is provided 'as-is', without any express or implied warranty. - In no event will the authors be held liable for any damages arising from the - use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it freely, - subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a - product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - -------------------------------------------------------------------------------- -1m. sevenz-rust2 — Apache-2.0 (design reference) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/sevenz/ takes its reader structure from -sevenz-rust2, re-implemented natively so the crate can keep -`#![forbid(unsafe_code)]` and use exav's own PPMd7 model. - -Upstream: https://github.com/hasenbanck/sevenz-rust2 -Copyright (c) the sevenz-rust2 authors. - -Licensed under the Apache License, Version 2.0. A copy is available at -http://www.apache.org/licenses/LICENSE-2.0. The license requires retaining the -copyright, patent, trademark and attribution notices from the source, and -reproducing any NOTICE text the upstream distributes. - - -------------------------------------------------------------------------------- -2. ISC-licensed components -------------------------------------------------------------------------------- - -ring 0.17.14 (Apache-2.0 AND ISC), rustls 0.23.41, rustls-webpki, untrusted. -These carry ISC permission-notice terms; reproduce the copyright notice and the -permission/disclaimer notice in distributions. - - -------------------------------------------------------------------------------- -3. MIT / Apache-2.0 dual-licensed components (the bulk of the graph) -------------------------------------------------------------------------------- - -The majority of exav's dependencies are dual-licensed "MIT OR Apache-2.0" (or a -superset including Unlicense / 0BSD / Zlib), so the MIT or Apache terms apply at -the distributor's choice. Notable examples: - - aho-corasick, memchr, daachorse pattern matching / Aho-Corasick - serde, rmp-serde serialization - clap CLI parsing - sha1, sha2, md-5, digest (RustCrypto) hashing - flate2, miniz_oxide, bzip2, zip decompression (sevenz-rust2 is dev-only: - it builds 7z test fixtures; the shipped - 7z reader is native, §1m) - lzfse_rust, delharc, lzxd, DMG/LHA/CAB codecs - bitstream-io, stringprep - cfb OLE/CFB compound-file reader - fatfs, ext4-view FAT and ext2/3/4 filesystem readers - walkdir directory traversal - regex path-filter regexes, and the search - engine behind exav-grep - libc syscall bindings (daemon) - ureq, rustls, webpki-roots HTTPS for the signature updater - wasm-bindgen, js-sys, web-sys, WebAssembly/JS bindings — these reach - wasm-bindgen-futures the published npm artifact - -MIT-only components (the MIT copyright and permission notice must be retained; -the MIT text is in the exav LICENSE file): cfb, hfsplus, apfs (the two latter -via DMG/HFS+/APFS support). - -RustCrypto crates are MIT OR Apache-2.0. Zlib-licensed components present: -miniz_oxide, tinyvec. Unicode-3.0: the ICU / unicode-ident -crates. Other permissive licenses in the graph: CC0-1.0 and MIT-0 (ppmd-rust via -sevenz-rust2), CDLA-Permissive-2.0 (webpki-roots, TLS root store), bzip2-1.0.6 -(libbz2-rs-sys). - -Format documentation consulted ------------------------------ - -Some decoders were written from format descriptions rather than from any code. -Recorded here because a decoder's provenance is part of what makes it auditable: - - EGG ESTsoft Corp., "EGG Format Specification, Version 1.0" — the - vendor's own published document. No ESTsoft code was used. - - AZO ESTsoft's own compression algorithm, absent from the EGG spec. - `crates/exav-unpack/src/formats/azo.rs` and `azo_tables.rs` are a - Rust port of the AZO decoder in `EggDotNet` - (https://github.com/akolman/EggDotNet), MIT, Copyright (c) Andrew - Kolman. ESTsoft's own UnEgg library was NOT used: its licence - forbids commercial use without approval and forbids using it to - develop a compression algorithm. - - HWP3 Preamble offsets taken from `java-hwp` - (https://github.com/ddoleye/java-hwp), Apache-2.0. - Copyright the java-hwp authors. Validated against - `testHWP_3.0.hwp` from the Apache Tika test corpus - (https://github.com/apache/tika), Apache-2.0, Copyright the - Apache Software Foundation — vendored as - `crates/exav-unpack/tests/fixtures/hwp3/tika_testHWP_3.0.hwp`. - - ALZ Structure cross-checked against `unalz` (zlib license) and - `unar`, used as external oracles rather than as sources. - -Full MIT and Apache-2.0 license texts are widely available and are reproduced in -each crate's source distribution; the exav LICENSE file contains the MIT text. - - -------------------------------------------------------------------------------- -Build-only tooling -------------------------------------------------------------------------------- -Everything above concerns code that reaches a shipped artifact. The -documentation site under `www/` builds with an npm toolchain whose lockfile -includes packages under stronger terms, LGPL-3.0-or-later among them. None of it -is linked into, bundled with, or distributed alongside any exav binary, crate or -WASM module: it runs at build time to produce static HTML and is listed here so -that a licence scan of the repository has an answer rather than a finding. +Copyright (C) 2018-present iced project and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/exav/Cargo.toml b/crates/exav/Cargo.toml index 95ce2dcf..e1447163 100644 --- a/crates/exav/Cargo.toml +++ b/crates/exav/Cargo.toml @@ -134,7 +134,6 @@ exav-core = { path = "../exav-core", version = "0.0.2", default-features = false # feature so the default pure-Rust build never links it. exav-update = { path = "../exav-update", version = "0.0.2", optional = true } clap.workspace = true -anyhow.workspace = true walkdir.workspace = true libc = "0.2.186" regex = "1.12.4" diff --git a/crates/exav/NOTICE b/crates/exav/NOTICE index 97811f32..96a30096 100644 --- a/crates/exav/NOTICE +++ b/crates/exav/NOTICE @@ -1,483 +1,7 @@ -exav — THIRD-PARTY NOTICES -========================== +exav: third-party notices -exav itself is licensed under the MIT License (see the LICENSE file). +This crate is part of exav and contains no third-party source. The notices +for the components exav links are in the NOTICE file at the root of the exav +repository, which also ships in every release archive: -The exav binary statically links a number of third-party Rust crates. Those -crates retain their own licenses; the binary distribution must carry their -notices. This file enumerates the major components and reproduces the notice -texts that the BSD-3-Clause and Apache-2.0-WITH-LLVM-exception licenses require. - -This is a hand-maintained summary of the principal components, not a byte-exact -dump of every transitive crate's license. To regenerate a complete, per-crate -attribution list from the resolved dependency graph, install one of: - - cargo install cargo-about # then: cargo about generate about.hbs - cargo install cargo-bundle-licenses - -and run it against this workspace. `cargo deny check licenses` (run in CI) -confirms every dependency resolves to a permissive license on the allowlist in -deny.toml. - -Versions below reflect Cargo.lock at the time of writing (last verified -2026-09-09). - - -------------------------------------------------------------------------------- -1. yara-x — BSD-3-Clause (source derivative, and a development dependency) -------------------------------------------------------------------------------- - -exav's YARA engine is native, in `crates/exav-core/src/yara/`. Parts of it are -derived from yara-x's source: the module implementations (`pe`, `elf`, `dotnet`, -`math`, `hash`, `string`) and the compiler's base64 handling, including field -semantics and the RVA-to-offset and import-hash routines. Each such file carries -a per-file attribution header, and the license text is reproduced below and in -`crates/exav-core/LICENSE-YARA-X`. - -`yara-x-parser` is a regular OPTIONAL DEPENDENCY, enabled by the default-on -`yara` feature of `crates/exav-core`, so a default build does link it. No other -yara-x crate is a dependency of any kind. - -The differential test that compares exav's engine against yara-x -(`crates/exav-core/tests/yara_difftest.rs`) invokes yara-x as a PROGRAM — the -`yr` binary from `PATH` — rather than linking it, and skips when that binary is -absent. - -Components: yara-x-parser 1.18. -Upstream: https://github.com/VirusTotal/yara-x -Copyright (c) VirusTotal and contributors. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other BSD-3-Clause components linked by exav (same 3-clause terms above, with -their respective copyright holders): subtle, tlsh2. - - -------------------------------------------------------------------------------- -1b. libarchive RAR5 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR5 (unpack50) decompressor in -crates/exav-unpack/src/formats/rar5_unpack.rs is a Rust port of the -decompression core of libarchive's archive_read_support_format_rar5.c. As a -source derivative, BSD-2-Clause requires retaining the upstream copyright and -licence text reproduced below. - -Upstream: https://github.com/libarchive/libarchive -Copyright (c) 2018 Grzegorz Antoniak (http://antoniak.org) -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1c. nwaples/rardecode RAR3 reader — BSD-2-Clause (source derivative) -------------------------------------------------------------------------------- - -The RAR3/RAR4 (unpack29) decompressor in -crates/exav-unpack/src/formats/rar3_unpack.rs is a Rust port of the -decompression core of the Go package nwaples/rardecode. As a source -derivative, BSD-2-Clause requires retaining the upstream copyright and licence -text reproduced below. - -Upstream: https://github.com/nwaples/rardecode -Copyright (c) 2015, Nicholas Waples -All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- -1d. ppmd-rust PPMd7 (var.H) model — CC0-1.0 OR MIT-0 (source derivative) -------------------------------------------------------------------------------- - -The PPMd variant H decoder in crates/exav-unpack/src/formats/ppmd7/ is vendored -and adapted from the `ppmd-rust` crate (version 1.4.0), generalised over a range -decoder trait so it can be driven by either the 7-Zip or the RAR range coder. -`ppmd-rust` is dual-licensed CC0-1.0 OR MIT-0 (public-domain-equivalent) and is -itself a Rust port of Igor Pavlov's 7-Zip PPMd7, which implements -Dmitry Shkarin's public-domain PPMd var.H (2001). It does NOT derive from -UnRAR/RARLAB. - -Upstream: https://crates.io/crates/ppmd-rust -License: CC0-1.0 OR MIT-0 (https://spdx.org/licenses/CC0-1.0, - https://spdx.org/licenses/MIT-0). Both place the work in (or treat it - as) the public domain; no attribution is legally required, and this - entry is recorded for completeness. - -The RAR-specific framing wired into crates/exav-unpack/src/formats/rar3_unpack.rs -(the PPMd block header: order/sub-allocator-memory/escape bits; the RAR -Subbotin carry-less range decoder in ppmd7/rangedec.rs; and the escape-symbol -handling for the PPMd↔LZSS conversion) is reproduced from the public format -description in libarchive's BSD-2-Clause archive_read_support_format_rar.c / -archive_ppmd7.c (already attributed in section 1b above), NOT from UnRAR. - - -------------------------------------------------------------------------------- -1e. dmg-core UDIF reader — Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The UDIF/DMG block reader in crates/exav-unpack/src/formats/udif.rs is vendored -and adapted from the `dmg-core` crate (version 0.1.2), changed to use -`lzma-rust2` instead of `lzma-rs` for XZ decompression. `dmg-core` is licensed -Apache-2.0; the Apache License requires retaining the attribution notices and -reproducing any upstream NOTICE text. - -Upstream: https://crates.io/crates/dmg-core -Copyright (c) Albert Hui / SecurityRonin. -Licensed under the Apache License, Version 2.0. You may obtain a copy at -http://www.apache.org/licenses/LICENSE-2.0 (see also the Apache-2.0 terms -referenced in section 2 below). Upstream ships no separate NOTICE file. - - -------------------------------------------------------------------------------- -1f. rust-cab CAB reader — MIT (source derivative) -------------------------------------------------------------------------------- - -The CAB (Microsoft Cabinet) reader in crates/exav-unpack/src/formats/cab_parse/ -is vendored and adapted from the `rust-cab` crate — write support, the datetime -module, unused public types, and tests removed; bounds checks added for -folder data-block and short-read cases. Licensed MIT; the MIT License requires -retaining the copyright and permission notice (reproduced in the exav LICENSE -file) in distributions. - -Upstream: https://github.com/mdsteele/rust-cab -Copyright (c) Matthew D. Steele . - - -------------------------------------------------------------------------------- -1g. unarc-rs ARJ reader — MIT OR Apache-2.0 (source derivative) -------------------------------------------------------------------------------- - -The ARJ reader in crates/exav-unpack/src/formats/arj_parse/ is vendored and -adapted from the `unarc-rs` crate — multi-volume support, volume providers, and -external error types removed; wired to `delharc` for LH6 decompression. -Dual-licensed MIT OR Apache-2.0; the MIT or Apache terms apply at the -distributor's choice (both reproduced/referenced elsewhere in this file). - -The ZOO structures in crates/exav-unpack/src/formats/zoo_parse.rs are vendored -and adapted from the same crate, for the same reason the ARJ reader was: linking -it brings in a second `zip` with default features, and Cargo's feature -unification then enables the `zip` crate's own LZMA/bzip2/deflate64 decoders -inside exav, bypassing exav's per-member budget. Adapted to exav's error -handling, with the variable-length part of a directory entry (long filename, -directory name) actually read. - -The ZOO test archives in crates/exav-unpack/tests/fixtures/zoo/ (`store.zoo`, -`default.zoo`, `high_per.zoo`) are that project's own test corpus, redistributed -under the same terms. They hold one identical member under the three -compression methods, which is what lets the codecs be checked against stored -bytes rather than against themselves. - -Upstream: https://github.com/mkrueger/unarc-rs -Copyright (c) Mike Krüger. - - -------------------------------------------------------------------------------- -1h1. unshield / explode — MIT (linked dependency + test fixtures) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/ishield_z.rs reads InstallShield `.z` archives -through the `unshield` crate, whose only real dependency is `explode` — a leaf -decoder for the PKWARE Data Compression Library "implode" codec those archives -use. Both MIT. - -The fixtures in crates/exav-unpack/tests/fixtures/ishieldz/ are that project's -own examples, redistributed under the same terms. `undhr.z` is shipped alongside -`undhr.md`, the uncompressed original, which is what lets the decoder be checked -against the real file rather than against itself. - -This is a DIFFERENT format from the InstallShield `ISc(` InstallScript cabinet, -whose only implementation is the LGPL `unshield` C project — never read, and the -reason that format stays recognised-but-unopened. - -Upstream: https://github.com/agrif/unshield , https://github.com/agrif/explode -Copyright (c) Aaron Griffith. - - -------------------------------------------------------------------------------- -1h2. iced-x86 — MIT (generated opcode tables) -------------------------------------------------------------------------------- - -crates/exav-x86/src/generated.rs holds the SIMD, system and VEX regions of the -x86 opcode maps. It is not hand-written and not transcribed from a manual: it is -emitted by crates/exav-x86/tools/gen-tables, which decodes a probe for every -(prefix, opcode, /digit, mod) combination with `iced-x86` and records what came -back. The tables are therefore derived from `iced-x86`'s data, and the MIT -License requires retaining its copyright and permission notice (the MIT text is -in the exav LICENSE file). - -`iced-x86` is also the oracle that exav-x86's differential tests and the -`x86_decode` fuzz target compare against — including the decoding that -exav-pe-emu drives. It is a dev-dependency only: the decoder exav-pe-emu uses at -run time is exav-x86, and no iced code reaches a shipped artifact. - -Upstream: https://github.com/icedland/iced -Copyright (C) 2018-present iced project and contributors. - - -------------------------------------------------------------------------------- -1h. oletools test sample — BSD-2-Clause (test fixture) -------------------------------------------------------------------------------- - -The Excel 4.0 macro test fixture -crates/exav-unpack/tests/fixtures/xlm_excel4_sample.xls is a benign sample -redistributed from the `oletools` project's test corpus, used to verify the XLM -macro-sheet surfacing in formats/xlm.rs. Licensed BSD-2-Clause. - -Upstream: https://github.com/decalage2/oletools -Copyright (c) Philippe Lagadec. - - -------------------------------------------------------------------------------- -1i. aPLib reference (depack) — public domain (freeware) -------------------------------------------------------------------------------- - -The aPLib depacker in crates/exav-unpack/src/formats/aplib.rs is a clean-room -implementation of the published aPLib on-disk format. The bit/gamma reader -follows the algorithm published in the freely-distributable aPLib package's -reference `depack.c` by Jørgen Ibsen, which the author released for unrestricted -use ("free to use"). No attribution is legally required; this entry is recorded -for completeness. It does not derive from any GPL source. - -Upstream: https://ibsensoftware.com/products_aPLib.html - - -------------------------------------------------------------------------------- -1j. Microsoft Script Encoder decode tables — public domain -------------------------------------------------------------------------------- - -The substitution/decode tables in crates/exav-unpack/src/formats/screnc.rs are -the published Microsoft Script Encoder (`#@~^`) constants, as documented in -Didier Stevens' public-domain `decode-vbe.py`. Recorded here for completeness; -no attribution is legally required and it does not derive from any GPL source. - -Upstream: https://blog.didierstevens.com/programs/ - - -------------------------------------------------------------------------------- -1k. AutoIt-Ripper — MIT (reference implementation) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/autoit.rs implements the compiled-AutoIt3 script -layout — the EA05 record structure, the Mersenne-Twister keystream and its seeds -— from the MIT-licensed AutoIt-Ripper reference together with the public format -description. - -Upstream: https://github.com/nazywam/AutoIt-Ripper -Copyright (c) nazywam and contributors. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - - -------------------------------------------------------------------------------- -1l. NSIS format headers — zlib -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/nsis.rs implements the installer layout — the -firstheader and block-table structures — from NSIS's own zlib-licensed format -header `Source/exehead/fileform.h`, together with public reverse-engineering. - -Upstream: https://github.com/kichik/nsis -Copyright (c) Nullsoft and contributors. - - This software is provided 'as-is', without any express or implied warranty. - In no event will the authors be held liable for any damages arising from the - use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it freely, - subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a - product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - -------------------------------------------------------------------------------- -1m. sevenz-rust2 — Apache-2.0 (design reference) -------------------------------------------------------------------------------- - -crates/exav-unpack/src/formats/sevenz/ takes its reader structure from -sevenz-rust2, re-implemented natively so the crate can keep -`#![forbid(unsafe_code)]` and use exav's own PPMd7 model. - -Upstream: https://github.com/hasenbanck/sevenz-rust2 -Copyright (c) the sevenz-rust2 authors. - -Licensed under the Apache License, Version 2.0. A copy is available at -http://www.apache.org/licenses/LICENSE-2.0. The license requires retaining the -copyright, patent, trademark and attribution notices from the source, and -reproducing any NOTICE text the upstream distributes. - - -------------------------------------------------------------------------------- -2. ISC-licensed components -------------------------------------------------------------------------------- - -ring 0.17.14 (Apache-2.0 AND ISC), rustls 0.23.41, rustls-webpki, untrusted. -These carry ISC permission-notice terms; reproduce the copyright notice and the -permission/disclaimer notice in distributions. - - -------------------------------------------------------------------------------- -3. MIT / Apache-2.0 dual-licensed components (the bulk of the graph) -------------------------------------------------------------------------------- - -The majority of exav's dependencies are dual-licensed "MIT OR Apache-2.0" (or a -superset including Unlicense / 0BSD / Zlib), so the MIT or Apache terms apply at -the distributor's choice. Notable examples: - - aho-corasick, memchr, daachorse pattern matching / Aho-Corasick - serde, rmp-serde serialization - clap CLI parsing - sha1, sha2, md-5, digest (RustCrypto) hashing - flate2, miniz_oxide, bzip2, zip decompression (sevenz-rust2 is dev-only: - it builds 7z test fixtures; the shipped - 7z reader is native, §1m) - lzfse_rust, delharc, lzxd, DMG/LHA/CAB codecs - bitstream-io, stringprep - cfb OLE/CFB compound-file reader - fatfs, ext4-view FAT and ext2/3/4 filesystem readers - walkdir directory traversal - regex path-filter regexes, and the search - engine behind exav-grep - libc syscall bindings (daemon) - ureq, rustls, webpki-roots HTTPS for the signature updater - wasm-bindgen, js-sys, web-sys, WebAssembly/JS bindings — these reach - wasm-bindgen-futures the published npm artifact - -MIT-only components (the MIT copyright and permission notice must be retained; -the MIT text is in the exav LICENSE file): cfb, hfsplus, apfs (the two latter -via DMG/HFS+/APFS support). - -RustCrypto crates are MIT OR Apache-2.0. Zlib-licensed components present: -miniz_oxide, tinyvec. Unicode-3.0: the ICU / unicode-ident -crates. Other permissive licenses in the graph: CC0-1.0 and MIT-0 (ppmd-rust via -sevenz-rust2), CDLA-Permissive-2.0 (webpki-roots, TLS root store), bzip2-1.0.6 -(libbz2-rs-sys). - -Format documentation consulted ------------------------------ - -Some decoders were written from format descriptions rather than from any code. -Recorded here because a decoder's provenance is part of what makes it auditable: - - EGG ESTsoft Corp., "EGG Format Specification, Version 1.0" — the - vendor's own published document. No ESTsoft code was used. - - AZO ESTsoft's own compression algorithm, absent from the EGG spec. - `crates/exav-unpack/src/formats/azo.rs` and `azo_tables.rs` are a - Rust port of the AZO decoder in `EggDotNet` - (https://github.com/akolman/EggDotNet), MIT, Copyright (c) Andrew - Kolman. ESTsoft's own UnEgg library was NOT used: its licence - forbids commercial use without approval and forbids using it to - develop a compression algorithm. - - HWP3 Preamble offsets taken from `java-hwp` - (https://github.com/ddoleye/java-hwp), Apache-2.0. - Copyright the java-hwp authors. Validated against - `testHWP_3.0.hwp` from the Apache Tika test corpus - (https://github.com/apache/tika), Apache-2.0, Copyright the - Apache Software Foundation — vendored as - `crates/exav-unpack/tests/fixtures/hwp3/tika_testHWP_3.0.hwp`. - - ALZ Structure cross-checked against `unalz` (zlib license) and - `unar`, used as external oracles rather than as sources. - -Full MIT and Apache-2.0 license texts are widely available and are reproduced in -each crate's source distribution; the exav LICENSE file contains the MIT text. - - -------------------------------------------------------------------------------- -Build-only tooling -------------------------------------------------------------------------------- - -Everything above concerns code that reaches a shipped artifact. The -documentation site under `www/` builds with an npm toolchain whose lockfile -includes packages under stronger terms, LGPL-3.0-or-later among them. None of it -is linked into, bundled with, or distributed alongside any exav binary, crate or -WASM module: it runs at build time to produce static HTML and is listed here so -that a licence scan of the repository has an answer rather than a finding. +https://github.com/sylvinus/exav/blob/main/NOTICE diff --git a/crates/exav/src/daemon.rs b/crates/exav/src/daemon.rs index 09fcf07e..e554396d 100644 --- a/crates/exav/src/daemon.rs +++ b/crates/exav/src/daemon.rs @@ -1196,8 +1196,9 @@ fn current_address_space() -> Option { #[cfg(unix)] pub(crate) fn fit_limits_to_job_memory(opts: &mut ScanOptions, job_memory: u64) { /// Of the memory a job is granted, the share extraction buffers may claim. - /// The rest covers the matcher's own working set (chiefly the lowercase - /// copy it makes of each buffer it scans) plus allocator slack. + /// The rest covers the matcher's own working set (such as the lowercase + /// copy of a buffer a case-insensitive signature is checked in) plus + /// allocator slack. const EXTRACTION_SHARE_NUM: u64 = 1; const EXTRACTION_SHARE_DEN: u64 = 2; @@ -2700,9 +2701,9 @@ fn scan_tree_allmatch(db: &Scanner, opts: &ScanOptions, path: &str) -> Vec Vec { if path.is_empty() { return vec!["ALLMATCHSCAN: missing path ERROR".to_string()]; @@ -2712,9 +2713,6 @@ fn scan_one_allmatch(db: &Scanner, opts: &ScanOptions, path: &str) -> Vec o, Err(e) => return vec![format!("{path}: {e} ERROR")], }; - if opts.max_scan_size.is_some_and(|max| size > max) { - return vec![scan_one_path(db, opts, path)]; - } // Isolate a panic on a crafted file into an ERROR for this target. let found = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { analyze_all_seekable(db, file, size, opts) @@ -2813,18 +2811,20 @@ impl StreamPayload { } /// Every detection in the payload, at any size (see - /// [`exav_core::analyze_all_seekable`]). + /// [`exav_core::analyze_all_seekable`]), for a stream of `size` bytes: + /// past `--max-input-bytes`, more than the payload holds. pub(crate) fn all_matches( &self, db: &Scanner, opts: &ScanOptions, + size: u64, ) -> io::Result<(Vec<(String, exav_core::Method)>, AllMatchOutcome)> { match self { StreamPayload::Mem(v) => { - exav_core::analyze_all_seekable(db, std::io::Cursor::new(&v[..]), self.len(), opts) + exav_core::analyze_all_seekable(db, std::io::Cursor::new(&v[..]), size, opts) } - StreamPayload::Disk(tmp, n) => { - exav_core::analyze_all_seekable(db, tmp.reopen()?, *n, opts) + StreamPayload::Disk(tmp, _) => { + exav_core::analyze_all_seekable(db, tmp.reopen()?, size, opts) } } } diff --git a/crates/exav/src/main.rs b/crates/exav/src/main.rs index b66ff9ac..b23486ad 100644 --- a/crates/exav/src/main.rs +++ b/crates/exav/src/main.rs @@ -2,7 +2,7 @@ //! //! Exit codes and output match clamscan (0 = clean, 1 = found, 2 = error; //! `PATH: Signature FOUND` / `PATH: OK`), so a script reading either keeps -//! working. The *flags* are exav's own — one clamscan has and exav does not +//! working. The *flags* are exav's own: one clamscan has and exav does not //! stops the run rather than being swallowed, so a migrated command line never //! scans under settings nobody asked for. //! @@ -44,7 +44,7 @@ use clap::Parser; use exav_core::{loader, scan_path, ScanOptions, ScanReport, Scanner, Verdict, VerdictCategory}; /// Categories the daemon names before the closing ` ERROR` when a verdict is -/// `PARTIAL` — as opposed to a hard scan error, which has no category. The +/// `PARTIAL`, as opposed to a hard scan error, which has no category. The /// client reads them back so its summary and exit code match a local one-shot /// scan of the same file. const PARTIAL_TAGS: [&str; 3] = ["LIMITS-EXCEEDED", "UNSCANNABLE", "PASSWORD-PROTECTED"]; @@ -53,7 +53,7 @@ const PARTIAL_TAGS: [&str; 3] = ["LIMITS-EXCEEDED", "UNSCANNABLE", "PASSWORD-PRO /// /// The wire grammar is `: ERROR`, the same /// `reason CATEGORY STATUS` order the one-shot CLI prints. So the category is -/// the second-to-last word — not a prefix of anything, and not a substring +/// the second-to-last word: not a prefix of anything, and not a substring /// search: a reason is free text and a path like `/data/UNSCANNABLE/x` would /// otherwise turn a real error into a partial, moving it out of the error /// counter and off stderr. @@ -70,7 +70,7 @@ use walkdir::WalkDir; /// Print one result line to stdout and mirror it into `--log`. /// /// Every verdict goes through here so the log can never disagree with the -/// terminal — a log that is missing a detection is worse than no log. +/// terminal. A log that is missing a detection is worse than no log. macro_rules! outln { ($($arg:tt)*) => {{ let line = format!($($arg)*); @@ -98,7 +98,7 @@ fn read_path_list(list: &std::path::Path) -> std::io::Result> { /// Read a `--passwords-from` file: one password per line, kept byte-identical /// except for the line ending. Unlike [`read_path_list`], nothing is trimmed -/// or skipped — spaces, `#` and empty lines are all legal inside a password, +/// or skipped: spaces, `#` and empty lines are all legal inside a password, /// and only a file's final newline is not one (an actually-empty password is /// a blank line anywhere else). `-` is refused: it would read scan stdin, /// which is already the scan input. @@ -135,7 +135,7 @@ fn log_open(path: &std::path::Path) -> std::io::Result<()> { } /// Mirror one result line into the `--log` file. Writing to stdout stays the -/// caller's job — the log is an addition, never a redirection, so piping still +/// caller's job. The log is an addition, never a redirection, so piping still /// behaves and a broken log cannot swallow a detection. /// /// Line and newline go out in one `write_all`. The prefork daemon's workers are @@ -166,8 +166,8 @@ pub(crate) fn log_line(line: &str) { /// /// exav reads ClamAV's databases and speaks its protocol; it does not take its /// command line. Accepting `clamscan`'s spellings as aliases meant two names for -/// every bound — twice the documentation, and a second way for a command line to -/// be subtly wrong — in exchange for letting an invocation be pasted across, +/// every bound (twice the documentation, and a second way for a command line to +/// be subtly wrong) in exchange for letting an invocation be pasted across, /// which nobody does twice. What replaces them is a pointer to the table that /// says what maps to what. const AFTER_HELP: &str = "\ @@ -181,8 +181,8 @@ Coming from ClamAV: /// Every flag falls back to an environment variable, and the environment /// belongs to the process while a test does not. So any test that sets a -/// variable — or that parses a command line a variable could change the meaning -/// of — holds this for its duration, and the whole crate shares the one lock: +/// variable, or that parses a command line a variable could change the meaning +/// of, holds this for its duration, and the whole crate shares the one lock: /// two test modules with a lock each would not exclude one another. #[cfg(test)] pub(crate) fn env_guard() -> std::sync::MutexGuard<'static, ()> { @@ -211,12 +211,12 @@ fn env_switch() -> clap::builder::BoolishValueParser { /// them could say "the default" out loud. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] enum SendAs { - /// `SCAN ` — the daemon opens the file itself. + /// `SCAN `: the daemon opens the file itself. #[default] Path, - /// `INSTREAM` — the bytes go over the connection. + /// `INSTREAM`: the bytes go over the connection. Contents, - /// `FILDES` — an open descriptor over a Unix socket (SCM_RIGHTS). + /// `FILDES`: an open descriptor over a Unix socket (SCM_RIGHTS). Fd, } @@ -315,17 +315,17 @@ struct Cli { )] sigs: PathBuf, - /// Where --auto-update fetches signatures from. Repeatable (comma- or - /// whitespace-separated in the environment); the sources merge. + /// Where --auto-update fetches signatures from. Repeatable (comma-separated + /// in the environment); the sources merge. /// /// https://host/main.cvd an exact source, fetched verbatim - /// https://host/db/ a mirror base — the trailing slash makes it one, + /// https://host/db/ a mirror base (the trailing slash makes it one), /// expanding to /{main,daily,bytecode}.cvd /// /etc/exav/sources a file of the above, one per line (`#` comments /// OK), or a `freshclam.conf` /// /// One flag rather than one per shape: which of the three a value is, is - /// legible from the value. A `freshclam.conf` works as-is — exav reads its + /// legible from the value. A `freshclam.conf` works as-is: exav reads its /// *source* directives (`DatabaseMirror`/`PrivateMirror`, /// `DatabaseCustomURL`) and warns about the lines it ignores, including /// `DatabaseDirectory` (that is --sig-dir; it is not a source). @@ -344,7 +344,7 @@ struct Cli { /// URL of a prebuilt `.exavdb` to pull and serve, instead of fetching /// signature files. Re-checked and hot-reloaded on change by --auto-update, - /// with a cheap conditional `HEAD` — which is why its default re-check is + /// with a cheap conditional `HEAD`, which is why its default re-check is /// far more frequent than a full source fetch's. Basic auth via `user:pass@`. /// Needs a build with `--features http-update`. /// @@ -396,7 +396,7 @@ struct Cli { /// Compile the loaded signatures into a prebuilt `.exavdb` database, write it /// to FILE, and exit. The file loads directly with `-d` for a near-instant /// cold start and can be distributed as-is. Run this on a host with enough - /// RAM (compiling the full signature set needs several GB); the resulting + /// RAM (compiling takes more memory than loading the result); the resulting /// database loads cheaply everywhere. #[arg(long = "build-db", value_name = "FILE", env = "EXAV_BUILD_DB")] build_db: Option, @@ -408,20 +408,20 @@ struct Cli { /// clamd:///var/run/exav.sock the clamd protocol over a Unix socket /// icap://0.0.0.0:1344 ICAP (RFC 3507), for a proxy's hook /// icap://0.0.0.0:1344/avscan ICAP answering on that service only - /// 0.0.0.0:3310 no scheme — clamd - /// /var/run/exav.sock no scheme, a path — clamd over a socket + /// 0.0.0.0:3310 no scheme: clamd + /// /var/run/exav.sock no scheme, a path: clamd over a socket /// /// An ICAP service name is the path of the URL a proxy is configured with, /// so it goes where it already lives: paste `icap://scanner:1344/avscan` /// out of a squid.conf unchanged. With no path exav answers on all three - /// names a c-icap `virus_scan` deployment does — avscan, srv_clamav, - /// virus_scan — so it stands in for one without knowing which the proxy + /// names a c-icap `virus_scan` deployment does (avscan, srv_clamav, + /// virus_scan), so it stands in for one without knowing which the proxy /// asks for. Naming one replaces that set rather than adding to it. /// /// A `?key=value` tail sets what belongs to this listener alone: /// /// mode=660 permission bits for a Unix socket. Default 0600, - /// owner only — every user the mode admits can submit + /// owner only. Every user the mode admits can submit /// scans and read the verdicts. /// max-connections=200 concurrent connections accepted here. Default 128 /// for clamd, 100 for ICAP (which also advertises it @@ -458,7 +458,7 @@ struct Cli { /// replied, `2` if it did not. Scans nothing. [clamdscan: --ping] /// /// The address is `--connect` when given, and otherwise the listener this - /// same configuration would serve — so a container health check is + /// same configuration would serve, so a container health check is /// `exav --ping` and needs no address of its own. That matters because the /// address is often not knowable where the check is written: `EXAV_LISTEN` /// can move the port or serve ICAP instead, and a check pinned to @@ -566,7 +566,7 @@ struct Cli { /// fd an open descriptor over the Unix socket (`FILDES`). A daemon /// running as another user reads the file without permission to /// open the path. Cheaper than `contents` (no copy), and Unix - /// sockets only — it rides on SCM_RIGHTS. + /// sockets only, since it rides on SCM_RIGHTS. /// /// `-` (stdin) has no path to name, so it always goes as contents. #[arg( @@ -581,7 +581,7 @@ struct Cli { /// Honour the clamd `SHUTDOWN` command, letting any client that can reach /// the daemon stop it. Off by default. /// - /// A scanner that is not running does not report infected — it reports + /// A scanner that is not running does not report infected: it reports /// nothing, and a pipeline that reads "no answer" as "fine" passes /// everything. clamd honours `SHUTDOWN`; exav does not unless asked. #[arg(long = "allow-shutdown", env = "EXAV_ALLOW_SHUTDOWN", value_parser = env_switch())] @@ -646,25 +646,6 @@ struct Cli { )] max_scan_memory: Option, - /// Peak memory for building one matcher shard (`--build-db` only): each - /// large partition is split so no single Aho-Corasick construction exceeds - /// about this many bytes. It bounds that transient, not the total: the - /// parsed signatures stay resident underneath it, so the peak is this plus - /// that floor. K/M/G/T suffixes; `off` or `0` means one shard per - /// partition. - /// - /// Each shard is one more walk of every scanned buffer, but shards hold - /// disjoint pattern sets, so measured scan time does not change noticeably - /// with it. Treat it as a build-memory knob: lower it on a small build - /// host, otherwise use the largest value the host can afford. - #[arg( - long = "build-shard-bytes", - value_name = "SIZE|off", - env = "EXAV_BUILD_SHARD_BYTES", - value_parser = parse_size - )] - max_build_memory: Option, - /// Prefork only (requires --workers N): recycle a worker process after this /// many jobs to bound slow leaks/fragmentation; `off` or `0` never /// recycles. Mirrors Apache MaxRequestsPerChild. [default: 1000] @@ -834,7 +815,7 @@ struct Cli { /// default), `none`, or a comma-separated list. /// /// base64 A run long enough to hold an executable, decoded and rescanned - /// when it starts with an executable magic — how a PE reaches a + /// when it starts with an executable magic: how a PE reaches a /// machine inside a PowerShell, JS or RTF dropper. Also the /// base64 assets a markup document embeds, such as a `data:` URI /// image on a phishing page. @@ -897,17 +878,17 @@ struct Cli { /// Heuristic detectors to switch on, over and above the signature database: /// `none` (default), `all`, or a comma-separated list. /// - /// macros `Heuristics.OLE2.ContainsMacros` — an OLE2/OOXML + /// macros `Heuristics.OLE2.ContainsMacros`: an OLE2/OOXML /// document carrying VBA macros. - /// broken `Heuristics.Broken.Executable` — a PE/ELF/Mach-O + /// broken `Heuristics.Broken.Executable`: a PE/ELF/Mach-O /// magic whose headers do not parse. - /// broken-media `Heuristics.Broken.Media.*` — a structurally + /// broken-media `Heuristics.Broken.Media.*`: a structurally /// invalid GIF, PNG, TIFF or JPEG. /// partition-intersection overlapping partition entries in a disk image. - /// phishing `Heuristics.Phishing.Email.*` — a link whose + /// phishing `Heuristics.Phishing.Email.*`: a link whose /// visible text spoofs its href, hides the host /// behind userinfo, or is an IP under a brand name. - /// packed `Heuristics.Packed.*` — names the packer or + /// packed `Heuristics.Packed.*`: names the packer or /// protector wrapping an executable exav cannot /// unpack. Reported alongside the unscannable /// signal, not instead of it. @@ -916,12 +897,12 @@ struct Cli { /// Applied at database load, not per scan. /// exav-heuristics exav-exclusive statistical suspicion with no /// ClamAV equivalent: TLSH fuzzy matching, the - /// static ML scorer + /// hand-weighted static scorer /// (`Heuristics.Static.Suspect.*`) and /// packed-with-injection-imports. Higher /// false-positive risk, so opt-in. /// - /// What an *unscannable* object becomes is not here — that is a verdict + /// What an *unscannable* object becomes is not here: that is a verdict /// question, and --partial-as answers it. #[arg( long = "detect", @@ -954,9 +935,9 @@ struct Cli { /// ok Deliver it as clean. Exit 0, OK, an ICAP 204. This is what /// ClamAV does for an encrypted archive and what c-icap does past /// MaxObjectSize; a real trade, not a mistake, and exav will not - /// make it quietly — every such object is logged. + /// make it quietly: every such object is logged. /// found Report it as a detection named `Heuristics.*`. Exit 1, FOUND, - /// X-Infection-Found — an ordinary hit to any client, and what + /// X-Infection-Found: an ordinary hit to any client, and what /// ClamAV's --alert-exceeds-max / --alert-encrypted produce. /// error Report it as an operational failure. Exit 2, for a caller that /// would rather not learn a fourth exit code. @@ -968,7 +949,7 @@ struct Cli { /// /// On the clamd wire `partial` and `error` are both an `ERROR` reply: that /// protocol's vocabulary is closed, and a real client reads a word it does - /// not know as OK — a fail-open exav will not risk. They differ only where + /// not know as OK: a fail-open exav will not risk. They differ only where /// there is an exit code to differ in. #[arg( long = "partial-as", @@ -994,8 +975,8 @@ struct Cli { /// Read passwords from FILE, one per line, appended after `--passwords`. /// Lines are kept verbatim (only the line ending is stripped), so a - /// password containing a comma or leading/trailing spaces — inexpressible - /// on the command line — goes here. Passwords on a command line stay + /// password containing a comma or leading/trailing spaces (inexpressible + /// on the command line) goes here. Passwords on a command line stay /// visible in process listings; a file does not. #[arg( long = "passwords-from", @@ -1012,7 +993,7 @@ struct Cli { /// under ClamAV's vocabulary where the two engines name the same fact /// differently. It leaves `--max-matcher-bytes`, spill settings, `--detect`, /// and update/network/worker settings on exav defaults. This DELIBERATELY - /// REDUCES exav's detection capability so results reproduce clamscan's — it + /// REDUCES exav's detection capability so results reproduce clamscan's. It /// is a diff-testing mode, NOT recommended for production. Off by default /// (full capability). Each preset flag can still be set or overridden on its /// own; an explicit flag wins over the preset. @@ -1022,7 +1003,7 @@ struct Cli { /// Measure where scan time goes, per matcher. /// /// Scanning files, this replaces the normal output with a CSV row per file - /// (`_us`, `_calls`, `_bytes` per matcher) — a performance matrix over a + /// (`_us`, `_calls`, `_bytes` per matcher): a performance matrix over a /// dataset. On a listener there is no per-file output to put it in, so the /// same numbers accumulate and are reported through the clamd `STATS` /// command as a `MATCHERSTATS` line. @@ -1050,7 +1031,7 @@ struct Cli { )] slow_scan_secs: Option, - /// Seconds between the scan-totals lines a listener writes to its log — + /// Seconds between the scan-totals lines a listener writes to its log: /// scans, bytes, mean and slowest scan, throughput. `off` disables them. /// [default: 300] /// @@ -1073,7 +1054,8 @@ struct Cli { #[arg(long = "json", env = "EXAV_JSON", value_parser = env_switch())] json: bool, - /// Print informational findings (type, entropy, imphash, ml score). Those + /// Print informational findings: the detected type, and under + /// `--detect exav-heuristics` the imphash, section entropy and static score. Those /// come from the local scanner, and a daemon reply carries a verdict and /// nothing else, so in client mode this prints what the client itself knows: /// which daemon answered, and the command sent for each target. @@ -1171,7 +1153,7 @@ pub(crate) enum Descent { } /// How far a named directory is walked. Naming a directory means the directory: -/// scanning only its top level by default — as `clamscan` does — answers a +/// scanning only its top level by default, as `clamscan` does, answers a /// question nobody asked, and answers it in the shape of a clean result, because /// the files that were never opened look exactly like the ones that were fine. /// `--no-recursive` is the opt-out, and it reads the same on every surface that @@ -1186,8 +1168,8 @@ pub(crate) fn descent(cli: &Cli) -> Descent { /// Walk `root` for regular files, keeping the paths the walk could not reach. /// -/// Every scan surface — one-shot `-r`, the client, CONTSCAN, the daemon's -/// all-match tree — needs the same two facts, and the second is the one easy to +/// Every scan surface (one-shot `-r`, the client, CONTSCAN, the daemon's +/// all-match tree) needs the same two facts, and the second is the one easy to /// drop. An unreadable directory or a file removed mid-walk is a part of the /// tree nothing looked at; a surface that silently omits it reports on what it /// managed to reach and calls that the answer. That is a clean verdict over @@ -1279,7 +1261,7 @@ fn clamd_endpoint(cli: &Cli) -> Option { /// The ICAP endpoint, if one was asked for. /// /// Without the `icap` feature there is no such listener and the whole ICAP path -/// compiles out — an `icap://` address is then refused at startup rather than +/// compiles out: an `icap://` address is then refused at startup rather than /// silently ignored, because a listener that was asked for and never bound is a /// deployment that thinks it is scanning. #[cfg(feature = "icap")] @@ -1312,7 +1294,7 @@ fn updater_only(cli: &Cli) -> bool { } /// Default daemon worker count: one per CPU core on Unix (the prefork pool), -/// 0 elsewhere (the thread model — Unix-only `fork` isn't available). +/// 0 elsewhere (the thread model, since Unix-only `fork` isn't available). fn default_workers() -> usize { #[cfg(unix)] { @@ -1349,14 +1331,14 @@ fn metrics_interval(cli: &Cli) -> std::time::Duration { /// Unix filter does. /// /// Rust's runtime sets `SIGPIPE` to `SIG_IGN` before `main`, so a write to a -/// closed pipe returns `EPIPE` instead of killing the process — and `println!` +/// closed pipe returns `EPIPE` instead of killing the process, and `println!` /// turns that error into a panic. `exav /data | head -3` would then print a Rust /// backtrace at a user who did something completely ordinary. /// /// Restoring the default disposition makes the process die on the signal /// instead, silently, which is what `head` closing its end is supposed to mean. -/// The listeners want the opposite — a client hanging up must not stop a daemon -/// — so each of them sets `SIG_IGN` back when it starts serving. +/// The listeners want the opposite: a client hanging up must not stop a daemon, +/// so each of them sets `SIG_IGN` back when it starts serving. #[cfg(unix)] fn restore_default_sigpipe() { // SAFETY: `signal` here only sets this process's own disposition for one @@ -1374,7 +1356,7 @@ fn restore_default_sigpipe() {} /// first for the sake of the one-shot scan. #[cfg(unix)] fn ignore_sigpipe() { - // SAFETY: as above — this process's own disposition for one signal, before + // SAFETY: as above, this process's own disposition for one signal, before // any connection is accepted. unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN); @@ -1391,7 +1373,7 @@ fn ignore_sigpipe() {} /// muscle memory, before clap rejects them as unknown. /// /// exav's flags are its own and clamscan's names are deliberately not hidden -/// aliases (see the flag matrix) — but `error: unexpected argument '-r' found` +/// aliases (see the flag matrix), but `error: unexpected argument '-r' found` /// followed by a tip about `-- -r` tells someone nothing about what to use /// instead, and `-r` in particular asks for behaviour that is already the /// default. The whole audience for this tool arrives with those flags in their @@ -1443,8 +1425,8 @@ fn clamscan_flag_hint(args: &[String]) -> Option { } "--structured-ssn-count" => "use --dlp-ssns", "--structured-cc-count" => "use --dlp-credit-cards", - // The renamed limits. Aliases were rejected on purpose — one - // spelling per bound — and the migration guide's promise for that + // The renamed limits. Aliases were rejected on purpose (one + // spelling per bound), and the migration guide's promise for that // trade is "a clear error naming the exav flag", which is this. "--max-filesize" => "use --max-input-bytes, the largest top-level input scanned", "--max-scansize" => { @@ -1482,17 +1464,25 @@ fn clamscan_flag_hint(args: &[String]) -> Option { /// does not, with what replaced it. The variable is checked too: clap ignores /// one it does not know, which would drop the setting without a word. fn removed_flag(args: &[String]) -> Option { - const EXTRACTED: &str = "--max-extracted-bytes was removed in 0.0.2: \ - --max-process-bytes bounds the memory a scan may use, and \ - --max-object-bytes one object"; - let given = args - .iter() - .any(|a| a.split('=').next() == Some("--max-extracted-bytes")); - (given || std::env::var_os("EXAV_MAX_EXTRACTED_BYTES").is_some()).then(|| { - if given { - EXTRACTED.to_string() + const REMOVED: [(&str, &str, &str); 2] = [ + ( + "--max-extracted-bytes", + "EXAV_MAX_EXTRACTED_BYTES", + "--max-process-bytes bounds the memory a scan may use, and \ + --max-object-bytes one object", + ), + ( + "--build-shard-bytes", + "EXAV_BUILD_SHARD_BYTES", + "building a database no longer has a matcher to split", + ), + ]; + REMOVED.iter().find_map(|&(flag, var, instead)| { + let said = format!("{flag} was removed in 0.0.2: {instead}"); + if args.iter().any(|a| a.split('=').next() == Some(flag)) { + Some(said) } else { - format!("EXAV_MAX_EXTRACTED_BYTES is set, and {EXTRACTED}") + std::env::var_os(var).map(|_| format!("{var} is set, and {said}")) } }) } @@ -1517,7 +1507,7 @@ fn main() -> ExitCode { // documented precedence is that a flag on the command line wins over the // matching variable. Without this it does not: the container image sets // `EXAV_LISTEN`, so `docker run … image /scan` and `docker exec … --connect` - // both die on "a run does one or the other" — the image's own one-shot + // both die on "a run does one or the other": the image's own one-shot // example, and any exec-form HEALTHCHECK, which has no shell to unset it. // // Only an environment-supplied listener yields, and only to an explicit @@ -1584,8 +1574,8 @@ fn main() -> ExitCode { // reports anything, so they have to be installed before the first one runs. // // `--clamav-compat` reports a partial as `ok`, because that is what a stock - // ClamAV build answers for this whole class — over `--max-filesize`, an - // encrypted archive, a container it cannot decode: `OK`, exit 0. A + // ClamAV build answers for this whole class (over `--max-filesize`, an + // encrypted archive, a container it cannot decode): `OK`, exit 0. A // differential run that answered `PARTIAL` where clamscan answers `OK` would // report a difference on every such file that is nothing to do with // detection. An explicit `--partial-as` still wins, as every value in the @@ -1670,12 +1660,10 @@ fn main() -> ExitCode { return ExitCode::from(2); } - // The per-job limits are enforced by the kernel inside worker processes, so - // they only exist with a prefork pool. Reject them with workers=0 rather - // than silently ignoring them. - // The prefork pool (and its per-job limits) only exists for the daemon, with - // workers > 0. Default to the CPU-core count on Unix; --workers 0 forces the - // in-process thread model. One-shot scans never use the pool. + // The prefork pool (and its per-job limits, which the kernel enforces inside + // worker processes) only exists for the daemon. Default to the CPU-core + // count on Unix; `--workers threads` (0 here) is the in-process thread + // model. One-shot scans never use the pool. let pool_workers = if clamd_endpoint(&cli).is_some() { cli.workers.unwrap_or_else(default_workers) } else { @@ -1741,7 +1729,7 @@ fn main() -> ExitCode { if pool_workers == 0 { // `--max-jobs-per-worker` counts jobs before a worker is recycled, so it // means nothing without workers. The other two bound a scan, and a scan - // outside the pool needs bounding just as much — arguably more, since + // outside the pool needs bounding just as much, arguably more, since // there is no parent to reap a run that never ends. if cli.max_jobs_per_worker.is_some() { eprintln!( @@ -1753,11 +1741,11 @@ fn main() -> ExitCode { // A clamd listener under `--workers threads` also lands here, and it is // a long-running server rather than one scan. `apply_oneshot_limits` arms a // process-wide `ITIMER_REAL`, which for a server means the whole daemon - // exits that many seconds after startup — mid-scan, or while idle. The + // exits that many seconds after startup, mid-scan or while idle. The // thread model has no per-job timer to enforce the flag with, so it is // refused rather than turned into a countdown to shutdown. // - // The ICAP server is the same shape — threads, no per-job kill — so the + // The ICAP server is the same shape (threads, no per-job kill), so the // flag is refused there for the same reason. if serves && cli.max_scan_time.is_some() { // Refused rather than quietly downgraded: the flag means "kill the @@ -1822,7 +1810,7 @@ fn main() -> ExitCode { }; // Refuse to operate with no real signature database (absent, empty, or a - // valid-but-signature-less DB) — a scan or daemon would report real malware as + // valid-but-signature-less DB): a scan or daemon would report real malware as // clean against the near-zero-coverage EICAR-only baseline, a silent miss. // Applies to one-shot scans, the listeners, and --build-db alike. Opt into the // baseline explicitly with --allow-no-db (testing/CI only). @@ -1870,7 +1858,7 @@ fn main() -> ExitCode { // Keep the three layers in their intended order outside the pool too: the // in-core budget decides first and produces a verdict, and the kernel cap // is only the backstop. Without this the 1 GiB default budget can exceed - // an address space the operator just asked for, and the kernel wins — so a + // an address space the operator just asked for, and the kernel wins, so a // scan that should report a limit gets killed for hitting one. #[cfg(unix)] if pool_workers == 0 { @@ -1901,14 +1889,22 @@ fn main() -> ExitCode { if cli.allow_http_scan { scan_url(s, &db, &opts, &cli, &mut totals) } else { - totals.errors += 1; - eprintln!("{s}: URL scanning disabled; pass --allow-http-scan ERROR"); + report_error( + s, + "URL scanning disabled; pass --allow-http-scan", + &cli, + &mut totals, + ); } } #[cfg(not(feature = "http-scan"))] Some(s) if s.starts_with("http://") || s.starts_with("https://") => { - totals.errors += 1; - eprintln!("{s}: URL scanning needs a build with `--features http-scan` ERROR"); + report_error( + s, + "URL scanning needs a build with `--features http-scan`", + &cli, + &mut totals, + ); } _ => scan_target(path, &db, &opts, &cli, &filters, &mut totals), } @@ -1927,24 +1923,24 @@ fn main() -> ExitCode { /// What a finished run exits with. /// -/// 0 clean — everything was scanned, nothing matched +/// 0 clean: everything was scanned, nothing matched /// 1 a detection /// 2 an error: exav could not do its job (an unreadable path, a database that /// would not load). This is `clamscan`'s meaning of 2, and only that. -/// 3 not scanned: exav worked, but something could not be fully examined — -/// `LIMITS-EXCEEDED`, `UNSCANNABLE`, `PASSWORD-PROTECTED`. +/// 3 not scanned: exav worked, but something could not be fully examined +/// (`LIMITS-EXCEEDED`, `UNSCANNABLE`, `PASSWORD-PROTECTED`). /// /// The last two are separated because they ask different things of a caller. A /// `2` says the scanner is broken or misconfigured and the run's result cannot /// be trusted; a `3` says the scanner worked and this particular object needs a -/// policy decision. Collapsing them into one code — which is what `clamscan` -/// does, by calling the whole third class `OK` and exiting 0 — is what leaves an +/// policy decision. Collapsing them into one code (which is what `clamscan` +/// does, by calling the whole third class `OK` and exiting 0) is what leaves an /// operator unable to tell "my scanner is down" from "someone uploaded an /// encrypted zip". /// /// A detection outranks both. Finding malware is conclusive: that a limit was /// also hit, or another file failed to open, does not make the match less true. -/// An error outranks a partial file for the opposite reason — it casts doubt +/// An error outranks a partial file for the opposite reason: it casts doubt /// on the whole run, where a partial file is a fact *about that file*. fn exit_code(totals: &Totals) -> ExitCode { if totals.infected > 0 { @@ -1961,7 +1957,7 @@ fn exit_code(totals: &Totals) -> ExitCode { /// The path the daemon should watch for on-disk signature changes, mirroring how /// `load_db` picks its source. For `--database` this is the given path whether it /// is a **directory** (a sidecar/freshclam rewriting the volume) or a single -/// **file** (a prebuilt `.exavdb` database atomically swapped in place) — the mtime poll +/// **file** (a prebuilt `.exavdb` database atomically swapped in place); the mtime poll /// handles both, so a database-file deployment hot-reloads on swap without needing /// an explicit `RELOAD`. `None` only for the built-in baseline (no source path). fn reload_watch_dir(cli: &Cli) -> Option { @@ -2045,7 +2041,7 @@ fn run_listeners(cli: &Cli, db: Scanner, pool_workers: usize) -> ExitCode { if clamd.is_none() { let server = icap_server.expect("an icap:// address is the only listener asked for"); // A proxy that hangs up mid-response must cost one connection, not the - // listener — the same reason the clamd daemon does this. Set here rather + // listener, the same reason the clamd daemon does this. Set here rather // than inside the ICAP module, which is `forbid(unsafe_code)` and stays // that way. ignore_sigpipe(); @@ -2186,7 +2182,7 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { } let mut opts = ScanOptions::default(); // Report under ClamAV's vocabulary where the two engines name the same fact - // differently. Affects names only — nothing is detected in one mode and not + // differently. Affects names only: nothing is detected in one mode and not // the other. opts.clamav_compat = compat; // What a scan makes of an object too large to hold goes to the spill files @@ -2199,7 +2195,7 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { // `0` is ClamAV's spelling of "no limit" for both size flags, and exav's own // `--max-scan-secs`/`--max-process-bytes` already read it that way. Taking it // literally instead turns a request for no limit into a limit of zero, which - // refuses every file with a byte in it — the opposite of what was asked, and + // refuses every file with a byte in it: the opposite of what was asked, and // an operator reading ClamAV's documentation has no reason to expect it. An // explicit `0` wins over the compat default: the flag was given. opts.max_scan_size = match cli.max_input_bytes { @@ -2225,7 +2221,7 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { opts.deep_analysis_max = b; } // --max-matcher-bytes: the cumulative scan-reach (CPU/time) limit. Decoupled - // from memory — a streamed member is bounded by this, not by the buffer cap, + // from memory: a streamed member is bounded by this, not by the buffer cap, // so raising it scans larger members (in RAM bounded by // --max-object-bytes) at the cost of scan time only. if let Some(s) = cli.max_scanned_bytes.map(no_limit_at_zero) { @@ -2246,7 +2242,7 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { // Narrowing the unpacking reach to stock ClamAV's, and naming signatures the // way clamscan does, are both only ever wanted for a differential run, so - // the preset is the whole interface — flags of their own would be two more + // the preset is the whole interface; flags of their own would be two more // ways to ask for one mode. opts.restrict_extractors = compat; opts.unofficial_suffix = compat; @@ -2265,7 +2261,7 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { opts.decode_base64 = decode.base64(); // `clamav_heuristics` (PDF ObfuscatedNameObject, imphash `.imp` matching) is on - // by default from `ScanOptions::default()` — a faithful out-of-the-box scan — + // by default from `ScanOptions::default()` (a faithful out-of-the-box scan), // and `--clamav-compat` / `--detect exav-heuristics` keep it on. Nothing turns it // off from the CLI, so no assignment here. opts.passwords = cli.password.clone(); @@ -2297,7 +2293,7 @@ fn build_scan_options(cli: &Cli) -> ScanOptions { /// nothing to send to would scan nothing while looking like it had. /// Build the spill settings from the flags and install them. /// -/// The budgets nest — RAM, then one object, then the process — so a +/// The budgets nest (RAM, then one object, then the process), so a /// configuration that inverts the nesting is refused here rather than at the /// first object large enough to expose it. An operator who sets a 4 GiB /// per-object cap under a 1 GiB total has said two things that cannot both hold, @@ -2336,7 +2332,7 @@ fn configure_spill(cli: &Cli) -> Result<(), String> { }; if !cfg.enabled { // The disk budgets describe a disk nothing will be written to, so they - // are not checked against each other — but saying both is a + // are not checked against each other, but saying both is a // contradiction worth naming rather than silently resolving. if cli.max_spill_bytes.is_some() || cli.max_total_spill_bytes.is_some() { return Err("--spill-dir off leaves nothing for --max-spill-bytes / \ @@ -2414,7 +2410,7 @@ const CLIENT_HONOURS: &[&str] = &[ /// `--max-input-bytes 1M` on a client is a size cap nobody applies: it parses, /// it looks like it is in force, and the daemon scans under its own limits. /// exav's answer to a flag it cannot honour is to stop, and this is that answer -/// for the client — the same reasoning `--partial-as` already gets below, +/// for the client: the same reasoning `--partial-as` already gets below, /// applied to the rest of the engine rather than to the one flag someone /// happened to try. /// @@ -2458,7 +2454,7 @@ fn detectors(cli: &Cli) -> policy::Detectors { fn check_flag_conflicts(cli: &Cli) -> Result<(), String> { // The policy belongs to whoever scans. A client only ever sees the reply the // daemon already decided, so this flag would parse, look like it was in - // force, and change nothing — and it cannot be made to work: once the daemon + // force, and change nothing. And it cannot be made to work: once the daemon // has reported `OK` for something it passed, the fact is gone from the wire. if cli.partial_as.is_some() && cli.connect.is_some() { return Err( @@ -2578,7 +2574,7 @@ fn check_flag_conflicts(cli: &Cli) -> Result<(), String> { } // A --connect client dials the address and honours none of its server-side // tunables, so an address carrying any is refused rather than quietly - // dropped — a pasted listen address the operator believes is tuned. Only + // dropped: a pasted listen address the operator believes is tuned. Only // the marked options count; a future client-side key passes through. if let Some(addr) = cli.connect.as_deref() { if let Ok(e) = endpoint::Endpoint::parse(addr) { @@ -2593,7 +2589,7 @@ fn check_flag_conflicts(cli: &Cli) -> Result<(), String> { } } // Serving is chosen before `--build-db` is ever looked at, so without this - // the pair does not fail — it silently serves, builds nothing, and leaves + // the pair does not fail: it silently serves, builds nothing, and leaves // the operator waiting on a database that is never written. if serves(cli) && cli.build_db.is_some() { return Err( @@ -2603,7 +2599,7 @@ fn check_flag_conflicts(cli: &Cli) -> Result<(), String> { ); } // An `icap://` address in a build with no ICAP listener is a listener that - // was asked for and will never be bound — the shape of deployment that + // was asked for and will never be bound: the shape of deployment that // believes it is scanning. Refused rather than ignored. #[cfg(not(feature = "icap"))] if listeners(cli) @@ -2637,7 +2633,7 @@ fn baseline_sig_count() -> usize { } /// Whether `db` carries no real detection capability beyond the built-in -/// baseline — an absent DB, an empty/junk `--sig-dir`, OR a *valid-but-empty* +/// baseline: an absent DB, an empty/junk `--sig-dir`, OR a *valid-but-empty* /// loaded database (e.g. a build server that shipped a signature-less `.exavdb`). /// Checking the loaded count, not just whether a source path exists, is what /// makes this hard to fool: a reachable daemon in this state would report real @@ -2668,11 +2664,6 @@ fn guard_not_empty(db: Scanner, source: &str, allow_no_db: bool) -> Result Option { - cli.max_build_memory.filter(|&b| b != 0) -} - fn load_db(cli: &Cli) -> Result { // `--clamav-compat` selects exact ClamAV naming: the `.UNOFFICIAL` suffix on // signatures from an unofficial database. Provenance is @@ -2680,10 +2671,9 @@ fn load_db(cli: &Cli) -> Result { // report time from `ScanOptions::unofficial_suffix`, so one loaded database // serves both compat and non-compat scans. let suffix = cli.clamav_compat; - let bmem = build_shard_budget(cli); let pua = detectors(cli).pua(); if let Some(path) = &cli.database { - return loader::load_with_options_mem(path, pua, suffix, bmem).map_err(|e| e.to_string()); + return loader::load_with_options(path, pua, suffix).map_err(|e| e.to_string()); } if cli.sigs.is_dir() { // Use the data dir if it actually contains something loadable. @@ -2691,8 +2681,7 @@ fn load_db(cli: &Cli) -> Result { .map(|mut d| d.next().is_some()) .unwrap_or(false) { - return loader::load_with_options_mem(&cli.sigs, pua, suffix, bmem) - .map_err(|e| e.to_string()); + return loader::load_with_options(&cli.sigs, pua, suffix).map_err(|e| e.to_string()); } } Ok(Scanner::builtin()) @@ -2735,7 +2724,7 @@ fn scan_target( } /// Whether a path's *filename* marks it as one part of a byte-split archive. -/// Names only — nothing is opened to decide this. +/// Names only: nothing is opened to decide this. fn is_volume_part(path: &Path) -> bool { path.file_name() .and_then(|n| n.to_str()) @@ -2747,7 +2736,7 @@ fn is_volume_part(path: &Path) -> bool { /// /// Reported under the archive's own name (`dir/big.7z`), not a fragment's: that /// is the object that was scanned, and it is what an operator needs to see. The -/// per-part lines already printed stand — each said only that the fragment is +/// per-part lines already printed stand: each said only that the fragment is /// not itself malware, which remains true. /// /// Grouped per directory: `a/big.7z.001` and `b/big.7z.002` are unrelated files @@ -2869,8 +2858,8 @@ impl<'a> Input<'a> { let size = file.metadata()?.len(); exav_core::analyze_all_seekable(db, file, size, opts).map(Some) } - Self::Held(held, _) if held.short.is_none() => { - held.payload.all_matches(db, opts).map(Some) + Self::Held(held, n) if held.short.is_none() => { + held.payload.all_matches(db, opts, *n).map(Some) } Self::Held(..) => Ok(None), } @@ -2887,6 +2876,12 @@ fn scan_one(path: &Path, db: &Scanner, opts: &ScanOptions, cli: &Cli, totals: &m } } +/// The bytes of an input of `size` bytes a scan reads: its first +/// `--max-input-bytes` when it is larger. +fn scanned_of(size: u64, opts: &ScanOptions) -> u64 { + opts.max_scan_size.map_or(size, |max| size.min(max)) +} + /// Scan one object, named `path` in the output. fn scan_input( path: &Path, @@ -2901,9 +2896,9 @@ fn scan_input( } totals.scanned += 1; let size = input.size(); - totals.data_scanned += size; + totals.data_scanned += scanned_of(size, opts); // Isolate each file: a parser panic on a crafted input must not abort - // the whole run, and must count as an error — never a clean result. + // the whole run, and must count as an error, never a clean result. let t0 = std::time::Instant::now(); if cli.profile { exav_core::profile::enable(); @@ -2924,14 +2919,14 @@ fn scan_input( Ok(Ok(r)) => { let v = &r.verdict; // Counted, not just labelled, and counted the way `report_result` - // counts — through the exhaustive `VerdictCategory`. The CSV row + // counts: through the exhaustive `VerdictCategory`. The CSV row // records what happened, but the exit code is the machine-readable // answer, and a run whose files all hit limits exiting 0 tells a // script every one of them was scanned and clean. match v.category() { VerdictCategory::Infected => totals.infected += 1, // As in `report_result`: `--partial-as error` changes no - // verdict, only which counter — and so which exit code — + // verdict, only which counter (and so which exit code) // this object contributes to. VerdictCategory::Partial => { if policy::current().for_tag(v.status_tag()) == policy::PartialStatus::Error @@ -2950,8 +2945,8 @@ fn scan_input( Verdict::Unscannable { reason } => ("unscannable", reason.clone()), Verdict::PasswordProtected { reason } => ("password-protected", reason.clone()), // `Verdict` is `#[non_exhaustive]`. This is a profiling - // column, not a verdict decision — the counters above already - // classified it — so an unrecognised outcome is labelled + // column, not a verdict decision (the counters above already + // classified it), so an unrecognised outcome is labelled // rather than guessed at, and never labelled clean. _ => ("other", String::new()), } @@ -2987,7 +2982,7 @@ fn scan_input( /// /// In `--json` mode this emits an object like every other result. Printing only /// to stderr leaves a JSONL consumer with fewer objects than it sent paths, and -/// nothing in the stream saying which path is missing or why — under +/// nothing in the stream saying which path is missing or why; under /// `--quiet` there is no trace at all. A record that says "error" is the /// difference between a consumer that can react and one that cannot tell. fn report_error(name: &str, message: &str, cli: &Cli, totals: &mut Totals) { @@ -3031,8 +3026,8 @@ fn scan_stdin(db: &Scanner, opts: &ScanOptions, cli: &Cli, totals: &mut Totals) } } -/// Scan an http(s):// URL via range requests, fetching only the bytes the -/// scan touches (e.g. a ZIP's directory + the members it reads). +/// Scan an http(s):// URL via range requests. One within `--max-object-bytes` +/// is fetched whole; a larger one is read through the block cache. #[cfg(feature = "http-scan")] fn scan_url(url: &str, db: &Scanner, opts: &ScanOptions, cli: &Cli, totals: &mut Totals) { totals.scanned += 1; @@ -3107,8 +3102,8 @@ fn ping_target(cli: &Cli) -> Result { /// Ask the daemon at `target` whether it is answering. /// -/// One exchange in the protocol the endpoint actually speaks — `PING`/`PONG` on -/// clamd, `OPTIONS` on ICAP — because a TCP connect alone goes green on a daemon +/// One exchange in the protocol the endpoint actually speaks (`PING`/`PONG` on +/// clamd, `OPTIONS` on ICAP), because a TCP connect alone goes green on a daemon /// that accepts and then answers nothing, which is the failure a health check /// exists to catch. fn ping_once(target: &endpoint::Endpoint) -> io::Result { @@ -3152,7 +3147,7 @@ fn ping_once(target: &endpoint::Endpoint) -> io::Result { fn run_ping(cli: &Cli) -> ExitCode { // `clamdscan --ping 1` means one attempt; `clamdscan --ping 5:2` means five, // two seconds apart. exav probes once and leaves retrying to whatever is - // asking — but a number here came from someone expecting attempts, and + // asking, but a number here came from someone expecting attempts, and // silently treating it as a file to scan (or ignoring it) would let them // believe they had configured a retry they have not. if !cli.paths.is_empty() { @@ -3205,8 +3200,8 @@ fn read_reply(r: &mut impl std::io::BufRead) -> io::Result> { /// Read every reply a command produced, up to the daemon closing the /// connection. That close is the end marker for anything sent outside a -/// session, and the only one a command whose reply count is not announced — -/// `CONTSCAN`, `ALLMATCHSCAN` — has. +/// session, and the only one a command whose reply count is not announced +/// (`CONTSCAN`, `ALLMATCHSCAN`) has. /// /// The second half of the pair is the reason the stream ended, when it ended /// badly. The lines that did arrive are still returned, because they name files @@ -3244,7 +3239,7 @@ fn split_session_id(msg: &str) -> (Option, &str) { /// Read one session command's replies, up to the `PONG` that ends them. /// /// A session announces no reply count and sends no terminator between commands, -/// and a scan can answer with any number of messages — `SCAN` on a directory +/// and a scan can answer with any number of messages: `SCAN` on a directory /// answers one per file in it. A client that reads a fixed single line reports /// the first verdict, drops every other one, and then reads the leftovers as /// the next command's answer. `PING` is therefore sent behind each scan: the @@ -3302,7 +3297,7 @@ fn client_verbose_cmd(cli: &Cli, verb: &str, target: &str) { } /// With `--verbose`, name the daemon that is about to answer: the endpoint the -/// client reached and the engine/signature version it reports. Best-effort — a +/// client reached and the engine/signature version it reports. Best-effort: a /// daemon that cannot be reached is reported by the scan itself. fn client_banner(cli: &Cli) { use std::io::{BufReader, Write}; @@ -3431,7 +3426,7 @@ fn client_send_file(cli: &Cli, path: &Path) -> io::Result { /// (`EXINSTREAM MULTI`) so the daemon rejoins them and scans the archive they /// form, reporting the verdict on each part. /// -/// Streaming each part on its own would collect a clean answer per fragment — +/// Streaming each part on its own would collect a clean answer per fragment: /// true of every fragment, and no answer at all about the archive, which is the /// object the malware is in. In path mode the daemon does this job from /// `CONTSCAN`; here the bytes have to be sent together for it to be possible. @@ -3529,7 +3524,7 @@ fn set_entry_line(target: &str, entry: &serde_json::Value) -> String { Some(s) => format!(" (in {s})"), None => String::new(), }; - // The same grammar every other line uses — `reason CATEGORY STATUS` — built + // The same grammar every other line uses (`reason CATEGORY STATUS`), built // from the reply's own `status` / `category` / `reason`. The wire word for a // partial is `ERROR`, as it is everywhere on the clamd protocol. match field("status") { @@ -3576,7 +3571,7 @@ fn run_client(cli: &Cli) -> ExitCode { // bytes, not a tree it could descend. let sending_contents = cli.send_as.unwrap_or_default().sends_contents(); // `-` is stdin. There is no path to name, so it goes as content whatever the - // mode — which is exactly why it works against a daemon on another host. + // mode, which is exactly why it works against a daemon on another host. let mut scan_stdin = false; let mut files = Vec::new(); let mut walk_errors = Vec::new(); @@ -3592,7 +3587,7 @@ fn run_client(cli: &Cli) -> ExitCode { // A directory recurses by default in client mode, with or without `-r`: // clamdscan has no such flag, so a command line migrated from it names a // tree and expects the tree scanned, and the daemon descends into any - // path it is handed anyway. `--no-recursive` still opts out — the client + // path it is handed anyway. `--no-recursive` still opts out: the client // does the walking itself, so it is the one thing here that can honour // it, and a flag that is quietly dropped is how a partial scan comes back // looking complete. `--exclude`/`--include` apply for the same reason, @@ -3614,8 +3609,8 @@ fn run_client(cli: &Cli) -> ExitCode { } if walk.files.iter().any(|f| is_volume_part(f)) { // Handed to `CONTSCAN` whole rather than scanned file by file. - // The daemon reports a set's verdict on each PART's line — the - // part is the file an operator has to act on — so asking for the + // The daemon reports a set's verdict on each PART's line (the + // part is the file an operator has to act on), so asking for the // files individually and the set separately would report each // part twice and disagree with itself. rejoin_dirs.push(p.clone()); @@ -3677,7 +3672,7 @@ fn run_client(cli: &Cli) -> ExitCode { }; // One command per file, so the extra lines are further signatures // for the same file and the summary counts it once. A file the - // daemon never answered about is counted too — it was named, and the + // daemon never answered about is counted too: it was named, and the // summary says what became of each name. totals.scanned += 1; for line in lines { @@ -3715,14 +3710,14 @@ fn run_client(cli: &Cli) -> ExitCode { let name = abs.display().to_string(); // A URL is not a path the daemon can open: ask for SCANURL and let // the daemon decide (it honours it only with --allow-http-scan). The - // client needs no flag of its own — it fetches nothing itself. + // client needs no flag of its own: it fetches nothing itself. let verb = if name.starts_with("http://") || name.starts_with("https://") { "SCANURL" } else { "SCAN" }; client_verbose_cmd(cli, verb, &name); - // `PING` rides behind the scan as its end marker — see + // `PING` rides behind the scan as its end marker. See // [`read_session_replies`] for why a session needs one. let cmd = format!("z{verb} {name}\0zPING\0"); if conn @@ -3772,7 +3767,7 @@ fn run_client(cli: &Cli) -> ExitCode { /// /// The connection is the framing: the daemon closes it after a single command /// outside a session, and that close is the end marker `CONTSCAN` and -/// `ALLMATCHSCAN` have — neither says how many files it is about to answer for. +/// `ALLMATCHSCAN` have: neither says how many files it is about to answer for. /// /// A target the daemon owed an answer about and did not give one is reported /// here, so no caller can mistake an empty reply list for nothing to find. @@ -3878,7 +3873,7 @@ fn print_daemon_reply(line: &str, cli: &Cli, totals: &mut Totals) { // A verdict earned by a rejoined multi-volume set is reported on each part // as `: FOUND (in )`. The status word is therefore not last, // and a check anchored to the end of the line reads a detection as an - // unrecognised line — which prints and counts as clean. Strip the annotation + // unrecognised line, which prints and counts as clean. Strip the annotation // before classifying; the printed line keeps it, because it is what tells an // operator which archive the part belongs to. let verdict = strip_set_annotation(line); @@ -3950,7 +3945,7 @@ fn json_daemon_reply(line: &str, cli: &Cli, totals: &mut Totals) { if let Some(tag) = partial_category(line) { totals.limits += 1; // The reason is what is left once the category and the status word - // are taken off — the same string `emit_json_result` puts under + // are taken off: the same string `emit_json_result` puts under // `reason` for a local scan. Split from the FRONT here, unlike the // other two branches: a reason is a sentence and can carry `": "`, // where a signature name cannot. @@ -3998,9 +3993,9 @@ impl ReadWrite for std::net::TcpStream { } } -/// `--all-matches` scan of one object: report every matching signature. Works -/// on a buffered copy (bounded by deep-analysis-max); a larger object falls -/// back to a normal single-match scan so it is never silently skipped. +/// `--all-matches` scan of one object: report every matching signature, at any +/// size. A stream that could not be held whole gets the single-match scan +/// instead, so it is never silently skipped. fn scan_one_allmatch( path: &Path, input: &Input, @@ -4013,13 +4008,10 @@ fn scan_one_allmatch( // Counted here as well as on the single-match path: a summary reporting // "Data scanned: 0.00 MB" for a multi-megabyte archive reads exactly like a // scan that skipped its contents, and is expensive to tell apart from one. - totals.data_scanned += input.size(); + totals.data_scanned += scanned_of(input.size(), opts); let name = path.display().to_string(); - // `max_scan_size` is how much the operator said may be scanned at all, and - // the all-match walk does not enforce it: a file over it must reach the - // single-match path, where the ceiling is, or `--all-matches` would answer - // `OK` for a file `--max-input-bytes` says was never fully examined. So - // must an input not held whole. + // An input not held whole has no all-match walk, and gets the single-match + // scan. let fall_back = |totals: &mut Totals| { // The single-match scan scans the budgeted prefix before reporting the // limit, so a detection in the part that did fit still wins. @@ -4034,9 +4026,6 @@ fn scan_one_allmatch( Err(_) => report_error(&name, "internal error while scanning", cli, totals), } }; - if opts.max_scan_size.is_some_and(|max| input.size() > max) { - return fall_back(totals); - } let found = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| input.all_matches(db, opts))); let found = match found { @@ -4076,7 +4065,7 @@ fn scan_one_allmatch( } } // `--partial-as ok` asked for partials to pass and `--partial-as - // found` for them to read as detections — which this object already + // found` for them to read as detections, which this object already // is. Either way the extra line would be noise the operator asked // not to get, so it is only emitted for the two statuses that mean // "tell me". @@ -4134,7 +4123,7 @@ fn scan_one_allmatch( } } } - // No detections — but "found nothing" and "did not look at all of it" are + // No detections, but "found nothing" and "did not look at all of it" are // different answers, and printing OK for both is a silent clean. // // Turned back into a `ScanReport` and handed to `report_result` rather @@ -4178,8 +4167,8 @@ fn emit_json_summary(totals: &Totals, elapsed: std::time::Duration) { } fn report_result(name: &str, mut report: ScanReport, cli: &Cli, totals: &mut Totals) { - // Every scan the CLI reports comes through here — human output, JSON, - // counters and therefore the exit code — so the partial policy is + // Every scan the CLI reports comes through here (human output, JSON, + // counters and therefore the exit code), so the partial policy is // applied once, in front of all of them. Applied per output mode instead, a // `pass` would have had to be remembered four times, and forgetting the // counters would mean an object the operator asked to pass still exiting 2. @@ -4194,8 +4183,8 @@ fn report_result(name: &str, mut report: ScanReport, cli: &Cli, totals: &mut Tot match v.category() { VerdictCategory::Infected => totals.infected += 1, // `--partial-as error` is the one status `apply` cannot express in the - // report: it changes no verdict, only which counter — and therefore - // which exit code — this object contributes to. The line still names + // report: it changes no verdict, only which counter (and therefore + // which exit code) this object contributes to. The line still names // the category, so the report has to keep it. VerdictCategory::Partial => { if policy::current().for_tag(v.status_tag()) == policy::PartialStatus::Error { @@ -4228,7 +4217,7 @@ fn report_result(name: &str, mut report: ScanReport, cli: &Cli, totals: &mut Tot print!("\x07"); } } - // Limit hit / undecodable / encrypted — all "work happened and stopped + // Limit hit / undecodable / encrypted: all "work happened and stopped // short". One grammar with every other line: `path: [reason ][CATEGORY ] // STATUS`, the status word last, which is where `clamscan` puts `OK` and // `FOUND` and therefore where anything reading these lines looks. @@ -4264,7 +4253,7 @@ fn report_result(name: &str, mut report: ScanReport, cli: &Cli, totals: &mut Tot /// The status word for a report, in JSON as on a line. /// -/// The same four values everywhere — `OK`, `FOUND`, `ERROR`, `PARTIAL` — each +/// The same four values everywhere (`OK`, `FOUND`, `ERROR`, `PARTIAL`), each /// naming the exit code it contributes, so a machine consumer and a human read /// the same vocabulary and neither needs a translation table. fn status_str(c: VerdictCategory, tag: &str) -> &'static str { @@ -4328,7 +4317,7 @@ fn emit_json_result(name: &str, report: &ScanReport, cli: &Cli) { } /// ClamAV functionality level exav emulates (see `engine::EXAV_FLEVEL`), and the -/// ClamAV release that flevel corresponds to — reported as the engine version so +/// ClamAV release that flevel corresponds to, reported as the engine version so /// clamscan-parsing tooling sees a recognised, recent engine. pub(crate) const CLAMAV_COMPAT_VERSION: &str = "1.4.3"; @@ -4361,7 +4350,7 @@ fn print_summary(db: &Scanner, totals: &Totals, elapsed: std::time::Duration, ve /// /// `0` is refused rather than accepted as the thread model, which is what it /// used to mean. A count of zero reads as "no workers", not "a different process -/// architecture" — and the two models differ in isolation, in whether a single +/// architecture", and the two models differ in isolation, in whether a single /// job can be killed, and in whether the listeners share one set of counters. /// None of that is something a reader infers from a digit. fn parse_workers(s: &str) -> Result { @@ -4382,7 +4371,7 @@ fn parse_workers(s: &str) -> Result { /// Parse a number of seconds, or `off`. /// /// `off` rather than `0`, because for a duration `0` has an honest second -/// reading — "immediately", "every time" — and a flag whose disable value is +/// reading ("immediately", "every time"), and a flag whose disable value is /// also a plausible setting is one an operator can get backwards without ever /// seeing an error. `0` is refused and says which word to use. fn parse_secs_or_off(s: &str) -> Result { @@ -4606,7 +4595,7 @@ mod tests { /// `--ping` exists, but not clamdscan's `attempts[:interval]` argument. /// - /// It parses — `1` is simply a path as far as clap is concerned — so the + /// It parses (`1` is simply a path as far as clap is concerned), so the /// refusal has to come from `run_ping`. Left to itself the number would be /// quietly dropped and the operator would believe they had asked for five /// attempts when they had asked for one. @@ -4734,7 +4723,7 @@ mod tests { ); // What the client itself does: walk, send, print. Short spellings - // included — `-v` is `--verbose`, not an unknown flag. + // included: `-v` is `--verbose`, not an unknown flag. assert!(ignored(&[ "--connect", "/run/exav.sock", @@ -4798,8 +4787,8 @@ mod tests { /// answer about, and only the marker says it has stopped. /// /// Reading a fixed single line takes the first verdict, drops the rest, and - /// leaves them in the socket for the next command to read as its own answer - /// — a detection lost and a verdict misattributed from one read. + /// leaves them in the socket for the next command to read as its own answer: + /// a detection lost and a verdict misattributed from one read. #[test] fn a_session_reply_runs_to_its_marker() { let wire: &[u8] = b"1: /t/a: OK\x001: /t/b: Eicar-Test-Signature FOUND\x00\ @@ -4856,7 +4845,7 @@ mod tests { } /// Only a leading number is a command id. A verdict line carries colons of - /// its own — the one after the path, and any inside the path — and reading + /// its own (the one after the path, and any inside the path), and reading /// one of those as the id would strip part of the file name off the reply. #[test] fn only_a_leading_number_is_a_session_id() { @@ -4877,7 +4866,7 @@ mod tests { /// text promises refusal is the default, so the default is the contract. /// /// This asserts the policy function rather than any one listening mode - /// because the defect it replaces was not in the policy — it was three + /// because the defect it replaces was not in the policy: it was three /// modes each deciding it separately, one of them the other way round. /// `shutdown_allowed` exists so there is only one answer to assert. /// @@ -4940,7 +4929,7 @@ mod tests { /// Pins the flag→field mapping in [`build_scan_options`], for exav's own /// flag names and for every clamscan alias that has to keep working. Both /// tables are asserted against the same sentinels, so an alias silently - /// detaching from its field — or landing on a neighbouring one — fails + /// detaching from its field, or landing on a neighbouring one, fails /// here rather than in someone's migrated command line. #[test] fn limit_flags_land_on_their_fields() { @@ -4976,19 +4965,6 @@ mod tests { ); } - // `off` on --build-shard-bytes is no cap, not one anchor per shard. - for (argv, want) in [ - (&["exav"][..], None), - (&["exav", "--build-shard-bytes", "off"][..], None), - (&["exav", "--build-shard-bytes", "0"][..], None), - ( - &["exav", "--build-shard-bytes", "1M"][..], - Some(1024 * 1024), - ), - ] { - assert_eq!(build_shard_budget(&Cli::parse_from(argv)), want, "{argv:?}"); - } - // With no flags, the defaults the --help text quotes. let opts = build_scan_options(&Cli::parse_from(["exav"])); assert_eq!(opts.max_scan_size, None); @@ -5009,25 +4985,28 @@ mod tests { assert_eq!(opts.limits.max_members, 10_000); } - /// `--max-extracted-bytes` is gone, and saying it, as a flag or in the - /// environment, is an error naming what replaced it rather than a setting - /// dropped without a word. + /// A removed flag, said as a flag or in the environment, is an error + /// naming why rather than a setting dropped without a word. #[test] - fn the_removed_extraction_flag_is_named() { + fn a_removed_flag_is_named() { let _env = env_guard(); let args = |a: &[&str]| a.iter().map(|s| s.to_string()).collect::>(); - for argv in [ - &["--max-extracted-bytes", "1G"][..], - &["--max-extracted-bytes=1G"], + for (argv, why) in [ + (&["--max-extracted-bytes", "1G"][..], "--max-process-bytes"), + (&["--max-extracted-bytes=1G"], "--max-process-bytes"), + (&["--build-shard-bytes", "1G"], "no longer"), + (&["--build-shard-bytes=off"], "no longer"), ] { let msg = removed_flag(&args(argv)).expect("refused"); - assert!(msg.contains("--max-process-bytes"), "{msg}"); + assert!(msg.contains(why), "{msg}"); } assert_eq!(removed_flag(&args(&["--max-object-bytes", "1G"])), None); - std::env::set_var("EXAV_MAX_EXTRACTED_BYTES", "1G"); - let msg = removed_flag(&[]); - std::env::remove_var("EXAV_MAX_EXTRACTED_BYTES"); - assert!(msg.is_some_and(|m| m.contains("EXAV_MAX_EXTRACTED_BYTES"))); + for var in ["EXAV_MAX_EXTRACTED_BYTES", "EXAV_BUILD_SHARD_BYTES"] { + std::env::set_var(var, "1G"); + let msg = removed_flag(&[]); + std::env::remove_var(var); + assert!(msg.is_some_and(|m| m.contains(var)), "{var}"); + } } /// Counts that are limits take `off`; the depth and the DLP thresholds, @@ -5091,14 +5070,14 @@ mod tests { /// Both halves matter, and the second one more. A spelling that stops /// parsing breaks a migrated command line, which the operator finds out /// about at once. A spelling that *starts* parsing is a flag the matrix - /// calls absent while exav quietly takes it — a reader then believes the + /// calls absent while exav quietly takes it: a reader then believes the /// setting is in effect, and nothing on the command line says otherwise. /// Adding a flag here is part of adding it to the page. #[test] fn clamav_spellings_are_accepted_or_refused_as_documented() { let _env = env_guard(); // exav does not take clamscan's command line. What it does share is the - // handful of short flags any scanner has, spelled the obvious way — and + // handful of short flags any scanner has, spelled the obvious way, and // the promise that anything else stops the run instead of being // swallowed, which is what the flag matrix documents. for argv in [ @@ -5220,7 +5199,7 @@ mod tests { .collect(); assert!( Cli::try_parse_from(&full).is_err(), - "{argv:?} is documented as absent, but parses — either implement \ + "{argv:?} is documented as absent, but parses: either implement \ it properly or correct the flag matrix; silently accepting it \ tells an operator the setting is in effect when it is not" ); @@ -5228,7 +5207,7 @@ mod tests { } /// An address with no `?mode=` gets the documented default, and it is the - /// one the daemon actually binds with — owner-only, so a socket left + /// one the daemon actually binds with: owner-only, so a socket left /// unqualified is never reachable by another local user. #[test] fn an_unqualified_socket_is_owner_only() { @@ -5309,7 +5288,7 @@ mod tests { /// `--connect` takes a bare address: query options tune the listener, and /// a client silently dropping them would scan under settings nobody - /// applied. Only the marked server-side options refuse — a future + /// applied. Only the marked server-side options refuse; a future /// client-side key passes through by construction. #[test] fn connect_refuses_listener_tuning() { @@ -5562,7 +5541,7 @@ mod tests { .is_ok()); // `0` is "no ceiling" here as on every other --max- size flag, so it - // cannot make the nesting fail — and cannot be mistaken for the way to + // cannot make the nesting fail, and cannot be mistaken for the way to // turn spilling off. assert!(configure_spill(&cli(&["--max-spill-bytes", "0"])).is_ok()); assert!(configure_spill(&cli(&["--max-total-spill-bytes", "0"])).is_ok()); @@ -5580,8 +5559,8 @@ mod tests { assert!(err(&["--spill-dir", "off", "--max-spill-bytes", "1G"]).contains("--spill-dir off")); } - /// The updater-only deployment — keep the signature volume current, serve - /// nothing — is inferred rather than declared. + /// The updater-only deployment (keep the signature volume current, serve + /// nothing) is inferred rather than declared. /// /// A flag saying it could contradict itself: the flag plus a listener is a /// run whose two halves disagree, and it takes a conflict check to catch. @@ -5596,7 +5575,7 @@ mod tests { assert!(updater_only(&cli(&["--auto-update"]))); // An empty address names no listener, which is how a container removes - // the one its image's ENV set — so it is an updater too, not a server + // the one its image's ENV set, so it is an updater too, not a server // that binds nothing. assert!(updater_only(&cli(&["--auto-update", "--listen", ""]))); for serving in [ @@ -5612,7 +5591,7 @@ mod tests { } /// Every setting reads the flag first, the environment next, and its own - /// default last — one rule, applied to every kind of setting there is: a + /// default last. One rule, applied to every kind of setting there is: a /// switch, a path, a number, a size, a listen address and a repeatable list. /// /// The rule is what makes a container configurable by environment without @@ -5705,7 +5684,7 @@ mod tests { /// A switch set in the environment is read the way a container writes one, /// and a value that is neither true nor false stops the run instead of being - /// guessed at — a typo would otherwise be indistinguishable from not setting + /// guessed at: a typo would otherwise be indistinguishable from not setting /// it, and `EXAV_AUTO_UPDATE=ture` would silently update nothing. #[test] fn an_environment_switch_is_true_false_or_an_error() { @@ -5742,7 +5721,7 @@ mod tests { /// When a verdict comes from a rejoined multi-volume set the daemon reports /// it on each part as `: FOUND (in )`. A classifier anchored /// to the end of the line does not see `FOUND` there, falls through to the - /// "unrecognised" arm, and prints it as an ordinary line — so the client + /// "unrecognised" arm, and prints it as an ordinary line, so the client /// shows a detection, counts nothing, and exits 0. A caller reading the exit /// code is told the tree is clean while the detection is on its screen. #[test] diff --git a/crates/exav/src/policy.rs b/crates/exav/src/policy.rs index 666f2489..9543f2a8 100644 --- a/crates/exav/src/policy.rs +++ b/crates/exav/src/policy.rs @@ -2,13 +2,13 @@ //! //! Two questions an operator answers, each with one flag: //! -//! * [`PartialStatus`] — what becomes of an object exav could not fully +//! * [`PartialStatus`]: what becomes of an object exav could not fully //! examine. Blocked, reported as a detection, or delivered. -//! * [`Detectors`] — which heuristic detectors run at all, over and above the +//! * [`Detectors`]: which heuristic detectors run at all, over and above the //! signature database. //! //! One flag each rather than a boolean per condition, for two reasons. A -//! boolean per condition cannot express "all of them" — a reader has to know +//! boolean per condition cannot express "all of them": a reader has to know //! the whole set to switch it on. And "exav could not fully examine this" is one //! condition with three possible outcomes, so spreading it over a family of //! switches lets an operator ask for two of them at once and leaves the answer @@ -85,7 +85,7 @@ pub(crate) fn apply(report: &mut ScanReport, policy: PartialAs) { /// `Heuristics.Exav.Unscannable`. /// /// `Heuristics.` is the prefix ClamAV puts on a policy finding rather than a -/// database entry, and `.Exav.` says which scanner synthesised it — so no +/// database entry, and `.Exav.` says which scanner synthesised it, so no /// signature set can collide with it and no analyst can mistake it for a hit. pub(crate) fn heuristic_name(tag: &str) -> String { let mut out = String::from("Heuristics.Exav."); @@ -102,7 +102,7 @@ pub(crate) fn heuristic_name(tag: &str) -> String { /// Which status a partial verdict is reported as. /// /// The values are the four statuses themselves, so the name of the value is the -/// name of the outcome — and, because status and exit code are 1:1, the value +/// name of the outcome, and, because status and exit code are 1:1, the value /// also names the exit code it produces. Nothing to look up. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub(crate) enum PartialStatus { @@ -113,7 +113,7 @@ pub(crate) enum PartialStatus { Partial, /// Deliver it as clean: `OK`, exit `0`, an ICAP `204`. /// - /// Never silently — a passed object is logged wherever it happens, and the + /// Never silently: a passed object is logged wherever it happens, and the /// ICAP response still carries the headers saying what was skipped. Ok, /// Call it a detection, named `Heuristics.*`: `FOUND`, exit `1`, @@ -125,7 +125,7 @@ pub(crate) enum PartialStatus { /// For a caller that would rather not learn a fourth exit code, or that /// wants any unexaminable object to stop the pipeline as loudly as a broken /// scanner does. Identical to `partial` on every surface that has no exit - /// code of its own — the clamd wire and ICAP. + /// code of its own: the clamd wire and ICAP. Error, } @@ -154,9 +154,9 @@ impl fmt::Display for PartialStatus { } } -/// The status tags a partial verdict reports under — the values +/// The status tags a partial verdict reports under (the values /// [`Verdict::status_tag`](exav_core::Verdict::status_tag) returns for the -/// `PartialAs` category — paired with the name an operator writes. +/// `PartialAs` category), paired with the name an operator writes. /// /// The two spellings differ on purpose: the wire tag is shouted /// (`PASSWORD-PROTECTED`) because it appears in a protocol reply, and the flag @@ -176,7 +176,7 @@ pub(crate) struct PartialAs { } impl PartialAs { - /// One status for every category — what a bare `--partial-as ok` means, and + /// One status for every category: what a bare `--partial-as ok` means, and /// what `--clamav-compat` installs. pub(crate) fn uniform(status: PartialStatus) -> Self { Self { @@ -189,7 +189,7 @@ impl PartialAs { /// (`password-protected=ok,limits-exceeded=found`). /// /// A category exav does not know is refused rather than ignored. A setting - /// that parsed but named nothing would read as one that never fires — an + /// that parsed but named nothing would read as one that never fires: an /// operator believing they had opened a hole they had not, or closed one /// they had not, and finding out from traffic. pub(crate) fn parse(s: &str) -> Result { @@ -236,11 +236,11 @@ impl PartialAs { } /// Whether any condition is set to `pass`, for the line a listener - /// announces itself with — a deployment delivering what it could not + /// announces itself with: a deployment delivering what it could not /// examine should be legible from its logs alone. /// - /// Its one caller is the ICAP listener, so a build without `icap` — two of - /// which the feature-flag reference offers as copy-paste examples — has + /// Its one caller is the ICAP listener, so a build without `icap` (two of + /// which the feature-flag reference offers as copy-paste examples) has /// nothing calling it. The tests below still do, but they are not compiled /// into the binary the warning is raised against. #[cfg_attr(not(feature = "icap"), allow(dead_code))] @@ -281,8 +281,8 @@ impl fmt::Display for PartialAs { /// database, off unless asked for. /// /// `pua` sits in this list even though it is applied at database load rather -/// than at scan time. From the command line it is the same kind of decision — -/// "also look for this" — and splitting it out as `--detect-pua` only meant one +/// than at scan time. From the command line it is the same kind of decision +/// ("also look for this"), and splitting it out as `--detect-pua` only meant one /// more flag to find. pub(crate) const DETECTORS: [&str; 8] = [ "macros", @@ -359,7 +359,7 @@ impl Detectors { pub(crate) fn pua(&self) -> bool { self.has("pua") } - /// TLSH fuzzy matching, the ML scorer and packed-injection-imports — one + /// TLSH fuzzy matching, the static scorer and packed-injection-imports: one /// engine switch, so one name here. It carries the `exav-` prefix because /// ClamAV has no equivalent: every other value names something both engines /// can look for, this one names what only exav does. diff --git a/crates/exav/tests/docs_match_cli.rs b/crates/exav/tests/docs_match_cli.rs index 9034d585..9b4a4924 100644 --- a/crates/exav/tests/docs_match_cli.rs +++ b/crates/exav/tests/docs_match_cli.rs @@ -3,8 +3,8 @@ //! Every drift between them found so far was silent and reader-facing: a flag //! renamed in the code and left standing in a table, a page telling an operator //! to run something the binary rejects. Nothing else in the build compares the -//! two — the docs build checks links and frontmatter, not whether the commands -//! on the page can be typed — so this does. +//! two (the docs build checks links and frontmatter, not whether the commands +//! on the page can be typed), so this does. //! //! Skipped, not failed, when `www/` is absent: the published crate ships without //! it, and a test that cannot see the docs has nothing to say about them. @@ -108,7 +108,7 @@ fn every_flag_is_documented() { assert!( undocumented.is_empty(), "these flags exist but are in no table on the CLI reference page:\n {}\n\n\ - A flag nobody can find is a flag nobody uses — add it to \ + A flag nobody can find is a flag nobody uses: add it to \ www/src/content/docs/reference/cli.md.", undocumented.join("\n ") ); @@ -164,7 +164,7 @@ fn every_documented_exav_command_uses_real_flags() { if !rest.starts_with('-') && !rest.contains(" -") { continue; } - // Some examples exist to show a command being REFUSED — the + // Some examples exist to show a command being REFUSED: the // migration guide demonstrates the error a renamed flag produces. // The refusal quoted underneath is the marker: it is what the reader // is being shown, so a command followed by one is meant to fail. @@ -229,7 +229,7 @@ fn every_documented_exav_command_uses_real_flags() { fn the_help_text_only_names_flags_exav_has() { // Flags the help names on purpose that exav does not have. Each is either // another tool's, or a spelling being explained precisely because it is not - // the one exav uses — the sentences would be wrong without them. Kept + // the one exav uses; the sentences would be wrong without them. Kept // explicit and short so adding one is a decision rather than a habit. const NOT_EXAVS: &[(&str, &str)] = &[ ("features", "cargo's, in `--features http-update`"), @@ -306,7 +306,7 @@ fn the_help_text_only_names_flags_exav_has() { /// /// `--help` is not the only place a flag name is written down. A runtime warning /// told operators to "set a smaller `--max-memory`", a flag that has never -/// existed — and it fires on any host where the per-job budget exceeds RAM, so +/// existed, and it fires on any host where the per-job budget exceeds RAM, so /// it is one of the messages people see most. Advice naming a flag that does not /// parse is worse than no advice: it reads as authoritative and costs a retry to /// disprove. @@ -317,7 +317,7 @@ fn the_help_text_only_names_flags_exav_has() { #[test] fn every_flag_named_in_a_message_exists() { // Mentions that are not exav flags: another tool's, or a spelling discussed - // precisely because exav does not use it. Short and explicit — each entry is + // precisely because exav does not use it. Short and explicit: each entry is // a decision, and the list not growing is the point. const NOT_EXAVS: &[&str] = &[ // clamscan's, named in the flag-mapping hints and the matrix references. @@ -365,6 +365,7 @@ fn every_flag_named_in_a_message_exists() { "password", // Removed from exav, named in the error that says what replaced it. "max-extracted-bytes", + "build-shard-bytes", ]; let real = flags_in_help(); @@ -428,7 +429,7 @@ fn every_flag_named_in_a_message_exists() { } // Named messages rather than a count: a threshold passes as soon as the // scan finds anything, and what matters is that it reaches the places a flag - // name is actually printed — a refusal, a fallback warning, a missing-input + // name is actually printed: a refusal, a fallback warning, a missing-input // error. If one of these stops being mentioned, this test has stopped // reading the file it was written for. for expect in ["connect", "listen", "workers", "spill-dir"] { @@ -525,7 +526,7 @@ fn docs_name_the_exav_flag_for_every_renamed_clamscan_flag() { /// The environment-variable spelling the CLI reference promises is the one the /// binary reads: uppercase, dashes to underscores, `EXAV_` in front. /// -/// A variable that does not follow it is unreachable in practice — nobody looks +/// A variable that does not follow it is unreachable in practice: nobody looks /// it up, they derive it from the flag. #[test] fn env_var_names_follow_the_documented_rule() { @@ -579,3 +580,76 @@ fn env_var_names_follow_the_documented_rule() { wrong.join("\n ") ); } + +/// Every `EXAV_*` variable the code reads is a flag's, or a diagnostic named +/// `EXAV_DEBUG_*`: the configuration reference says so, and an operator +/// reading an `EXAV_*` name elsewhere takes it for a setting. +#[test] +fn every_variable_is_a_flag_or_a_debug_one() { + let out = Command::new(env!("CARGO_BIN_EXE_exav")) + .arg("--help") + .output() + .expect("run exav --help"); + let help = String::from_utf8_lossy(&out.stdout); + let flags: BTreeSet = help + .lines() + .filter_map(|l| l.trim_start().strip_prefix("[env: ")) + .map(|rest| { + rest.chars() + .take_while(|c| *c != '=' && *c != ']') + .collect() + }) + .collect(); + assert!( + flags.len() > 40, + "only {} flag variables parsed", + flags.len() + ); + // Read only to tell whoever still sets it what replaced it. + let removed = ["EXAV_MAX_EXTRACTED_BYTES", "EXAV_BUILD_SHARD_BYTES"]; + + let mut files = Vec::new(); + let mut dirs = vec![repo_root().join("crates")]; + while let Some(d) = dirs.pop() { + for e in std::fs::read_dir(&d).unwrap().flatten() { + let p = e.path(); + let name = e.file_name(); + if p.is_dir() && name != "target" && name != "node_modules" { + dirs.push(p); + } else if p.extension().is_some_and(|x| x == "rs") { + files.push(p); + } + } + } + let mut stray = BTreeSet::new(); + for f in &files { + let text = std::fs::read_to_string(f).unwrap(); + for call in [ + "var(\"", + "var_os(\"", + "env(\"", + "set_var(\"", + "remove_var(\"", + ] { + for (at, _) in text.match_indices(call) { + let rest = &text[at + call.len()..]; + if !rest.starts_with("EXAV_") { + continue; + } + let var: String = rest.chars().take_while(|c| *c != '"').collect(); + if !var.starts_with("EXAV_DEBUG_") + && !flags.contains(&var) + && !removed.contains(&var.as_str()) + { + stray.insert(format!("{var} ({})", f.display())); + } + } + } + } + assert!(files.len() > 100, "only {} sources found", files.len()); + assert!( + stray.is_empty(), + "read, but neither a flag's variable nor `EXAV_DEBUG_*`:\n {}", + stray.into_iter().collect::>().join("\n ") + ); +} diff --git a/crates/exav/tests/json_output.rs b/crates/exav/tests/json_output.rs index 3186609c..9b50c8c7 100644 --- a/crates/exav/tests/json_output.rs +++ b/crates/exav/tests/json_output.rs @@ -52,7 +52,7 @@ fn json_output_is_valid_jsonl_with_expected_fields() { .collect(); // First: the EICAR detection. `status` is the same word the human line ends - // with, and `category` is absent — it sub-classifies a PARTIAL and there is + // with, and `category` is absent: it sub-classifies a PARTIAL and there is // nothing to sub-classify here. assert_eq!(objs[0]["status"], "FOUND"); assert!(objs[0]["category"].is_null()); @@ -104,14 +104,10 @@ fn json_infected_only_suppresses_clean() { } /// `--all-matches` must not answer `OK` for a file `--max-input-bytes` says was -/// never fully examined. -/// -/// The all-match engine entry takes bytes rather than a file, so it cannot see -/// the ceiling; the CLI has to route an over-limit file to the single-match path, -/// which enforces it. Without that, adding a flag about *how many signatures to -/// report* silently turns a `PARTIAL` into a clean pass — the exact failure the -/// never-a-silent-clean invariant exists to prevent, and one no signature change -/// would ever reveal. +/// never fully examined. Otherwise a flag about *how many signatures to report* +/// silently turns a `PARTIAL` into a clean pass: the exact failure the +/// never-a-silent-clean invariant exists to prevent, and one no signature +/// change would ever reveal. #[test] fn all_matches_still_reports_a_file_past_the_size_ceiling() { let db = TempDir::new().unwrap(); @@ -134,6 +130,14 @@ fn all_matches_still_reports_a_file_past_the_size_ceiling() { let stdout = String::from_utf8_lossy(&out.stdout); let first = stdout.lines().next().unwrap_or_default().to_string(); let v: serde_json::Value = serde_json::from_str(&first).expect(&first); + // The summary counts the bytes scanned: the first 1 MiB, not the 3. + let last = stdout.lines().last().unwrap_or_default(); + let summary: serde_json::Value = serde_json::from_str(last).expect(last); + assert_eq!( + summary["summary"]["data_scanned_bytes"], + 1 << 20, + "{extra:?}" + ); seen.push(( out.status.code(), v["status"].clone(), @@ -149,12 +153,50 @@ fn all_matches_still_reports_a_file_past_the_size_ceiling() { assert_eq!(seen[0].0, Some(3)); } +/// The same over stdin, where only the first bytes are held: `--all-matches` +/// reports the stream's size, as a single-match scan does, not what it held. +#[test] +fn all_matches_on_stdin_past_the_size_ceiling_reports_its_size() { + use std::io::Write; + let db = TempDir::new().unwrap(); + let mut seen = Vec::new(); + for extra in [&[][..], &["--all-matches"][..]] { + let mut child = exav() + .arg("-d") + .arg(db.path()) + .args(["--json", "--max-input-bytes", "4096"]) + .args(extra) + .arg("-") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("run exav"); + child + .stdin + .take() + .unwrap() + .write_all(&[b'a'; 10_000]) + .unwrap(); + let out = child.wait_with_output().unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout); + let first = stdout.lines().next().unwrap_or_default().to_string(); + let v: serde_json::Value = serde_json::from_str(&first).expect(&first); + seen.push((out.status.code(), v["status"].clone(), v["reason"].clone())); + } + assert_eq!( + seen[0], seen[1], + "--all-matches must give the same answer as a single-match scan" + ); + let reason = seen[0].2.as_str().unwrap_or_default(); + assert!(reason.contains("file size 10000 exceeds"), "{reason}"); +} + /// `--profile` must not change the answer, only add timings beside it. /// /// It prints a CSV row and returns before the reporting path where every other /// mode applies `--partial-as`, so the policy has to be applied there too. /// Without that, asking for over-limit files to pass still exits 3 under -/// `--profile` — a flag about *where the time went* silently overriding a flag +/// `--profile`: a flag about *where the time went* silently overriding a flag /// about *what the verdict is*, in the direction a CI gate would notice only by /// failing. #[test] @@ -211,7 +253,7 @@ fn profile_reports_the_same_verdict_as_a_plain_scan() { /// A detection does not mean the search finished. In `--all-matches` the status /// word is `FOUND`, which cannot also say `PARTIAL`, so an incomplete search has -/// to be reported alongside the detections instead of replacing them — on its +/// to be reported alongside the detections instead of replacing them: on its /// own line in the normal grammar, and as `partial`/`partial_reasons` in JSON. /// /// Before this, the all-match path discarded the outcome whenever it had @@ -219,30 +261,23 @@ fn profile_reports_the_same_verdict_as_a_plain_scan() { #[test] fn all_matches_reports_an_incomplete_search_alongside_detections() { let db = TempDir::new().unwrap(); - // `Test.Hit` is the detection. `Test.Wild` exists to burn the per-group - // step budget: its anchor repeats tens of thousands of times and the tail it - // searches for is never present, so every hit scans to the end of the file. - std::fs::write( - db.path().join("t.ndb"), - "Test.Hit:0:*:4558415654455354\nTest.Wild:0:*:51574552*5a584356\n", - ) - .unwrap(); + // `Test.Hit` is the detection, in the first bytes of the sample. + std::fs::write(db.path().join("t.ndb"), "Test.Hit:0:*:4558415654455354\n").unwrap(); let files = TempDir::new().unwrap(); let sample = files.path().join("sample.bin"); let mut data = b"EXAVTEST".to_vec(); data.extend(std::iter::repeat_n(*b"QWER", 50_000).flatten()); std::fs::write(&sample, &data).unwrap(); - let run = |cap: Option<&str>, json: bool| { + let run = |limit: Option<&str>, json: bool| { let mut c = exav(); c.arg("-d").arg(db.path()).arg("--all-matches"); if json { c.arg("--json"); } - if let Some(cap) = cap { - // Forces the repeated-anchor cap to refuse the group after its first - // charged verify, which is what flags the search truncated. - c.env("EXAV_MAX_GROUP_STEPS", cap); + if let Some(limit) = limit { + // Only the first bytes are scanned, so the search stops short. + c.arg("--max-input-bytes").arg(limit); } c.arg(&sample).output().expect("run exav") }; @@ -259,7 +294,7 @@ fn all_matches_reports_an_incomplete_search_alongside_detections() { // Search truncated: the detection still stands, and the incompleteness is // reported next to it in the `[reason ][CATEGORY ]STATUS` grammar. - let out = run(Some("1"), false); + let out = run(Some("4K"), false); let text = String::from_utf8_lossy(&out.stdout); assert!(text.contains("Test.Hit FOUND"), "{text}"); let partial = text @@ -277,7 +312,7 @@ fn all_matches_reports_an_incomplete_search_alongside_detections() { ); // Same thing in JSON, as fields rather than a second record. - let out = run(Some("1"), true); + let out = run(Some("4K"), true); let text = String::from_utf8_lossy(&out.stdout); let row: serde_json::Value = serde_json::from_str(text.lines().next().unwrap()).unwrap(); assert_eq!(row["status"], "FOUND"); diff --git a/crates/exav/tests/value_vocabulary.rs b/crates/exav/tests/value_vocabulary.rs index 3979e9e6..fa282a85 100644 --- a/crates/exav/tests/value_vocabulary.rs +++ b/crates/exav/tests/value_vocabulary.rs @@ -56,7 +56,6 @@ fn every_byte_flag_takes_the_same_sizes() { "--max-process-bytes", "--max-spill-bytes", "--max-total-spill-bytes", - "--build-shard-bytes", "--icap-preview-bytes", ]; const SIZES: &[&str] = &["--spill-threshold-bytes", "--icap-max-header-bytes"]; diff --git a/crates/exav/tests/walk_errors.rs b/crates/exav/tests/walk_errors.rs index 2714248a..af98bce7 100644 --- a/crates/exav/tests/walk_errors.rs +++ b/crates/exav/tests/walk_errors.rs @@ -2,8 +2,8 @@ //! //! An unreadable directory is the quietest way to lose files: the walk yields an //! error instead of entries, and an error that is dropped leaves a run that -//! reports on what it reached and exits 0. To an operator — and to a pipeline -//! reading the exit code — that is indistinguishable from a tree with nothing +//! reports on what it reached and exits 0. To an operator, and to a pipeline +//! reading the exit code, that is indistinguishable from a tree with nothing //! wrong in it, even when the part nobody could open holds the malware. //! //! Every surface that walks a tree goes through one walker, so this pins the @@ -28,7 +28,7 @@ fn eicar() -> &'static [u8] { } /// Returns `None` when the platform or the test environment cannot make a -/// directory unreadable — running as root defeats mode 0, and non-Unix has no +/// directory unreadable: running as root defeats mode 0, and non-Unix has no /// equivalent. Skipping beats asserting something the environment cannot show. #[cfg(unix)] fn unreadable_tree() -> Option { @@ -115,6 +115,36 @@ fn an_error_is_printed_under_quiet_and_logged() { ); } +/// A URL this run will not scan (no `--allow-http-scan`, or a build without +/// `http-scan`) is an error like any other: logged, and a JSON record. +#[test] +fn a_refused_url_is_logged_and_reported_in_json() { + let dir = TempDir::new().expect("temp dir"); + let log = dir.path().join("scan.log"); + let url = "https://example.invalid/sample.bin"; + let out = exav() + .args(["--json", "--log"]) + .arg(&log) + .arg(url) + .output() + .expect("run exav"); + let stdout = String::from_utf8_lossy(&out.stdout); + assert_eq!(out.status.code(), Some(2), "{stdout}"); + assert!( + stdout.contains(url) && stdout.contains(r#""status":"ERROR""#), + "{stdout}" + ); + let out = exav() + .arg("--log") + .arg(&log) + .arg(url) + .output() + .expect("run exav"); + assert_eq!(out.status.code(), Some(2)); + let logged = std::fs::read_to_string(&log).expect("read the log"); + assert!(logged.contains(url) && logged.contains("ERROR"), "{logged}"); +} + /// The summary counts a file not fully examined as partial, not as an error: /// the two exit differently, and the summary has to agree with the exit code. #[test] diff --git a/deny.toml b/deny.toml index cff308bb..cac6d6b2 100644 --- a/deny.toml +++ b/deny.toml @@ -9,11 +9,6 @@ yanked = "deny" # release available; none is reachable in exav's usage. Revisit on every # dependency bump (a fixed release should drop the ignore, not keep it). ignore = [ - # bincode is flagged unmaintained (informational, not a vulnerability). Used - # only for fixed-layout, big-endian header parsing of DMG/encrcdsa headers - # (crates/exav-unpack/src/formats/dmg.rs). No decode of untrusted - # length-prefixed collections; no known defect. - "RUSTSEC-2025-0141", # NOTE: RUSTSEC-2023-0071 (`rsa` "Marvin") is intentionally NOT listed. `rsa` # is not in the tree: the native YARA engine does `pe` via `goblin` # and needs no Authenticode RSA, so the advisory has nothing to apply to. diff --git a/docker-compose.yml b/docker-compose.yml index 782d914e..b9b9a12c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,17 @@ # Keeping exav's signatures fresh in Docker. exav bundles no database and no # mirror URL, so you choose the source. Three options: # -# 1. Prebaked/managed volume — mount a `/var/lib/exav` you populate yourself +# 1. Prebaked/managed volume: mount a `/var/lib/exav` you populate yourself # with `cvd`/`freshclam`. Just the `exav` service below, no updater. # -# 2. Built-in auto-update — set EXAV_SIG_SOURCES on the `exav` service to one +# 2. Built-in auto-update: set EXAV_SIG_SOURCES on the `exav` service to one # or more sources you trust (comma-separated): a CVD mirror base ends in `/` # and expands to main/daily/bytecode.cvd, any other http(s) URL is fetched # verbatim, and anything else is a file of either. It fetches on start and # every EXAV_UPDATE_INTERVAL_SECS after that. No sidecar needed; delete the # `updater` service. # -# 3. Sidecar (shown as the default) — a second container writes the shared +# 3. Sidecar (shown as the default): a second container writes the shared # volume and the scanner hot-reloads it. Here the sidecar is the same image # with `--auto-update` and nothing to listen on, which fetches and nothing # else. You could instead run the official `clamav/clamav` freshclam @@ -28,7 +28,9 @@ services: exav: image: ghcr.io/sylvinus/exav:latest ports: - - "3310:3310" # clamd-compatible; `clamdscan --stream` over TCP works + # clamd-compatible; `clamdscan --stream` over TCP works. Published on + # localhost only: the clamd protocol has no authentication. + - "127.0.0.1:3310:3310" volumes: - exav-db:/var/lib/exav restart: unless-stopped @@ -42,7 +44,7 @@ services: # The image's ENV names a clamd listener; empty removes it, which a # container runtime can do and unsetting cannot. EXAV_LISTEN: "" - EXAV_SIG_SOURCES: "${EXAV_SIG_SOURCES:?set EXAV_SIG_SOURCES to one or more signature sources you trust — a CVD mirror base ends in /}" + EXAV_SIG_SOURCES: "${EXAV_SIG_SOURCES:?set EXAV_SIG_SOURCES to one or more signature sources you trust; a CVD mirror base ends in /}" volumes: - exav-db:/var/lib/exav # The image's health check pings a listener, and this container has none. diff --git a/docs/BYTECODE.md b/docs/BYTECODE.md index 01d60e56..42eafc78 100644 --- a/docs/BYTECODE.md +++ b/docs/BYTECODE.md @@ -1,4 +1,4 @@ -# Bytecode signatures (`.cbc`) — design, scope, and security +# Bytecode signatures (`.cbc`): design, scope, and security ClamAV's most expressive signature type is **bytecode**: a small program (written in C, compiled to a custom VM bytecode, shipped as a `.cbc` file in @@ -7,7 +7,7 @@ malicious. It expresses detection logic that pattern/logical signatures can't. exav implements a reader and a **memory-safe, sandboxed interpreter** for this format. This document records what the format is, what's actually in the live -database, the scope we target, and — the main point — **why exav's approach is +database, the scope we target, and (the main point) **why exav's approach is structurally safer than ClamAV's**, which has a documented history of remote code execution in exactly this subsystem. @@ -19,16 +19,16 @@ code execution in exactly this subsystem. A `.cbc` file is line-oriented: -- **`ClamBC` header line** — format level, timestamp, compiler, type/function +- **`ClamBC` header line**: format level, timestamp, compiler, type/function counts, functionality-level range, and a validation magic (`0x53e5493e9f3d1c30`). Integers use a nibble encoding: a byte `0x6N` carries nibble `N` (so `` ` ``=0 … `o`=15); a number is a count byte followed by that many little-endian nibbles. -- **Trigger line** — a logical signature in `.ldb` form +- **Trigger line**: a logical signature in `.ldb` form (`Name;TDB;expr;subsigs…`). The program runs **only** when this signature matches a file. exav already has a full `.ldb` engine, so the trigger side is free. -- **Records** — `T` types, `E` API declarations, `G` globals, `A` function +- **Records**: `T` types, `E` API declarations, `G` globals, `A` function headers, `B` basic-block instruction streams, `S` strings. The instruction set is an LLVM-IR-like SSA form: arithmetic, bitwise, casts, @@ -40,7 +40,7 @@ accessors, hashing, `disasm_x86`, and `setvirusname` to report a hit. ## 2. What's actually in the live database -Analysis of `bytecode.cvd` (v339, Sep 2025) — all **85** programs: +Analysis of `bytecode.cvd` (v339, Sep 2025), all **85** programs: | Property | Finding | |---|---| @@ -89,7 +89,7 @@ historically via an LLVM JIT: - **CVE-2020-37167**: weak validation of bytecode function names in ClamBC before 0.103.0, a code-injection flaw (CWE-94), **CVSS 3.1 8.4**, local vector. -- **ClamAV < 0.102 `bytecode_vm` code execution** (exploit-db 47687) — the +- **ClamAV < 0.102 `bytecode_vm` code execution** (exploit-db 47687), in the bytecode VM/JIT path. - The optional **LLVM JIT** generated and ran native code from bytecode: a large attack surface (and dependency) that Cisco has been moving away from. @@ -98,47 +98,68 @@ exav's design removes these failure modes by construction: | Risk in ClamAV's C VM | exav | |---|---| -| Memory corruption or code injection in the VM (the class of the two above) | **Pure safe Rust, no `unsafe`** — every memory access is a bounds-checked slice; an out-of-range index panics into isolation, it cannot corrupt memory | -| Native code generation from bytecode (JIT spray, W^X issues) | **No JIT, ever** — interpret only | +| Memory corruption or code injection in the VM (the class of the two above) | **Pure safe Rust, no `unsafe`**: every memory access is a bounds-checked slice; an out-of-range index panics into isolation, it cannot corrupt memory | +| Native code generation from bytecode (JIT spray, W^X issues) | **No JIT, ever**: interpret only | | Untrusted program escaping the sandbox (syscalls, host memory) | Program sees only bounded `Vec`s and a fixed read-only file API; no syscalls, no host pointers | | Runaway program (CPU/memory exhaustion) | **Instruction budget**, scratch-memory cap, and call-depth limit | -| A parser/VM bug taking down the scan | **`catch_unwind` per program** (and per file) — one bad program is skipped, the scan continues | +| A parser/VM bug taking down the scan | **`catch_unwind` per program** (and per file): one bad program is skipped, the scan continues | | Malformed/hostile `.cbc` | Fallible parser (no panics); 100% of the live DB parses cleanly; a program that isn't fully understood is **never executed** | The net: the worst a hostile or buggy `.cbc` can do to exav is *be skipped or time out*. In ClamAV the worst case has been *remote code execution*. For anyone who disables ClamAV bytecode for safety, exav offers the capability **without -that trade-off** — a real reason to switch. +that trade-off**, a real reason to switch. ## 5. Dangers identified (and how they're handled) -- **Decompression/alloc bombs via `malloc`/codecs** → scratch-memory cap; the - codec APIs are stubs; allocation bounded. +- **Decompression/alloc bombs via `malloc`/codecs** → the codec APIs are + stubs; one `malloc` of 128 MiB less 8 bytes or more returns NULL, as in + ClamAV 1.4.3. `write` fails with -1, as ClamAV's does at its file and scan + size limits, once one extracted file would pass `--max-object-bytes` or + everything written would pass what is left of `--max-matcher-bytes`; the + program carries on, and the scan is `LIMITS-EXCEEDED` unless something else + is found. Past those, one extracted file holds at most 256 MiB, and a + program that writes more is stopped. The number of allocations is bounded + only by the instruction budget. - **Infinite loops** → instruction budget (programs are not guaranteed to halt). - **Pointer math (`GEP`) out of bounds** → modeled as `(region, offset)` with checked access; never a raw pointer. - **Reading beyond the file** → the file is a read-only slice; all API reads are clamped. - **Trigger-only false positives** → a program is the *confirmation* step; if it - can't run, exav reports nothing (never fires on the trigger alone). + can't run, exav never fires on the trigger alone. When exav stopped it (steps, + call depth, an unmodeled op or API, output past 256 MiB, PE header data exav + lacks), the scan is `LIMITS-EXCEEDED` unless something else is found; a + program's own fault (out of bounds, division by zero, an abort) is + discarded, as in ClamAV. +- **`__clambc_pedata` without a PE** → as in ClamAV 1.4.3, its 648 bytes read + as zeros on a file that is not a PE, and a read past them is out of bounds. + A file exav types as a PE but has no PE header data for (too large to load, + or rejected by exav's parser) may be one ClamAV has data for, so reading it + stops the program as above. +- **Output of a failed run** → what a program wrote before its run failed or + was stopped is scanned (except after a VM panic), as ClamAV scans the output + of a failed run; only its detection is discarded. A + `disasm_x86` call on bytes the decoder does not know ends the run, so no + output depends on a guess about them. - **`deserialize_unchecked`-style trust** → N/A *for the bytecode subsystem*: no `.cbc` content is loaded unchecked, and the DB itself is signature-verified - by `freshclam`/`cvdupdate`. (The one `deserialize_unchecked` in the project is - the prebuilt-database loader in `engine.rs`, which is a SHA-256-verified trusted - artifact — a separate trust model, see SECURITY.md.) + by `freshclam`/`cvdupdate`. (The prebuilt `.exavdb` is a separate trust + model: a trusted artifact whose CRC-32 catches damage, not tampering; see + SECURITY.md.) ## 6. Status **Done and verified against the real corpus:** -- Exact nibble number/data/**operand** decoder (`bytecode::decode`) — fuzzed. -- Header + trigger + API declarations (`bytecode::parse`) — **parses 100% of the +- Exact nibble number/data/**operand** decoder (`bytecode::decode`), fuzzed. +- Header + trigger + API declarations (`bytecode::parse`): **parses 100% of the 85 programs**. - **Function headers** (`A` records: args, return type, locals+flags, inst/block - counts) — **all 134 headers across the 85 files decode cleanly** (27,290 + counts): **all 134 headers across the 85 files decode cleanly** (27,290 instructions framed). - Bounded, memory-safe interpreter (`bytecode::exec`, public surface in `bytecode::runtime`) with the core opcode set, checked memory, - instruction/memory/depth budgets, host API surface — proven end-to-end on + instruction/memory/depth budgets, host API surface, proven end-to-end on constructed programs. - Loader integration ("Bytecode programs loaded: N"); fuzz target. Execution is **live in the normal scan path**, trigger-gated per program (`lib.rs` calls @@ -148,13 +169,13 @@ that trade-off** — a real reason to switch. - Confirmed: opcode enum + `operand_counts` table; CALL/GEPN read their arg count inline; inline-constant operands marked by a `0x4N`/`0x50` lead byte. -**Static decode — done (Phases A + B).** -- **A — instructions** (`bytecode::instr`): the exact per-instruction framing +**Static decode: done (Phases A + B).** +- **A, instructions** (`bytecode::instr`): the exact per-instruction framing (terminator marker, `'E'` end-of-function marker, per-opcode operand shapes incl. `CALL`/`GEP`/`BRANCH`/`RET`/`ICMP`, `0x4N`/`0x50` inline constants). - **Decodes every instruction of all 85 programs** — 134 functions, 27,290 + **Decodes every instruction of all 85 programs**: 134 functions, 27,290 instructions; each block consumes exactly to its terminator. -- **B — types + globals** (`bytecode::types`): the `T` type table and `G` +- **B, types + globals** (`bytecode::types`): the `T` type table and `G` constant globals (with component counting). **Decodes all 85** (1,257 globals) with no errors. @@ -178,14 +199,14 @@ reported inside bytecode execution. ## Real-world importance of the 85 programs -Numerically tiny — 85 of ~3.7M signatures (0.002%) — but uneven in value: +Numerically tiny (85 of ~3.7M signatures, 0.002%), but uneven in value: - **Unpackers (~6, `BC.Win.Packer`/`Packed`) matter most.** Some of ClamAV's unpacking is *implemented as bytecode*; an unpacker deobfuscates a packed PE so the other ~3.7M signatures can match the payload. Missing one - silently weakens detection across *many* packed samples — far beyond one sig. + silently weakens detection across *many* packed samples, far beyond one sig. - **Polymorphic / entry-point-obfuscated families** (`BC.Win.Virus` Xpaj, - `Ransom`) — per-sample algorithmic checks static sigs can't express; each + `Ransom`): per-sample algorithmic checks static sigs can't express; each catches a whole family. - **The long tail is legacy.** 36/85 are literally `BC.Legacy.Exploit` for 2010–2012 CVEs (one notable still-in-the-wild exception: CVE-2012-0158, the @@ -194,7 +215,7 @@ Numerically tiny — 85 of ~3.7M signatures (0.002%) — but uneven in value: Implication for the roadmap: prioritize the **unpacker and polymorphic-family** bytecodes (broad, current impact); the 36 single-CVE legacy checks are low -priority. And note the bigger picture — the **detection delta** from 85 mostly- +priority. And note the bigger picture: the **detection delta** from 85 mostly- legacy programs is modest, so exav's headline win here is the **memory-safe, non-JIT sandbox** (removing the RCE class), more than the raw extra coverage. diff --git a/docs/DATABASE.md b/docs/DATABASE.md index 0be1135f..5f445a19 100644 --- a/docs/DATABASE.md +++ b/docs/DATABASE.md @@ -2,16 +2,14 @@ Two things are easy to conflate, so this doc keeps them distinct: -- **signatures** — the raw ClamAV-format files (`.cvd`/`.cld` containers and the +- **signatures**: the raw ClamAV-format files (`.cvd`/`.cld` containers and the loose `.ndb`/`.ldb`/`.hsb`/… inside them) that you fetch with `cvdupdate`/`freshclam`; -- **the database** — the single compiled `.exavdb` file exav builds from those +- **the database**: the single compiled `.exavdb` file exav builds from those signatures (`--build-db`), which loads far faster and lighter. -This documents what the signatures actually contain, how exav turns each part -into a runtime structure, where the memory goes, and why the compiled database is -the answer to the build-time memory spike — based on measurements against a real -`daily.cvd` (ClamAV 1.4 era). +This documents what the signatures contain, how exav turns each part into a +runtime structure, and where the time and memory go when building and loading. ## What's in the signatures @@ -31,91 +29,69 @@ contents of one `daily.cvd` (≈356k signatures total): exav understands: `.cdb` (container metadata), `.imp` (PE import-hash), `.cbc` (bytecode), `.fp`/`.sfp` (allowlist), and YARA `.yar`/`.yara`. -The headline fact: **`.ldb` dominates the count and the cost.** Everything else -is comparatively cheap. - ## How each type is built and used -Different signature types compile to different runtime structures — there is -**not** one monolithic engine: +Different signature types compile to different runtime structures; there is no +one monolithic matcher: | Source | Runtime structure | Matching | |---|---|---| -| `.ndb` bodies **and** `.ldb` literal subsignatures | a shared **Aho-Corasick automaton** (one case-sensitive + one tiny case-insensitive), keyed on a literal *anchor* per body | an automaton hit fans out to every body sharing that anchor; each candidate body is then verified (wildcards/gaps/nibbles/alternation, offset, nocase) | -| `.ldb` `Trigger/PCRE/` subsignatures | `regex` objects, **compiled lazily** on first use, gated by the trigger expression | linear-time `regex` (DoS-safe); unsupported constructs (backref/lookaround) are skipped | +| `.ndb` bodies **and** `.ldb` subsignatures | an **anchor index**: one literal *anchor* per body, indexed by the bytes of it the signature set shares least (the whole anchor at 1 to 3 bytes, 4 of them at 4 or 5, two overlapping 6-byte windows past that, looked up at every other position), behind filters small enough to stay in cache | the index is looked up at each position of the object; a hit fans out to every body sharing that anchor, and each candidate is verified (wildcards/gaps/nibbles/alternation, offset, nocase) | +| bodies whose offset (`n`, `EOF-n`, `EP+n`, `Sn+n`, `SL+n`, with an optional `,m` range of at most 4096) pins them to a window of starts | per-offset tables, kept out of the index | checked at the places each can start | +| `.ldb` `Trigger/PCRE/` subsignatures | `regex` objects, **compiled lazily** on first use, gated by the trigger expression | linear-time `regex` (DoS-safe); lookaround/backreferences run on a backtracking engine under a step bound | | `.ldb` byte-compare subsignatures | small structs (`offset#options#comparisons`) | evaluated after the referenced subsig matches | -| `.hsb`/`.hdb` | a size-keyed **hash table** | whole-file digest lookup | -| `.mdb`/`.msb` | a section-hash table | per-PE-section digest lookup | +| `.hsb`/`.hdb` | size-keyed sorted **hash tables**, stored as flat blobs | whole-file digest lookup, computing only the digests some signature of that size needs | +| `.mdb`/`.msb` | a section-hash table | per-PE-section digest lookup, only for section sizes a signature names | | `.cdb` | container-metadata matchers | matched on archive members (name/size/encryption/position) | | `.imp` | size-constrained import-hash map | PE imphash lookup | -| `.cbc` | a sandboxed **bytecode interpreter** (no JIT) | trigger-gated programs run on extracted buffers | -| `.yar`/`.yara` | exav's **native YARA engine** — a tree-walking evaluator (no JIT, no runtime codegen) | full-ish YARA | - -So the engine is really: *one Aho-Corasick automaton (fed by ndb + ldb literal -subsigs) + several cheap hash tables + lazy regex + an interpreter.* - -## Where the memory goes - -Measured peak RSS by signature type (each loaded alone, scanning a 1-byte file): +| `.cbc` | a sandboxed **bytecode interpreter** (no JIT) | trigger-gated programs; triggers are matched in the same sweep as every other signature | +| `.yar`/`.yara` | exav's **native YARA engine**, with an atom prefilter over the same anchor index (no JIT, no runtime codegen) | full-ish YARA | -| Loaded | Peak RSS | -|---|---| -| 20,000 `.hsb` | ~2 MB | -| 11,575 `.mdb` | ~2 MB | -| 312 `.ndb` | ~2 MB | -| 5,000 `.ldb` | ~80 MB | -| 20,000 `.ldb` | ~284 MB | +So the engine is: *one anchor index (fed by ndb + ldb subsigs) + pinned-offset +tables + hash tables + lazy regex + an interpreter + YARA.* -`.ldb` costs ~14 KB/signature; hash tables are ~free. **All the memory is the -automaton**, and almost all of *that* is one specific thing — see below. +## Build versus load -### Build peak vs steady state vs database load +Measured on a 2026 `daily.cvd` alone, and on `main.cvd` + `daily.cvd` + +`bytecode.cvd`, with the 0.0.2 engine (peak RSS of the whole process): -For the full `daily.cvd` (≈356k sigs, 961,830 anchor patterns, 1.36M bodies): - -| Phase | Peak RSS | -|---|---| -| After parsing all signatures, **before** building the automaton | ~740 MB | -| **Building** the automaton (from raw signatures) | **~3.6 GB** | -| Final live structures (Body 148 MB + automaton 237 MB + groups 32 MB + …) | ~470 MB | -| **Loading the same signatures from a prebuilt database** | **~1.0 GB** | +| | daily | main + daily + bytecode | +|---|---|---| +| Parse the signatures and build (`--build-db`, or `-d` on the raw files) | 14 s, 1.3 GB | 23 s, 1.6 GB | +| Load the prebuilt `.exavdb` | 0.7 s, 0.55 GB | 1.3 s, 0.92 GB | +| `.exavdb` size | 140 MB | 352 MB | -The ~2.9 GB spike is the **daachorse double-array Aho-Corasick construction -transient** — a one-shot allocation burst while the automaton is built. It is -*not* steady-state storage (live data is ~470 MB) and it is *not* a leak: the -build frees its intermediates correctly (the dedup map is dropped before -construction, anchor buffers after, the two automatons build sequentially). The -transient simply can't be reduced by freeing things between steps — it's a -single construction event. +The build holds the parsed signature text and its working tables next to the +structures it produces; a load allocates about the final size and reads the +index, hash tables and anchor groups back with a copy. -### The prebuilt database is the answer +## The prebuilt database is the answer -`exav --build-db FILE -d ` serializes the built engine (the daachorse -automaton via its own format, the rest via MessagePack through rmp-serde) into a single `.exavdb`. -**Loading the database skips the construction transient entirely** — -deserialization allocates ~the final size, not the build peak. Measured: 62 MB -(database) vs 284 MB (build) for 20k ldb; **1.0 GB (database) vs 3.6 GB (build)** -for full daily.cvd. +`exav --build-db FILE -d ` serializes the built engine (flat arrays for +the index, the hash tables and the anchor groups; MessagePack through rmp-serde +for the rest) into a single `.exavdb`. Loading it skips parsing and the index +build. What can be computed once from the signatures is computed at build time +and stored, even when that makes the file larger: load time and scan speed come +first. Recommended deployment: **build the database once on a capable machine (or in -CI), distribute the `.exavdb`, and load it cheaply everywhere.** Constrained hosts -never pay the build peak. On a small *build* host, `--build-shard-bytes ` -bounds the per-shard construction transient (e.g. `--build-shard-bytes 1G` keeps -the full main+daily build near 3.3 GB peak) at a small scan-speed cost. +CI), distribute the `.exavdb`, and load it cheaply everywhere.** Constrained +hosts never pay the build. > Note: the environment matters too. RAM-backed `/tmp` (tmpfs) and a resident -> `clamd` can each consume ~1–2 GB; account for those when sizing a build host. - -**Updating a running database-based daemon** is just recompile → atomic-swap: the -daemon mtime-polls the database file (like clamd's `SelfCheck`), so swapping it in -hot-reloads within a poll tick — no explicit `RELOAD` needed. The framing -`MAGIC | VERSION | payload | SHA-256` lets the daemon reject a torn/wrong-version -database on reload and keep serving the current one. See +> `clamd` can each consume 1 to 2 GB; account for those when sizing a build host. + +**Updating a running database-based daemon** is recompile, then atomic swap: +the daemon mtime-polls the database file (like clamd's `SelfCheck`), so +swapping it in hot-reloads within a poll tick, with no explicit `RELOAD`. The +framing `MAGIC | VERSION | payload | CRC-32` lets the daemon reject a +torn/wrong-version database on reload and keep serving the current one. The +database is a trusted artifact: the CRC-32 catches damage, not tampering. See [Updating a running deployment](https://exav.org/guides/prebuilt-database/#updating-a-running-deployment). ## Future ideas - **`malloc_trim` after load** to return glibc-retained freed memory to the OS - (shrinks steady RSS; does not affect the transient build peak). -- **More compact body/token representation** (a flat byte-code instead of - `Vec` enums) to shave the ~148 MB of `Body` structures. + (shrinks steady RSS; does not affect the build peak). +- **A pre-decoded bytecode form**: operands resolved to frame offsets when a + program is loaded, so the interpreter does not look them up at every step. diff --git a/docs/DIFF_TESTING.md b/docs/DIFF_TESTING.md index b81e01fc..6d16b72f 100644 --- a/docs/DIFF_TESTING.md +++ b/docs/DIFF_TESTING.md @@ -2,7 +2,7 @@ How we validate exav's correctness as a drop-in: scan the **same files** with the **same signature database** using both engines and compare verdicts. -Disagreements are exav bugs — a false negative (clam detects, exav misses) or +Disagreements are exav bugs: a false negative (clam detects, exav misses) or a false positive (exav detects, clam clean). **This is a compliance harness, not a benchmark.** It records timings, and they @@ -18,7 +18,7 @@ The harness lives in `scripts/` (tracked, reproducible, no secrets): | Script | Role | |---|---| -| `scripts/difftest.sh` | the runner — three phases, one engine resident at a time | +| `scripts/difftest.sh` | the runner: three phases, one engine resident at a time | | `scripts/difftest-scan.py` | drives ONE engine over a manifest; used for both | | `scripts/difftest-compare.py` | joins the two result tables and buckets the differences | | `scripts/clamd-docker.sh` | standalone start/stop/status for the Dockerised clamd | @@ -39,7 +39,7 @@ The harness lives in `scripts/` (tracked, reproducible, no secrets): * **One engine is resident at a time.** Running both together on a small host meant ~2 GB of clamd plus ~1 GB of exav competing for RAM and disk: workers died to OOM, a stuck job head-of-line blocked the pool, and ~3 slow files - turned into ~36 rows recorded as errors — differences that were artefacts of + turned into ~36 rows recorded as errors, differences that were artefacts of the harness, not of the engines. * **Both engines are driven by the same client**, so a difference in results cannot come from a difference in how they were asked. @@ -49,16 +49,16 @@ The harness lives in `scripts/` (tracked, reproducible, no secrets): ### Two protocol facts worth knowing -* **clamd refuses every multi-reply command inside `IDSESSION`** — +* **clamd refuses every multi-reply command inside `IDSESSION`**: `ALLMATCHSCAN`, `CONTSCAN` and `MULTISCAN` all answer `Command invalid inside IDSESSION. ERROR` and hang up. Only single-reply commands (`SCAN`, `PING`, `VERSION`, `STATS`) are allowed, because the session tags one reply per command id. So all-match means one connection per file. (exav's daemon is more - permissive here and does allow it in a session — a divergence, not a bug, but + permissive here and does allow it in a session: a divergence, not a bug, but a clamd-compatible client cannot rely on it.) * **exav preforks one worker per core.** Driving more concurrent scans than there are workers puts the excess in the accept queue behind long jobs, where - they time out having never been looked at — recorded as `ERROR`, which reads + they time out having never been looked at, recorded as `ERROR`, which reads as a compliance difference and is not one. The runner therefore passes `--workers $JOBS` and matches `--max-scan-secs` to the client timeout. @@ -74,19 +74,19 @@ FRESH_CLAM=1 scripts/difftest.sh # rebuild the clam cache ``` `LIMIT` samples deterministically (fixed `SEED`) and **both engines scan exactly -that list** — the manifest is generated once and shared. Giving each engine its +that list**: the manifest is generated once and shared. Giving each engine its own random subset would compare different files and call the result a difference. ### What makes a cached result reusable -A results table is stamped with the profile it was produced under — the flags +A results table is stamped with the profile it was produced under: the flags (`COMPAT`, `HEURISTICS`, `JOBS`) **and a checksum of the manifest itself**. A run whose stamp differs discards the table rather than joining against it. The manifest checksum is the part that is easy to get wrong. Phase 1 skips the clam scan on *row count*, so a cache holding the right number of rows for the -wrong files would otherwise be reused — and since the compare joins by path, the +wrong files would otherwise be reused, and since the compare joins by path, the overlap would be empty and the run would confidently report on nothing. Changing `SEED` at a fixed `LIMIT`, or adding samples to the corpus, both produce exactly that shape. Neither `LIMIT` nor `SEED` alone is sufficient to detect it, which is @@ -98,11 +98,11 @@ why the manifest is fingerprinted instead. |---|---| | corpus on disk | 18,596 files, 34 GB | | what the manifest takes (≤ `MAXSZ`, 20 MB) | 8,978 files, 20.8 GB, mean 2.3 MB | -| cold disk read | ~8.6 MB/s — **the wall** | +| cold disk read | ~8.6 MB/s: **the wall** | | clamd, all-match | ~0.2–0.7 files/s depending on concurrency | | concurrency gain | ~1.75× from 1→8 jobs, not 8× | -Re-measure rather than trust the table — the corpus grows. The two rows are +Re-measure rather than trust the table: the corpus grows. The two rows are different populations and it matters which one an estimate uses: **less than half** the files on disk are inside the size cap. @@ -114,27 +114,27 @@ either scanner. Use `LIMIT` while iterating. Everything below is reproducible from a clean checkout; nothing secret is committed (the corpus and the API key are both gitignored). -1. **Build the binary under test** — `cargo build --release -p exav`. +1. **Build the binary under test**: `cargo build --release -p exav`. 2. **Tools:** Docker (runs the reference `clamd`), `clamdscan` on the host (Debian/Ubuntu: `apt-get install clamdscan`), and `python3` with `pyzipper` (`pip install pyzipper`) only if you fetch the corpus. -3. **Signatures** into the clamd DB dir the runner expects — both engines must +3. **Signatures** into the clamd DB dir the runner expects: both engines must load the *same* set. The runner fetches these itself on first run, so this step is only for supplying your own. `DBDIR` defaults to **`$TMPROOT/difdb_daily`**, and `TMPROOT` is resolved at runtime: `/var/tmp` when `/tmp` is tmpfs (the common case), else `/tmp`. So on a typical host the database lives at `/var/tmp/difdb_daily`, **not** - `/tmp/difdb_daily` — check before concluding it is missing: + `/tmp/difdb_daily`; check before concluding it is missing: ```sh scripts/difftest.sh --help 2>/dev/null; # or just: ls "$( [ "$(stat -f -c %T /tmp)" = tmpfs ] && echo /var/tmp || echo /tmp )/difdb_daily" # to supply your own instead of letting the runner fetch: - cvd config set --dbdir /var/tmp/difdb_daily && cvd update # Cisco's cvdupdate + cvd config set --dbs-directory /var/tmp/difdb_daily && cvd update # Cisco's cvdupdate # (or copy an existing daily.cvd there; `main.cvd` too for full coverage) ``` 4. **Corpus** under `corpus/samples/` (gitignored). Put your own samples there, - or fetch from MalwareBazaar — needs a free Auth-Key, read from + or fetch from MalwareBazaar, which needs a free Auth-Key, read from `MALWAREBAZAAR_API_KEY` in a gitignored `.env.local` at the repo root (never committed; get a key at ): ```sh @@ -145,10 +145,10 @@ committed (the corpus and the API key are both gitignored). ## Protocol -0. **Memory & temp layout on a small host.** The signature automaton is - memory-heavy to *build* (see [DATABASE.md](DATABASE.md)), and RAM is the usual - failure: - - `/tmp` is often **tmpfs (RAM-backed)** — anything there counts against RAM. +0. **Memory & temp layout on a small host.** The signature database takes more + memory to *build* than to load (see [DATABASE.md](DATABASE.md)), and RAM is + the usual failure: + - `/tmp` is often **tmpfs (RAM-backed)**: anything there counts against RAM. - A resident `clamd` from a previous run can hold ~1 GB. - clamd extracts **every scanned file's members into temp**; over a run that can reach gigabytes. The harness bind-mounts `$SOCK_DIR` as clamd's `/tmp`, @@ -175,12 +175,12 @@ committed (the corpus and the API key are both gitignored). The harness does this itself when needed, so run it by hand only to control *when* the memory peak happens. It **reuses a database that still loads** with the current binary and only rebuilds when it is missing or fails to load (a - format bump) — so a format-compatible binary change does **not** pay the + format bump), so a format-compatible binary change does **not** pay the RAM-heavy rebuild. To force a clean rebuild, `rm -f $DB` first. (On a host too small to build the database at all, build it elsewhere and copy it in, or point - `DB=` / `TMPROOT=` at disk — the script prints this hint if a build OOMs.) + `DB=` / `TMPROOT=` at disk; the script prints this hint if a build OOMs.) -3. **Run `scripts/difftest.sh`.** It manages everything — resolves a disk-backed +3. **Run `scripts/difftest.sh`.** It manages everything: it resolves a disk-backed temp dir, reuses-or-builds the database, runs each phase with only that engine resident, and tears down on exit. On a host where `/tmp` is tmpfs and `/var/tmp` is disk (the common case) a bare invocation "just works". @@ -206,7 +206,7 @@ Key choices, and why: recorded as engine disagreements. One at a time, they are not. - **`shuf` with a fixed seed.** Random order means even a partial run is a representative sample, and the fixed seed means the same `LIMIT` selects the - same files, so two runs are comparable. Don't use `head` on the corpus — + same files, so two runs are comparable. Don't use `head` on the corpus: `find` returns the curated family folders first, which biases the sample toward easily-detected families (e.g. Locky). - **Size cap (`MAXSZ`, 20 MB default).** exav currently buffers whole members @@ -216,15 +216,15 @@ Key choices, and why: ## Interpreting results - **AGREE / clean**: exav matches clamscan. This is the headline metric. -- **FN** (clamscan found, exav missed): a real gap — investigate the signature +- **FN** (clamscan found, exav missed): a real gap; investigate the signature type (`clamscan --debug` on the file shows what it matched). Known causes found this way: imphash ordinal encoding, section-hash on truncated PEs, and - **embedded-PE scanning** (a PE appended inside another file) — all since fixed. + **embedded-PE scanning** (a PE appended inside another file), all since fixed. - **EXAV_ONLY** (exav found, clamscan clean): *do not assume this is a false positive.* On the 8,978-file run of 2026-07-28, all 32 were checked and none - was an exav error — see below. + was an exav error (see below). - **Absolute hit rate is DB-bound, not a corpus problem.** With `daily` only (no - `main.cvd`, which holds most coverage) both engines detect a small fraction — + `main.cvd`, which holds most coverage) both engines detect a small fraction: e.g. clamscan with full `main+daily` flags ~13% of a random MalwareBazaar sample, and far less with `daily` alone. The *agreement* is what matters. @@ -233,7 +233,7 @@ Key choices, and why: | verdict | compat (as run) | full capability (re-scan) | |---|---|---| -| clean (both OK) | 4,137 | — | +| clean (both OK) | 4,137 | n/a | | ERROR | 3,542 | 29 (exav side) | | AGREE | 766 | 769 | | CAREFUL | 471 | 458 | @@ -248,15 +248,15 @@ fixes below. The two rows at the bottom are both "clamd named the malware and exav did not", and the difference between them decides how serious each is: -* **`FN`** — exav returned a clean `OK`. A user acting on that verdict runs the +* **`FN`**: exav returned a clean `OK`. A user acting on that verdict runs the file. This is the failure the engine exists to prevent, and it is the number that had to reach zero. -* **`CAREFUL_FN`** — exav returned `UNSCANNABLE` / `PASSWORD-PROTECTED` / +* **`CAREFUL_FN`**: exav returned `UNSCANNABLE` / `PASSWORD-PROTECTED` / `LIMITS-EXCEEDED`. It could not name the malware, but it did not claim the file was safe, so the file is still quarantinable and the gap is visible. -Both counts are now zero. The two samples that sat in `CAREFUL_FN` — an -MPRESS-packed dropper and a UPX image with a bare `PackHeader` — are unpacked +Both counts are now zero. The two samples that sat in `CAREFUL_FN` (an +MPRESS-packed dropper and a UPX image with a bare `PackHeader`) are unpacked and detected under clamd's own signature names. Neither needed a new decoder written from scratch. @@ -266,7 +266,7 @@ the harness rather than the engine, and the comparison is only sound across the ~5,400 rows where both engines actually answered. The run also ended by OOM-killing the exav daemon. -### The five FNs — three fixed, two are packer support +### The five FNs: three fixed, two are packer support Each was a distinct cause, which is the argument for triaging every one rather than the biggest bucket: @@ -274,7 +274,7 @@ than the biggest bucket: * two APKs where a packer set the "encrypted" flag on every member; * one JAR hiding classes behind trailing-slash names, which *also* needed `Target:12` re-enabled before the payload matched; -* two packed PEs — one MPRESS, one UPX. Both are now unpacked and detected. +* two packed PEs, one MPRESS, one UPX. Both are now unpacked and detected. Neither needed a decoder written from scratch, which is the interesting part: ClamAV unpacks MPRESS with a **bytecode program shipped in the signature DB**, which exav already loaded and could already run; and the UPX sample only @@ -285,7 +285,7 @@ than the biggest bucket: ### Measure added coverage at COMPAT=0, never at the default The harness runs `--clamav-compat` by default, which is right for checking -*agreement* — it holds limits and format reach fixed so a disagreement means a +*agreement*: it holds limits and format reach fixed so a disagreement means a real engine difference. It is the wrong mode for measuring what exav adds, because compat is defined as switching that reach off. @@ -300,14 +300,14 @@ A 3× understatement. Any claim of the form "exav detects things ClamAV misses" has to come from a `COMPAT=0` pass; anything else is measuring exav with its own extractors deliberately disabled. -### The EXAV_ONLY hits — zero false positives +### The EXAV_ONLY hits: zero false positives Every one was checked, and the method matters because the naive reading is wrong. Handing clamd **exav's own extracted members** (see `exav-unpack/examples/dump_members.rs`) makes clamd flag them under the *same signature name* while still calling the container clean. That is the oracle settling it: exav was not matching wrongly, it was reaching content clamd never -unpacked — `word/*.rtf` inside a docx, `/cab1.cab/MainExecutable` inside an MSI, +unpacked: `word/*.rtf` inside a docx, `/cab1.cab/MainExecutable` inside an MSI, `x.exe` inside a gzipped `.bat`. * **101 of 104** were hits nested inside extracted members. @@ -317,7 +317,7 @@ unpacked — `word/*.rtf` inside a docx, `/cab1.cab/MainExecutable` inside an MS RTF: none of the signature's seven subsignatures occurs in the raw file and all seven occur in the decoded image. -Four of them looked like a target-gating bug at first — `Target:1` (PE-only) +Four of them looked like a target-gating bug at first: `Target:1` (PE-only) signatures apparently firing on RTF files. They were not: the signature's bytes do not occur anywhere in the raw RTF, only inside a hex-encoded embedded PE. The detections were right and the *reporting* was wrong; 13 of the 17 sites that @@ -326,10 +326,10 @@ like container-level ones. Fixed, with `exav-core/tests/match_location.rs` guarding it. Without match locations this triage is guesswork, which is why that bug is worth more than its cosmetic appearance suggests. -### NAMEDIFF — benign +### NAMEDIFF: benign Both engines detect; they disagree only on which signature won a first-match race. The dominant case (16 of 25) is clamd's `Win.Ransomware.Wanna` versus -exav's `Win.Exploit.Doublepulsar` on the same samples — and WannaCry bundles +exav's `Win.Exploit.Doublepulsar` on the same samples, and WannaCry bundles DoublePulsar. Verified by hand: both of exav's required subsignatures are really present (offsets 249260 and 249335). No action. diff --git a/docs/WASM.md b/docs/WASM.md index 86a104cc..80a6c7eb 100644 --- a/docs/WASM.md +++ b/docs/WASM.md @@ -2,7 +2,7 @@ exav can run its entire scanning engine inside a WebAssembly sandbox, letting you load **untrusted signature databases** without giving them access to your -host. The scanner compiles to `wasm32-wasip1` — a standard WASI command — and +host. The scanner compiles to `wasm32-wasip1`, a standard WASI command, and runs under any compatible runtime (wasmtime, wasmer, wazero, etc.) with **zero custom host code**. @@ -31,23 +31,28 @@ wasmtime \ The `--dir` flags mount directories into the WASM sandbox: -- `--dir /path/to/sigs::/db` — signature DB mounted at `/db` inside the module -- `--dir .::.` — current working directory mounted at `/` (so relative paths work) +- `--dir /path/to/sigs::/db`: signature DB mounted at `/db` inside the module +- `--dir .::.`: current working directory mounted as `.` inside the module (so + relative paths work) ## How it works -The WASM module is a plain WASI command — same as any CLI tool, but sandboxed: +The WASM module is a plain WASI command, the same as any CLI tool, but sandboxed +(`crates/exav-core/src/bin/exav-wasm.rs`): 1. **Startup**: reads CLI args via WASI -2. **Load signatures**: walks the mounted signature directory recursively, - loading `.ndb`/`.ldb`/`.hdb`/`.cvd`/`.cld`/`.yar`/etc. files (including - CVD/CLD container extraction — handled inside the module) -3. **Build database**: compiles the signature matcher (Aho-Corasick automaton) -4. **Scan files**: for each file argument, reads it from the mounted filesystem, - runs the full scan (patterns, hashes, YARA, bytecode, structural analysis), - and writes a JSON `ScanReport` to stdout - -Progress and errors go to stderr. Results go to stdout — pipe to `jq`: +2. **Load signatures**: the first argument is a prebuilt `.exavdb`, or a + directory walked recursively for `.ndb`/`.ldb`/`.hdb`/`.cvd`/`.cld`/`.yar`/etc. + files (CVD/CLD extraction happens inside the module), then compiled into the + signature index. A database with no real signatures is refused (exit 2). +3. **Scan files**: each further argument is a file (directories are not + walked), read whole from the mounted filesystem and scanned with + `ScanOptions::default()`: patterns, hashes, YARA, bytecode, structural + analysis. No CLI flags apply. +4. **Report**: one JSON object per file on stdout; exit `1` if anything was + found, else `2` on an error, else `3` if a file was not fully scanned, else `0`. + +Progress and errors go to stderr. Results go to stdout; pipe them to `jq`: ```sh wasmtime --dir ./sigs::/db --dir .::. exav-wasm.wasm /db *.exe 2>/dev/null | jq '.verdict' @@ -55,14 +60,14 @@ wasmtime --dir ./sigs::/db --dir .::. exav-wasm.wasm /db *.exe 2>/dev/null | jq ## Output format -Each scanned file produces one JSON object on stdout: +Each scanned file produces one JSON object on stdout, the `ScanReport` plus +`file` (keys sorted), or `file` and `error` for a file that could not be read: ```json -{"verdict":"Clean","findings":[{"label":"type","detail":"PE32+ executable"}]} - -{"verdict":{"Infected":{"signature":"Win.Trojan.Agent-1234","offset":4096,"method":"Pattern"}},"findings":[]} - -{"verdict":{"LimitsExceeded":{"reason":"scan-size limit"}},"findings":[]} +{"file":"a.exe","findings":[{"detail":"PE","label":"type"}],"verdict":"Clean"} +{"file":"b.exe","findings":[{"detail":"PE","label":"type"}],"verdict":{"Infected":{"method":"Pattern","offset":4096,"signature":"Win.Trojan.Agent-1234"}}} +{"file":"big.iso","findings":[{"detail":"ISO","label":"type"}],"verdict":{"LimitsExceeded":{"reason":"..."}}} +{"error":"No such file or directory (os error 44)","file":"gone.exe"} ``` `Verdict` variants: `Clean`, `Infected`, `LimitsExceeded`, `Unscannable`, @@ -82,12 +87,12 @@ your attack surface. A malicious `.ndb` file could: Running in native code, any of these compromise the host. In a WASM sandbox: -- **Memory is bounded** — the module can't access memory outside its linear +- **Memory is bounded**: the module can't access memory outside its linear memory instance -- **No system calls** — WASI restricts file/network access to explicitly mounted +- **No system calls**: WASI restricts file/network access to explicitly mounted paths -- **Fuel limits** — wasmtime can cap instruction counts, killing runaway modules -- **No `unsafe` escape** — even if the Rust code has bugs, WASM type-checking +- **Fuel limits**: wasmtime can cap instruction counts, killing runaway modules +- **No `unsafe` escape**: even if the Rust code has bugs, WASM type-checking prevents memory corruption from escaping the sandbox ### Why not a custom host? @@ -96,14 +101,14 @@ An earlier design used a custom host binary (`exav-host`) that embedded the `wasmi` WASM interpreter and called exported functions via a raw pointer ABI. This had two problems: -1. **Trust shifts to the host** — users now had to trust both the WASM module AND the +1. **Trust shifts to the host**: users now had to trust both the WASM module AND the host binary. The host contained wasmi (less audited than wasmtime) plus custom pointer-manipulation code. -2. **No auditability** — the custom ABI (alloc/dealloc/load_file/scan) was +2. **No auditability**: the custom ABI (alloc/dealloc/load_file/scan) was opaque to users and hard to verify. -The new design uses a standard WASI command. Users bring their own wasmtime — +The new design uses a standard WASI command. Users bring their own wasmtime, a ByteCode Alliance project, audited, used in production by Fastly, Cloudflare, and others. The host code is wasmtime itself, not ours. @@ -122,7 +127,7 @@ and others. The host code is wasmtime itself, not ours. ### Memory safety -- The WASM module runs in a linear memory sandbox — no pointer escapes +- The WASM module runs in a linear memory sandbox: no pointer escapes - `exav-unpack` (the extraction engine) has `#![forbid(unsafe_code)]` - `exav-core` scanning code is safe Rust - Residual `unsafe` is in audited compression/crypto dependencies, not @@ -130,23 +135,24 @@ and others. The host code is wasmtime itself, not ours. ### Resource limits -- **WASM linear memory**: bounded by the runtime (default 4 GiB max, typically - much less) -- **Fuel/instructions**: wasmtime can limit total instructions per scan -- **Filesystem**: only mounted directories are accessible — the module can't +- **WASM linear memory**: up to 4 GiB for `wasm32` unless the runtime caps it + (wasmtime: `-W max-memory-size=`) +- **Fuel/instructions**: wasmtime can limit total instructions per run + (`-W fuel=`) +- **Filesystem**: only mounted directories are accessible; the module can't read `/etc/passwd` or write to disk - **Network**: no network access in WASI (unless explicitly enabled) ### Panic isolation If the WASM module panics (e.g., on corrupt input), wasmtime catches it as a -trap — the host process is unaffected. This is the same isolation as the native -`catch_unwind` boundary, but at the WASM level. +trap, and the host process is unaffected. This is the same isolation as the +native `catch_unwind` boundary, but at the WASM level. ## Building ```sh -# Requires Rust 1.85+ and the wasm32-wasip1 target +# Requires Rust 1.91+ and the wasm32-wasip1 target rustup target add wasm32-wasip1 cargo build --release --target wasm32-wasip1 -p exav-core --features wasi-bin ``` @@ -155,17 +161,25 @@ The output is `target/wasm32-wasip1/release/exav-wasm.wasm` (~5-6 MB). ## Limitations -- **No daemon mode** — each `wasmtime run` invocation loads the DB from scratch. +- **No daemon mode**: each `wasmtime run` invocation loads the DB from scratch. For high-throughput scanning, use a native `exav --listen` instead. -- **No stdin streaming** — files must be on a mounted filesystem (WASI doesn't +- **No stdin streaming**: files must be on a mounted filesystem (WASI doesn't support the pipe-from-stdin pattern that native `exav` uses for streaming). -- **No HTTP(S) scanning** — the WASM module can't make network requests, so - `exav://` URL targets aren't available. -- **~10-20% overhead** vs native — wasmtime JIT is fast, but the WASM +- **No HTTP(S) scanning**: the WASM module can't make network requests, so + `http(s)://` targets aren't available. +- **No flags**: every file is scanned with `ScanOptions::default()`: no + `--detect`, `--all-matches`, `--passwords` (only `.pwdb` passwords) or + `--partial-as`, no spill, and YARA's filename externals are undefined. + PUA signatures load only from a `.exavdb` built with `--detect pua`. +- **Memory**: the database and each scanned file (read whole) share the 4 GiB + `wasm32` address space. +- **~10-20% overhead** vs native: wasmtime JIT is fast, but the WASM sandboxing layer adds some cost. -- **DB loading is slower** — the full ClamAV DB build wants ~6 GiB transient - RAM; in a WASM sandbox this is bounded by the runtime's memory limit. Use a - prebuilt database for large signature sets. +- **DB loading is slower**: building the full ClamAV set from raw signatures + takes more memory than loading it, and in a WASM sandbox that is bounded by + the runtime's memory limit. Use a prebuilt database for large signature sets + (build and load costs are in the site's prebuilt-database guide, + ). ## Two WASM surfaces, and how a third gets added @@ -177,18 +191,18 @@ bindings go: | target | `wasm32-wasip1` | `wasm32-unknown-unknown` | | artifact | a WASI **command** (`[[bin]]`) | a **`cdylib`** + JS glue | | runs in | wasmtime / wasmer / wazero | a browser, or Node | -| talks over | argv, stdio, mounted dirs | `wasm-bindgen`, 848 lines of it | -| in the workspace | yes | no — it needs its own `[profile.release]` and `wasm-pack` metadata | +| talks over | argv, stdio, mounted dirs | `wasm-bindgen` | +| in the workspace | yes | no: it needs its own `[profile.release]` and `wasm-pack` metadata | The WASI side needs no bindings at all: a `main()` and the standard library. The browser side is the one with a cost, and the cost is the binding layer, not the -port — `exav-unpack`, `exav-core`, `exav-grep` and `exav-pe-emu` all already +port: `exav-unpack`, `exav-core`, `exav-grep` and `exav-pe-emu` all already compile clean for `wasm32-unknown-unknown` with no changes. **Bindings live in one crate, not one crate per library.** Cargo's `crate-type` is a property of the `[lib]` target and cannot be turned on by a feature, so `#[wasm_bindgen]` cannot live inside `exav-unpack` itself and produce a `cdylib` -only for WASM builds — every native build would link a `.so` too. That leaves two +only for WASM builds: every native build would link a `.so` too. That leaves two shapes, and the second is the one to grow into: - *One `-wasm` sibling per library.* Each is a crate, a `package.json`, a @@ -197,9 +211,9 @@ shapes, and the second is the one to grow into: - *One bindings crate, one surface per feature.* `unpack`, `scan`, `grep` and `pe-emu` become Cargo features of a single `cdylib`; the JS glue for byte marshalling, `ReadableStream` and error mapping is written once. Separate npm - packages still come out of it — the same source built N times with different + packages still come out of it (the same source built N times with different `--no-default-features --features …` and `--out-dir`, each rewritten to its own - package name — so a consumer who only wants the extractor still downloads only + package name), so a consumer who only wants the extractor still downloads only the extractor's bytes. Module size is the reason to keep the feature gates honest rather than ship one @@ -211,7 +225,7 @@ after `wasm-opt`. - **Native `exav`**: best performance, full features, but requires trusting the scanner with host access. -- **Docker container**: similar isolation to WASM but heavier — a full OS image +- **Docker container**: similar isolation to WASM but heavier: a full OS image vs a single `.wasm` file. - **Browser (exav-unpack-wasm)**: the archive *extractor* (not the full scanner) runs in-browser via wasm-bindgen. Useful for client-side archive preview, but diff --git a/docs/YARA.md b/docs/YARA.md index 86a6d3b6..c44586b6 100644 --- a/docs/YARA.md +++ b/docs/YARA.md @@ -1,4 +1,4 @@ -# YARA support — native engine, design, coverage, and reuse +# YARA support: native engine, design, coverage, and reuse ClamAV loads `.yar`/`.yara` rules by default, so a drop-in must too. exav ships a **native YARA engine**: it uses the YARA-X parser and compiles rule conditions to @@ -16,7 +16,7 @@ source and test vectors under that license, with attribution: the yara-x copyright + 3-clause text + disclaimer are retained (`LICENSE-YARA-X`), files substantially derived from yara-x carry a header note, and the exav project does **not** use the "YARA-X"/"VirusTotal" names to endorse itself. This is -categorically different from the hard "never derive from ClamAV" rule — that rule +categorically different from the hard "never derive from ClamAV" rule: that rule is GPL-specific and does not apply here. ## Where the code lives @@ -32,7 +32,8 @@ crates/exav-core/src/yara/ scanner.rs Rules / Scanner / ScanResults, rule references, global/private gating base64.rs base64/base64wide modifier expansion error.rs compile errors - modules/ math, hash, pe (goblin), string, time + atoms.rs required-literal atom prefilter + modules/ math, hash, pe (goblin), string, time, elf, dotnet ``` The engine is a module of `exav-core`, not a separate crate. It shares @@ -52,10 +53,10 @@ Because the parser AST is lifetime-bound to the source (`<'src>`), the compiler **lowers it into an owned IR** so `Rules` is `'static` and cacheable. That lowering (`compiler.rs`) is the bulk of the work. -- **Front-end — reused:** the `yara-x-parser` crate (tokenizer + AST). It is the +- **Front-end (reused):** the `yara-x-parser` crate (tokenizer + AST). It is the light half of yara-x (no wasmtime). We do not reimplement the grammar. - **Strings (`matcher.rs`):** each pattern compiles to a matcher that returns - **every** `(offset, length)` match — required for `#a` (count), `@a[i]` + **every** `(offset, length)` match, as required for `#a` (count), `@a[i]` (offset), `!a[i]` (length), `$a at N`, `$a in (a..b)`. Text patterns handle `nocase`/`ascii`/`wide`/`fullword`/`xor`/`base64`/`base64wide`; hex patterns are lowered to an equivalent byte-regex (`??` → any byte, `[a-b]` jumps → lazy @@ -70,9 +71,10 @@ lowering (`compiler.rs`) is the bulk of the work. reads, `filesize`, `entrypoint`, and module field/function access. - **Modules (`modules/`):** each imported module parses the scanned bytes once per scan into an owned `Value` tree (struct/array/map); conditions resolve field - access against it. `pe` uses `goblin`; `math`/`hash`/`string`/`time` are native. + access against it. `pe`, `elf` and `dotnet` use `goblin`; `math`/`hash`/ + `string`/`time` are native. -## What is reused from exav-core — and what deliberately isn't +## What is reused from exav-core, and what deliberately isn't Living inside exav-core buys a few **leaf-level** reuses: @@ -86,18 +88,16 @@ because their semantics are the wrong shape. The reasons are recorded here so nobody re-attempts it and breaks the diff harness: - **The AV pattern `engine`** is architected for *"does signature S match, and - where is its first hit"* over one shared Aho-Corasick automaton. It returns the - first match only (`verify` → `Option` start; `match_forward` discards the - end offset), dedups by signature name, and — critically — is **non-exhaustive by - design**: the verify-step budget (`SCAN_VERIFY_BUDGET`/`SCAN_TRUNCATED`) makes - wildcard verification *bail conservatively* under load. That is correct for a - clean/infected verdict but wrong for YARA, whose match set must be **complete - and deterministic**. It also has no path for YARA's string modifiers or regexp. + where is its first hit"* over one index of literal anchors. It returns the + first match only (`verify` → `Option` start; the simulator reports no end + offset) and dedups by signature name. That is correct for a clean/infected + verdict but wrong for YARA, whose match set must be **complete and + deterministic**. It also has no path for YARA's string modifiers or regexp. - **`engine/logic.rs`'s `Node`** is a fixed 5-variant boolean/count algebra with - static thresholds — no `not`/`none of`, no percentage or computed quantifiers, + static thresholds: no `not`/`none of`, no percentage or computed quantifiers, no anchors, no `defined`, no undefined propagation. - **The `.cbc` bytecode VM** is a typed LLVM-bitcode ISA with a ClamAV API and a - machine-word/pointer value model — **no "undefined", no struct/array/map + machine-word/pointer value model: **no "undefined", no struct/array/map values**. YARA's evaluator is built on `Option` and dynamic module structs; retargeting the VM would need a new codegen backend plus a parallel dynamic-value layer, at real risk to the parity baseline. @@ -110,9 +110,9 @@ as-is. Until then, the matcher and the `Cond` evaluator stay in `yara/`. ## Dependency footprint With the `yara` feature on, exav-core's YARA path adds only light, pure-Rust -crates: `yara-x-parser`, `regex-automata`, `base64`, `bstr`, `crc32fast` (and -reuses exav-core's existing `goblin`/hash/`memchr`). There is **no -wasmtime, no Cranelift, no `walrus`, no `rsa`** — the ~200-crate heavy subtree the +crates: `yara-x-parser` (which brings `bstr`), `regex-automata` and `base64` +(and reuses exav-core's existing `goblin`/hash/`crc32fast`/`memchr`). There is **no +wasmtime, no Cranelift, no `walrus`, no `rsa`**; the ~200-crate heavy subtree the old `yara-x` dependency pulled is gone. A non-`yara` build compiles all of this away. @@ -123,30 +123,30 @@ away. (boolean/`of`/counts/offsets/lengths/anchors/arithmetic/bitwise/float/comparisons/ string-ops/`matches`); `for … in`/`for … of`/`with`; integer reads (`uint/int 8/16/32` + `_be`); `filesize`; `entrypoint`; **external variables** -(`filename`/`filepath`/`extension`/`filetype`/`owner`, host-supplied from the -scan context); rule references; global/private rules; and the `pe`/`math`/`hash`/ -`string`/`time` modules. +(`filename`/`filepath`/`extension`); rule references; global/private rules; and +the `pe`/`math`/`hash`/`string`/`time`/`elf`/`dotnet` modules. **External variables** are declared string-typed by default (`Compiler::new` pre-declares the five standard ones; `Compiler::define_external` adds more) and set per scan (`Scanner::set_global`; exav-core threads the filename from `ScanOptions.filename` → `YaraDb::scan`, deriving `filename`/`filepath`/ -`extension` — the latter lowercased with leading dot, e.g. `".js"`). An unset -external is *undefined* (false in boolean context), matching YARA semantics. +`extension`, the latter lowercased with leading dot, e.g. `".js"`). `filetype` +and `owner` are declared but never set. An unset external is *undefined* (false +in boolean context), matching YARA semantics. -**Not yet supported — and rejected as explicit COMPILE ERRORS, never silently +**Not yet supported, and rejected as explicit COMPILE ERRORS, never silently mis-evaluated** (per exav's never-silent-incompleteness rule): the `macho`/`dex`/`cuckoo`/`androguard`/… modules; `include`; `floatN()` reads; -the `wide` modifier on *regexp* patterns; a bare regexp used as a whole condition; -and the unimplemented `pe.` fields (`rich_signature`, `version_info`, resources, -`signatures`, regexp `imports`/`exports` overloads). +a bare regexp used as a whole condition; `xor`/`base64` on a regexp pattern; +the unimplemented `pe.` fields (`rich_signature`, `version_info`, resources, +`signatures`, regexp `imports`/`exports` overloads); and `dotnet.classes`. ## Rule-rejection accounting (never silent) A real `.yar` feed may contain rules using unsupported constructs. exav compiles **per rule** (`Compiler::add_source_lenient`), so one unsupported rule can't drop a whole feed, and every rejection is recorded in `YaraDb.rejected` (name + reason) -and logged (`WARN`) at database-build time — the coverage gap is always visible, +and logged (`WARN`) at database-build time. The coverage gap is always visible, never an invisibly-missing rule. ## Condition step budget @@ -161,8 +161,9 @@ shared across all of its rules, charged one per IR node evaluated. A condition that runs out yields *undefined*, the same value any other unanswerable sub-expression yields, so it propagates through the operators without a special case and the rule does not match. `ScanResults::budget_exhausted()` reports that -it happened: the results are then a lower bound, and the ruleset is what needs -fixing. Large community feeds evaluate in the low millions of steps for a whole +it happened: the results are then a lower bound, and exav-core reports the scan +`LimitsExceeded` unless something matched. A pattern that reaches +`MAX_MATCHES` (1M, as in yara-x) is treated the same way. Large community feeds evaluate in the low millions of steps for a whole scan, so the ceiling costs real rules nothing. ## Measured real-world coverage @@ -176,66 +177,65 @@ Compiled against two public rulesets (harness: `examples/yara_coverage.rs`): | A commercial vendor ruleset | 173 | **100%**, 0 disagreements vs yara-x | **Correctness:** ~17,300 rules were cross-checked against the real `yara-x` over -varied inputs — **0 disagreements** on matching-rule sets. The coverage figures +varied inputs, with **0 disagreements** on matching-rule sets. The coverage figures therefore measure correct matching, not lenient parsing. Top rejection buckets (by rules blocked): | Gap | community-ruleset cost | Status / effort | |---|---|---| -| ~~external variables~~ (`filename`/`filepath`/`extension`) | ~~~10%~~ | **DONE** — 88%→98.3% | -| `wide` on regexp | ~1.2% | now the #1 remaining gap; moderate (widen HIR to UTF-16LE) | +| ~~external variables~~ (`filename`/`filepath`/`extension`) | ~~~10%~~ | **DONE**: 88%→98.3% | +| ~~`wide` on regexp~~ | ~~~1.2%~~ | **DONE** (widened HIR, UTF-16LE) | | cheap `pe` helpers (`is_dll`/`is_32bit`/`number_of_resources`/`overlay`) | ~0.1% | trivial (derivable from goblin) | | `pe.signatures`/`version_info` | ~0.3% | moderate (resource + Authenticode parsing) | | `floatN()` | 0 rules observed | trivial, not worth prioritizing | -| macho/dex/androguard/cuckoo | ~0 in practice | large, low value — **skip** | -| ~~dotnet~~ | — | **implemented**, validated field-by-field against `yr dump --module dotnet` | +| macho/dex/androguard/cuckoo | ~0 in practice | large, low value: **skip** | +| ~~dotnet~~ | n/a | **implemented**, validated field-by-field against `yr dump --module dotnet` | -With external variables closed, the remaining *relevant* gap in that community -ruleset is small -(`wide`-on-regexp + a few cheap `pe` helpers). Exotic modules are a trap: large -effort, near-zero real-world rules. +The remaining *relevant* gap in that community ruleset is a few cheap `pe` +helpers. Exotic modules are a trap: large effort, near-zero real-world rules. ## Performance -The string matcher uses a **required-literal atom Aho-Corasick prefilter** -(`yara/atoms.rs`) — the standard YARA scan architecture. Measured: on 173 real +The string matcher uses a **required-literal atom prefilter** +(`yara/atoms.rs`), the standard YARA scan architecture. Measured: on 173 real rules (a commercial vendor ruleset) over ~2.3 MB obfuscated JS, **~1.3 s/file → ~30–50 ms/file (~44×)**, with **byte-identical detections** (the yara-x A/B diff stays at 0 -disagreements; a build-time `EXAV_YARA_NO_GATE` kill switch proves gate-on == -gate-off on the real corpus). +disagreements). How it works: -- **Atom extraction** (compile time) — each pattern contributes one or more +- **Atom extraction** (compile time): each pattern contributes one or more *required literal atoms* (substrings that must appear in **every** match), chosen by selectivity (`anchor_score`): a long window for plain/`wide` - literals, the lowercased window for `nocase` (into a separate CI automaton to - avoid case-variant explosion), the `searched` bytes for `base64`, and — for - regexp/hex — the required prefix/suffix literal `Seq` from `regex-syntax`'s + literals, the lowercased window for `nocase` (looked up on the bytes + lowercased, so case variants are never enumerated), the `searched` bytes for + `base64`, and, for + regexp/hex, the required prefix/suffix literal `Seq` from `regex-syntax`'s extractor (the same analysis the `regex` crate trusts for its own prefilters). -- **One shared `daachorse` Aho-Corasick** over all atoms (two automata: - case-sensitive over the buffer, `nocase` over a lowercased copy). Scanned once. +- **One shared index of all atoms**, the one the signature engine finds its + anchors with (`grams/`), case-sensitive and `nocase` atoms in two partitions. + The object is read once, with no lowercased copy. - **The gate is FN-safe:** a pattern is skipped **only** when a literal that must appear in every match is provably absent; when no sound required literal exists - (`xor` — bytes transformed; `/a*/`, optional/alternation branches, `/.abc/` + (`xor`, whose bytes are transformed; `/a*/`, optional/alternation branches, `/.abc/` with no fixed prefix/suffix) the pattern is **ungated** and always scanned. On - that vendor ruleset all 1437 patterns gated, 0 ungated — hence the speedup. + that vendor ruleset all 1437 patterns gated, 0 ungated, hence the speedup. - Two independent matcher fixes: `anchored_matches` went **O(n²) → O(matches)** (leftmost-first unanchored sweep, same overlapping set), and literals use `memchr::memmem` SIMD search (advancing by 1 byte to keep YARA's overlapping counts). Note: YARA the format hands the engine the strings/condition split (the prefilter -affordance) but deliberately leaves the *atom algorithm* to each engine — so this +affordance) but deliberately leaves the *atom algorithm* to each engine, so this was the standard engine responsibility yara-x implements and exav initially skipped, not a format gap. **Still open (stretch, deferred to keep the diff green):** per-position windowed verify (verify only in a window around each atom hit, vs. re-scanning the whole -buffer for a gated pattern); **serializing the compiled automaton into `.exavdb`** -(today the yara `Rules` aren't serialized at all — only rule *source* is, and the -automaton rebuilds lazily on first scan, which is already fast); and yara-x-style -`slow_pattern` compile warnings for ungated/atomless rules. +buffer for a gated pattern); and yara-x-style `slow_pattern` compile warnings +for ungated/atomless rules. (The compiled rule set, atom index included, is +serialized into `.exavdb` as `YaraDb::compiled_blob`, so a prebuilt database +does not recompile.) ## Testing @@ -244,9 +244,9 @@ automaton rebuilds lazily on first scan, which is already fast); and yara-x-styl public API (`condition_true!`/`rule_true!`/`pattern_match!`-style macros). - **A/B differential** (`tests/yara_difftest.rs`): gives the same rules and the same inputs to exav's engine and to yara-x, and asserts the matching-rule sets - are **equal**. yara-x is invoked as the `yr` *binary*, not linked — nothing + are **equal**. yara-x is invoked as the `yr` *binary*, not linked, so nothing from it enters the dependency graph. The tests skip when `yr` is absent; - `cargo install yara-x-cli` provides it, and `EXAV_YR_BIN` overrides the lookup. + `cargo install yara-x-cli` provides it, and `EXAV_DEBUG_YR_BIN` overrides the lookup. - **Corpus coverage** (`examples/yara_coverage.rs`, `tests/yara_coverage_difftest.rs`): the coverage and at-scale correctness measurement above. @@ -260,20 +260,20 @@ use exav_core::yara::{Compiler, Scanner}; let mut c = Compiler::new(); c.add_source(r#"rule demo { strings: $a = "evil" condition: $a }"#)?; let rules = c.build(); -let hit = Scanner::new(&rules).scan(b"...evil...").matching_rules().next().is_some(); +let mut scanner = Scanner::new(&rules); +let hit = scanner.scan(b"...evil...")?.matching_rules().next().is_some(); ``` ## Roadmap -1. **Atom-prefilter performance rewrite** — *in progress*; see the Performance +1. **Atom-prefilter performance rewrite**: *in progress*; see the Performance section. It brings many-rule scanning from ~seconds to ~milliseconds per file. -2. **Deep sharing follow-up** — evolve `logic.rs` (generalize `Node` to YARA +2. **Deep sharing follow-up**: evolve `logic.rs` (generalize `Node` to YARA quantifiers/anchors) and the bytecode VM (undefined-aware dynamic value layer) so YARA and the AV side share them, with the diff harness pinning correctness. -3. **Cheap wins** — `wide` on regexp (now the #1 *coverage* gap); `pe.is_dll`/ - `is_32bit`/`number_of_resources`/`overlay`. -4. **Moderate** — `pe.version_info`/`signatures` (resource + Authenticode +3. **Cheap wins**: `pe.is_dll`/`is_32bit`/`number_of_resources`/`overlay`. +4. **Moderate**: `pe.version_info`/`signatures` (resource + Authenticode parsing; exav-core already has an authenticode module to draw specs from); regexp `imports`/`exports` overloads. -5. Exotic modules (macho/dex) only if a concrete need appears — the data +5. Exotic modules (macho/dex) only if a concrete need appears; the data says they're near-absent in real rulesets. diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index aabee7ef..aab40299 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -10,29 +10,29 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aes" -version = "0.8.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ - "cfg-if", "cipher", + "cpubits", "cpufeatures", ] [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -71,15 +71,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bit-set" version = "0.8.0" @@ -103,9 +94,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bitstream-io" @@ -118,20 +109,20 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] name = "block-padding" -version = "0.3.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -142,9 +133,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" [[package]] name = "byteorder" @@ -152,30 +143,20 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" -[[package]] -name = "bzip2-rs" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beeb59e7e4c811ab37cc73680c798c7a5da77fc9989c62b09138e31ee740f735" -dependencies = [ - "crc32fast", - "tinyvec", -] - [[package]] name = "cbc" -version = "0.1.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" dependencies = [ "cipher", ] [[package]] name = "cc" -version = "1.2.65" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", "jobserver", @@ -196,9 +177,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chrono" @@ -213,20 +194,32 @@ dependencies = [ [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "crypto-common", "inout", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "color_quant" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -239,11 +232,17 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -265,28 +264,30 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] -name = "daachorse" -version = "3.0.2" +name = "ctutils" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99251f238b74cd219a86fe6ea9328308ebb223fcbb5b8eb5aa400b847a41dded" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] [[package]] name = "deflate64" @@ -300,29 +301,30 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3658b90877f637514897c3dcbd628e0d290bcc7fee551a398b3994ebdc62d51e" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", "chrono", "memchr", ] [[package]] name = "des" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" dependencies = [ "cipher", ] [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", + "const-oid", "crypto-common", - "subtle", + "ctutils", ] [[package]] @@ -335,8 +337,7 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" name = "exav-core" version = "0.0.2" dependencies = [ - "aho-corasick", - "daachorse", + "crc32fast", "exav-unpack", "exav-x86", "fancy-regex", @@ -347,6 +348,7 @@ dependencies = [ "regex", "regex-automata", "regex-syntax", + "rmp", "rmp-serde", "rustc-hash", "rustdct", @@ -385,10 +387,8 @@ dependencies = [ "aes", "apfs", "base64", - "bincode", "bitstream-io", "byteorder", - "bzip2-rs", "cbc", "cfb", "constant_time_eq", @@ -414,12 +414,12 @@ dependencies = [ "quick-xml", "ruzstd", "salzweg", - "serde", "sha1", "sha2", "stringprep", "tar", "thiserror", + "tinyvec", "unshield", "xz4rust", "zip", @@ -444,7 +444,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecf18f4817ea4baf95d700eb763642e65f77a94e630867028024b6725848a028" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", "crc", ] @@ -491,18 +491,19 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.9.1", + "zlib-rs", ] [[package]] @@ -513,21 +514,21 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-task", @@ -535,26 +536,15 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi", - "wasip2", ] [[package]] @@ -593,7 +583,7 @@ dependencies = [ "indexmap", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -608,13 +598,22 @@ dependencies = [ [[package]] name = "hmac" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ "digest", ] +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -666,9 +665,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown", @@ -676,19 +675,19 @@ dependencies = [ [[package]] name = "inout" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ "block-padding", - "generic-array", + "hybrid-array", ] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ "getrandom", "libc", @@ -702,9 +701,9 @@ checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -713,15 +712,15 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -735,9 +734,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lzfse_rust" @@ -747,9 +746,9 @@ checksum = "5edeac3ec657925dececc672bdd58fa2212291886ddcb3e08aa46924c78340c6" [[package]] name = "lzma-rust2" -version = "0.16.4" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce716bf1a316f47a280fc76295f6495b5bea4752bca01c3b3885e101b1c23c02" +checksum = "ca93e534d1142d1d0dcca6d25fe302508a5dfb40b302802904577725ea0b695b" [[package]] name = "lznt1" @@ -768,18 +767,18 @@ checksum = "c17f346186eccb574ba5581acefc514f0c70a642db4f96e245034a0a158a7168" [[package]] name = "mail-parser" -version = "0.11.4" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c0e7e0704500930be5b6c629f30d23fd1dde4d1800e138e04b3fa302e64d51" +checksum = "4ec00bda90c6e645a54506c630c2820cd6b1890cfd2b0a169b50f74b2b8c7c86" dependencies = [ "hashify", ] [[package]] name = "md-5" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", "digest", @@ -787,9 +786,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "miniz_oxide" @@ -801,6 +800,16 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "no_std_io2" version = "0.9.4" @@ -821,9 +830,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -845,9 +854,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "pbkdf2" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ "digest", "hmac", @@ -875,7 +884,7 @@ dependencies = [ "crc32fast", "fdeflate", "flate2", - "miniz_oxide", + "miniz_oxide 0.8.9", ] [[package]] @@ -889,9 +898,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -907,24 +916,24 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] [[package]] name = "r-efi" -version = "5.3.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -934,9 +943,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -999,9 +1008,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ruzstd" @@ -1029,20 +1038,20 @@ dependencies = [ [[package]] name = "scroll_derive" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d" +checksum = "e1a36a382ed65dbcc0ab47fd5e9a94112417ccd34560a392ef3b7b0f0ec39148" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1050,29 +1059,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "sha1" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", "cpufeatures", @@ -1081,9 +1090,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures", @@ -1098,9 +1107,9 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "slab" @@ -1126,16 +1135,21 @@ dependencies = [ ] [[package]] -name = "subtle" -version = "2.6.1" +name = "syn" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] [[package]] name = "syn" -version = "2.0.118" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -1154,22 +1168,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -1185,24 +1199,15 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tlsh2" -version = "0.4.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d1ca7b6482798eaf06ef6a4a5e1211e370758888698cfff4a53142441ef5143" +checksum = "7184dfffa0d05e10284df327c6cb52f0d62c5aa6c9a8b9236fa151d4361882f1" [[package]] name = "transpose" @@ -1216,9 +1221,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" [[package]] name = "typed-path" @@ -1240,9 +1245,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-normalization" @@ -1271,34 +1276,19 @@ dependencies = [ [[package]] name = "uuid" -version = "1.23.4" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -1309,9 +1299,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1319,22 +1309,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] @@ -1376,7 +1366,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1387,7 +1377,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1414,17 +1404,11 @@ dependencies = [ "windows-link", ] -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - [[package]] name = "xz4rust" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1053c1562c16e3760559c390d3b690ae6b524c200eb9a7ddb5d43c77d000832d" +checksum = "606930b71bad6379f366994e4e6252a3462fa28bfc3695c33b07bef8f967d6d6" [[package]] name = "zip" @@ -1432,9 +1416,16 @@ version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ + "bitstream-io", "crc32fast", "flate2", "indexmap", "memchr", "typed-path", ] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" diff --git a/fuzz/fuzz_targets/ndb_compile.rs b/fuzz/fuzz_targets/ndb_compile.rs index 90bebe9c..d8997891 100644 --- a/fuzz/fuzz_targets/ndb_compile.rs +++ b/fuzz/fuzz_targets/ndb_compile.rs @@ -2,7 +2,7 @@ //! Fuzz the NDB/LDB signature compiler: parse arbitrary signature text through //! `EngineBuilder::add_ndb()` / `add_ldb()`, then build the engine. Catches //! panics in `parse_elems`, `compile_body`, `pick_anchor`, `parse_gap`, -//! `parse_alt`, AC automaton construction, and all the hex-wildcard parsing +//! `parse_alt`, the anchor index build, and all the hex-wildcard parsing //! codepaths that no other fuzz target reaches. use libfuzzer_sys::fuzz_target; use exav_core::engine::EngineBuilder; diff --git a/scripts/check-links.mjs b/scripts/check-links.mjs new file mode 100644 index 00000000..7c606fa3 --- /dev/null +++ b/scripts/check-links.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +// Check every internal link of the built docs site (www/dist): the page it +// names must exist, and so must the `#anchor` on it. +// +// Run on the rendered HTML rather than on the Markdown, so an anchor is checked +// against the id Starlight actually generated for the heading, not against a +// guess at its slug rule. No dependencies: the build output is plain HTML. +// +// USAGE: node scripts/check-links.mjs [DIST_DIR] (default: www/dist) + +import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repo = join(fileURLToPath(import.meta.url), '..', '..'); +const dist = process.argv[2] ?? join(repo, 'www', 'dist'); +if (!existsSync(dist)) { + console.error(`error: ${dist} not found; build the site first (npm run build in www/)`); + process.exit(2); +} + +function htmlFiles(dir) { + return readdirSync(dir, { withFileTypes: true }).flatMap((e) => { + const p = join(dir, e.name); + if (e.isDirectory()) return htmlFiles(p); + return e.name.endsWith('.html') ? [p] : []; + }); +} + +// `dist/a/b/index.html` is served as `/a/b/`, any other file under its own name. +function urlOf(file) { + const rel = relative(dist, file).split(sep).join('/'); + return '/' + (rel.endsWith('index.html') ? rel.slice(0, -'index.html'.length) : rel); +} + +// `&` last, so `&lt;` stays `<`. +const unescape = (s) => + s.replace(/"/g, '"').replace(/'/g, "'").replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&'); + +// A malformed `%` escape is a broken link, not a crash: `null`. +const decode = (s) => { + try { + return decodeURIComponent(s); + } catch { + return null; + } +}; + +const pages = new Map(); // url -> { ids, hrefs } +for (const file of htmlFiles(dist)) { + const html = readFileSync(file, 'utf8'); + const ids = new Set([...html.matchAll(/\sid="([^"]*)"/g)].map((m) => unescape(m[1]))); + const hrefs = [...html.matchAll(/]*?)?\shref="([^"]*)"/g)].map((m) => unescape(m[1])); + pages.set(urlOf(file), { ids, hrefs }); +} + +// The page a path lands on: `/x/` and `/x` both mean `/x/index.html`. +function target(pathname) { + if (pages.has(pathname)) return pathname; + const dir = pathname.endsWith('/') ? pathname : pathname + '/'; + if (pages.has(dir)) return dir; + return null; +} + +// A non-page file (an image, a stylesheet) only has to exist. +function assetExists(pathname) { + const path = decode(pathname); + if (path === null) return false; + const p = join(dist, path); + return existsSync(p) && statSync(p).isFile(); +} + +const broken = []; +for (const [from, { hrefs }] of pages) { + for (const href of hrefs) { + if (!href || /^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith('//')) continue; + const url = new URL(href, 'https://site.invalid' + from); + const page = target(url.pathname); + if (!page) { + if (!assetExists(url.pathname)) broken.push(`${from}: ${href} (no such page)`); + continue; + } + const anchor = decode(url.hash.slice(1)); + if (anchor === null) { + broken.push(`${from}: ${href} (malformed escape in the anchor)`); + } else if (anchor && !pages.get(page).ids.has(anchor)) { + broken.push(`${from}: ${href} (no #${anchor} on ${page})`); + } + } +} + +if (broken.length) { + console.error(`${broken.length} broken internal link(s):`); + for (const b of [...new Set(broken)].sort()) console.error(` ${b}`); + process.exit(1); +} +console.log(`checked internal links across ${pages.size} pages: all resolve`); diff --git a/scripts/difftest.sh b/scripts/difftest.sh index 8496850c..64f45c58 100755 --- a/scripts/difftest.sh +++ b/scripts/difftest.sh @@ -311,27 +311,14 @@ HEUR # ── Phase 2: exav ──────────────────────────────────────────────────────────── phase_exav() { [ -x "$EXAV" ] || { echo "FATAL: $EXAV not built (cargo build --release -p exav)"; exit 1; } - # A rebuilt exav can bump the database format, making a stale database - # unloadable, but `--build-db` needs several GB of RAM, so do NOT rebuild - # merely because the binary is newer. Rebuild when it is missing or fails to - # LOAD, which is the failure that actually matters. - local need=0 - if [ ! -f "$DB" ]; then need=1 - elif [ "$EXAV" -nt "$DB" ]; then - mkdir -p "$SOCK_DIR"; : > "$SOCK_DIR/.probe" - "$EXAV" -d "$DB" "$SOCK_DIR/.probe" >/dev/null 2>&1 \ - && { say "phase 2: database older than the binary but loads, reusing"; touch "$DB"; } \ - || need=1 - fi - if [ "$need" = 1 ]; then + # A database older than the binary is rebuilt: one that still loads may + # still hold what an older build made of the signatures, and the run would + # measure that instead of this binary. + if [ ! -f "$DB" ] || [ "$EXAV" -nt "$DB" ]; then need_signatures say "phase 2: building the exav database from $DBDIR" - local avail shard="" - avail=$(awk '/MemAvailable/{print $2}' /proc/meminfo 2>/dev/null || echo 0) - [ "$avail" -gt 0 ] && [ "$avail" -lt 8000000 ] && shard="--build-shard-bytes ${BUILD_SHARD_BYTES:-1G}" - # shellcheck disable=SC2086 - "$EXAV" -d "$DBDIR" --build-db "$DB" $shard || { - echo "FATAL: exav database build failed (it needs several GB of FREE RAM)"; exit 1; } + "$EXAV" -d "$DBDIR" --build-db "$DB" || { + echo "FATAL: exav database build failed"; exit 1; } fi # exav's results are NOT cached across binaries (that is the entire reason for diff --git a/scripts/format-chain-fuzz.py b/scripts/format-chain-fuzz.py index 4a02ac4b..1b018373 100755 --- a/scripts/format-chain-fuzz.py +++ b/scripts/format-chain-fuzz.py @@ -9,7 +9,7 @@ It has already paid for itself twice: * a two-byte HFS+ signature scanned across 64 KiB matched compressed data about - one time in twenty, and a false hit is not a wasted check — `detect` answers + one time in twenty, and a false hit is not a wasted check: `detect` answers `Dmg`, so the file's real format is never tried; * a FAT boot sector read as a partition table produced a "partition" identical to the whole image, which re-detected the same way until the recursion budget @@ -23,8 +23,9 @@ Usage: scripts/format-chain-fuzz.py [trials] [chain-length] [seed] Needs the same tools as scripts/make-matryoshka.sh, on PATH. Set TOOLS_PREFIX if -they live in a staging root rather than the system paths, and EXAV to point at a -built binary (default: target/debug/exav). +they live in a staging root rather than the system paths, EXAV to point at a +built binary (default: target/debug/exav), and DB at a signature database +(default: eicar.ndb). """ import os, random, subprocess, sys, shutil, tempfile @@ -38,7 +39,7 @@ f"{PREFIX}/{d}" for d in ("usr/lib", "lib") ) + ":" + ENV.get("LD_LIBRARY_PATH", "") EXAV = os.environ.get("EXAV", "target/debug/exav") -DB = os.environ.get("EXAV_DB", "eicar.ndb") +DB = os.environ.get("DB", "eicar.ndb") # Reversed, and flipped back at run time, so the 68-byte sequence is not stored # anywhere in this tree. See `exav_unpack::eicar` for why that matters: a file @@ -186,7 +187,7 @@ def main(): verdict = scan(final, cwd, len(order) + 10) # Only a *clean* verdict is a bug. A resource limit is the honest # answer for a chain that inflates a 68-byte payload through several - # megabytes of disk image and then recompresses it — the ratio guard + # megabytes of disk image and then recompresses it: the ratio guard # firing there is the scanner working, not failing. if "FOUND" in verdict: state = "ok " diff --git a/scripts/test-www.sh b/scripts/test-www.sh index 7cd7db90..a08a6e17 100755 --- a/scripts/test-www.sh +++ b/scripts/test-www.sh @@ -3,12 +3,13 @@ # # WHY: the docs are the project's single authoritative answer on format coverage, # CLI flags and ClamAV parity, and they are edited in the same commits as the -# code they describe. A build failure — a broken internal link, a malformed -# frontmatter block, a sidebar entry pointing at a page that no longer exists — +# code they describe. A build failure (a malformed frontmatter block, a sidebar +# entry pointing at a page that no longer exists, a page that does not render) # is a documentation bug that reaches readers, and nothing else in CI catches it. # -# `astro check` is the type/diagnostic pass; `astro build` is the one that -# resolves every internal link and renders every page. +# `astro check` is the type/diagnostic pass; `astro build` renders every page. +# Neither checks links written inside a page, so check-links.mjs then resolves +# every internal link and `#anchor` against the built HTML. # # USAGE: scripts/test-www.sh # REQUIREMENTS: node + npm. @@ -28,3 +29,4 @@ fi npx astro check npm run build +node "$REPO/scripts/check-links.mjs" "$REPO/www/dist" diff --git a/www/astro.config.mjs b/www/astro.config.mjs index bd6c8e90..15acda56 100644 --- a/www/astro.config.mjs +++ b/www/astro.config.mjs @@ -17,7 +17,7 @@ export default defineConfig({ starlight({ title: 'exav', tagline: 'A fast, memory-safe malware scanner written in Rust.', - // Mark only — the "exav" wordmark next to it is rendered as real text by + // Mark only: the "exav" wordmark next to it is rendered as real text by // Starlight, so it uses the site font rather than whatever the viewer's // OS happens to substitute inside an -loaded SVG. logo: { @@ -45,15 +45,15 @@ export default defineConfig({ { label: 'Usage guides', items: [ + { label: 'Migrating from ClamAV', slug: 'guides/migrating-from-clamav' }, + { label: 'Signatures', slug: 'guides/signatures' }, { label: 'Scanning', slug: 'guides/scanning' }, + { label: 'Prebuilt database', slug: 'guides/prebuilt-database' }, { label: 'Daemon mode', slug: 'guides/daemon' }, - { label: 'Sizing a server', slug: 'guides/sizing' }, { label: 'ICAP server', slug: 'guides/icap' }, { label: 'Docker', slug: 'guides/docker' }, - { label: 'Signatures', slug: 'guides/signatures' }, - { label: 'Prebuilt database', slug: 'guides/prebuilt-database' }, + { label: 'Sizing a server', slug: 'guides/sizing' }, { label: 'YARA rules', slug: 'guides/yara' }, - { label: 'Migrating from ClamAV', slug: 'guides/migrating-from-clamav' }, { label: 'WASM sandbox', slug: 'guides/wasm-sandbox' }, { label: 'Rust library usage', slug: 'guides/library-usage' }, { label: 'Troubleshooting & FAQ', slug: 'guides/troubleshooting' }, diff --git a/www/src/content/docs/concepts/architecture.md b/www/src/content/docs/concepts/architecture.md index bee2cba5..188452aa 100644 --- a/www/src/content/docs/concepts/architecture.md +++ b/www/src/content/docs/concepts/architecture.md @@ -4,7 +4,8 @@ description: How exav is organized, the crate layout, and how each ClamAV signat --- exav is a small set of focused crates around one scanning engine. This is the -high-level picture; implementation detail lives in the repository's `docs/`. +high-level picture; implementation detail lives in the repository's +[`docs/`](https://github.com/sylvinus/exav/tree/main/docs). ## Input @@ -78,15 +79,9 @@ about the file: a path it cannot open, a database that will not load, or a source that fails part way through a scan, whose unread bytes are not taken for the end of the file (see [Verdicts & exit codes](/reference/verdicts/)). -### Content-based typing - -File type is decided by magic bytes, not by extension: an executable renamed -`.jpg` is still typed and scanned as an executable. - -### Recursive unpacking, bounded - -See [Archive extraction](/concepts/archive-extraction/) for the budgets every -extraction runs under and what a member exav cannot read becomes. +File type comes from the content, not the extension: an executable renamed +`.jpg` is still scanned as an executable. The budgets every extraction runs +under are in [Archive extraction](/concepts/archive-extraction/). ## How the crates compose @@ -94,7 +89,8 @@ exav is a small Cargo workspace, not one binary. Two front ends drive the engine, `exav-core`: the CLI (which is also the daemon) and the WASI build. Two more use only the extractor, `exav-unpack`: the archive grep, and the WebAssembly bindings published to npm (built outside the workspace, with their -own profile). `exav-update`, enabled by the `http` feature, sits to the side, +own profile). `exav-update`, enabled by the `http-update` feature (part of +`http`), sits to the side, feeding fresh signatures out of band. Every arrow points down. Nothing below calls anything above it, which is what @@ -107,7 +103,7 @@ lets the extraction crates be taken on their own. depend on exav-core; exav-core, exav-grep and exav-unpack-wasm all depend on exav-unpack, which depends on exav-pe-emu, which depends on exav-x86. exav-core also depends on exav-x86 directly, for the bytecode disassembly - API. exav-update, optional behind the http feature, hangs off exav alone and + API. exav-update, optional behind the http-update feature, hangs off exav alone and feeds signature files out of band. @@ -152,7 +148,7 @@ lets the extraction crates be taken on their own. #![forbid(unsafe_code)] x86-32 sandbox decoder - optional: http feature + optional: http-update feature bytecode disasm runs a packer's own stub to recover the image it rebuilds @@ -193,27 +189,35 @@ than one matcher: | Source | Runtime structure | Matching | |---|---|---| -| `.ndb` bodies + `.ldb` literal subsignatures | a set of **Aho-Corasick automata**, one per (target type, case) pair, keyed on a literal anchor per body | an automaton hit fans out to every body sharing that anchor; each candidate is then verified (wildcards / gaps / nibbles / alternation / offset / nocase) | -| `.ldb` PCRE subsignatures | regexes compiled lazily, gated by the trigger expression | linear-time `regex`; patterns with backreferences use a backtracking engine under a step bound | +| `.ndb` / `.db` bodies + `.ldb` body subsignatures | an **index of literal anchors**, one per body, each tagged with its (target type, case) partition; a body pinned near a fixed offset is kept out of it | the index is looked up at each position of the object, and a hit fans out to every body sharing that anchor; each candidate is then verified (wildcards / gaps / nibbles / alternation / offset / nocase). A pinned body is checked where it can start | +| `.ldb` PCRE subsignatures | regexes compiled lazily, gated by the trigger expression | linear-time `regex`; patterns with lookaround or backreferences use a backtracking engine under a step bound | | `.hsb` / `.hdb` | a size-keyed hash table | whole-file digest lookup | | `.mdb` / `.msb` | a section-hash table | per-PE-section digest lookup | | `.cdb` | container-metadata matchers | matched on archive members (name/size/encryption/position) | | `.imp` | a size-constrained import-hash map | PE imphash lookup | -| `.cbc` | a [sandboxed bytecode interpreter](/concepts/bytecode-sandbox/) (no JIT) | trigger-gated programs run on extracted buffers | +| `.fdb` | an imphash map and a list of TLSH digests | imphash lookup; TLSH distance to each digest, under the signature's threshold | +| `.cbc` | a [sandboxed bytecode interpreter](/concepts/bytecode-sandbox/) (no JIT) | a program runs on an object when its trigger matches; a hook program on every file of its type | | `.yar` / `.yara` | a [native YARA engine](/guides/yara/) | near-full YARA, no runtime codegen | -So the engine is a set of Aho-Corasick automata (fed by `.ndb` and `.ldb` literal -subsignatures), several cheap hash tables, lazy regexes and interpreters. Almost -all of the memory cost is the automata; see the -[prebuilt database](/guides/prebuilt-database/). +The other formats (icons, certificates, file-type magic, allowlists, ignore +lists, phishing and passwords) are listed in +[Signatures](/guides/signatures/#formats-exav-loads). So the engine is an +anchor index, hash tables, lazy regexes and interpreters. Most of the memory is +the compiled bodies, the index and the hash tables; see the +[prebuilt database](/guides/prebuilt-database/#what-it-costs). ## How a body signature is matched +A body is hex with wildcards (`??`, nibbles such as `a?`), gaps (`{10-}`) and +alternations, optionally tied to an offset: `0:` (the start), `EOF-n`, `EP+n` +(the entry point) or `Sn+n` (section `n`). A `::f` modifier asks for a match +on word boundaries. + From anchor hit to detection - An object's bytes, and a lowercased view of them for - case-insensitive anchors, go through the partitions whose target fits the - file type. One Aho-Corasick sweep reports every anchor hit as a group and an + An object's bytes go through the partitions whose target + fits the file type. One sweep looks up the anchor index at each position, on + the bytes folded to lowercase, and reports every anchor hit as a group and an offset. Each hit fans out to every body sharing the anchor, and each is verified around the hit. A verified .ndb body is a detection. A verified .ldb subsignature is counted, and the logical expressions of the .ldb signatures @@ -236,14 +240,14 @@ all of the memory cost is the automata; see the object partitions, by (target, case) - one Aho-Corasick sweep + one sweep over the anchor index fan out, then verify each body .ndb body verified .ldb subsignature verified logical expression - its bytes, and a lowercased view + its bytes, read once only those whose target fits the file type run every anchor at once: hits as (group, offset) every body sharing the anchor: prefix back, suffix @@ -267,27 +271,39 @@ all of the memory cost is the automata; see the An anchor is a literal run each body must contain; everything else in the body (wildcards, nibbles, alternations, gaps, the offset) is checked only where an -anchor hit. Anchors are split by target type and case, so a scan runs only the -automata that can matter for the object's type, and a PE never walks the -HTML-only anchors. The automata that apply advance together over one read of -the object, whatever the number of signatures in them, so the processor waits on -their memory at the same time; case-insensitive ones read each byte lowercased. +anchor hit. Every anchor is in one index, tagged with its target type and case, +and a lookup reports only the partitions that can matter for the object's type, +so a PE never verifies an HTML-only body. An anchor is indexed by the bytes of +it the signature set shares least: the whole anchor at one to three bytes, four +of them at four or five, and past that two overlapping six-byte windows, of +which only the one at an even position is looked up. Filters small enough to +stay in cache turn away most positions before the tables are read. Every anchor is looked up on the bytes +folded to lowercase, and a case-sensitive one is then compared exactly; the +object is copied lowercased only when a case-insensitive body has to be +verified. The object is read once, whatever the number of signatures. + +A body whose offset pins it to a window of starts at most 4096 bytes wide, +counted from the start, the end, the entry point or a section (`0:`, `EP+0:`, +`100,50:`), is not +indexed: it is checked at the places it can start, and costs nothing elsewhere +in the object. Verification never backtracks: a body is a sequence of elements, and the positions it can have reached after each one are kept as a set of intervals, so a `{10-}` gap widens an interval instead of multiplying attempts. Backtracking is left to PCRE subsignatures with lookaround or backreferences, which are not regular and so cannot run on the linear engine: they run on a backtracking -engine under a step bound, after a linear prefilter (the pattern with its -lookarounds removed, which matches at least as much) has ruled most objects out. +engine under a step bound. For lookaround, a linear prefilter (the pattern with +its lookarounds removed, which matches at least as much) rules most objects out +first. ### How far a check reads around a hit How far a check reads around an anchor hit An object drawn as a strip of 64 KiB blocks. The sweep reads - it in order, a chunk at a time, carrying the automaton's state across chunk - seams. At an anchor hit, verification reads the body's prefix backward from + it in order, a chunk at a time, looking up the anchors that straddle a chunk + seam from the bytes on each side. At an anchor hit, verification reads the body's prefix backward from the anchor and its suffix forward, as far as the body needs; a gap with no upper bound can reach the end of the object. A block no longer in the cache is read again. The cache bounds what is held, not how far a check reaches. @@ -322,7 +338,7 @@ lookarounds removed, which matches at least as much) has ruled most objects out. - sweep: in order, a chunk at a time; the automaton's state carries across seams, so nothing is read twice + sweep: in order, a chunk at a time; an anchor across a seam is looked up there, so nothing is read twice prefix: read backward from the anchor suffix: read forward a {n-} gap can reach the end of the object @@ -333,9 +349,10 @@ lookarounds removed, which matches at least as much) has ruled most objects out. -Neither the sweep nor a check has a window to fall out of. The automata read -the object once, in order, and their state carries from one chunk to the next, -so an anchor split across a seam is found without re-reading anything. A check +Neither the sweep nor a check has a window to fall out of. The sweep reads the +object once, in order, and looks up the few positions before a seam from the +bytes on both sides of it, so an anchor split across a seam is found without +re-reading the chunk. A check asks for the bytes it needs at their offset, before or after the hit, from memory or, past `--max-object-bytes`, from the [block cache](/concepts/streaming-memory/#past-it-through-a-block-cache). @@ -345,154 +362,14 @@ YARA strings) run as lazy DFAs stepped through it, forward to find where a match ends and backward from there to find where it starts. A PCRE with lookaround runs its lookaround-free superset that way: when that finds nothing, the PCRE cannot match either; when it finds something, the scan is `LIMITS-EXCEEDED` -unless something else is found, as it is when a lazy DFA meets a construct it -cannot follow (a Unicode word boundary next to a non-ASCII byte). - -## Reading an object not held in memory - - - One source, read through a block cache - A file, an HTTP range reader or a spill file is read through a - block cache of 64 KiB blocks holding 8 MiB. Every check asks it for bytes at - an offset: the signature sweep a chunk at a time in order, verification and - the container walk at any offset, the regexes stepped through, the hashes, - typing's search and carving in one shared pass. The checks that parse an object whole run only when it fits within - --max-object-bytes. - - - - - - - SOURCES - WHAT READS IT - - - - - - - - - - - - - - - file - HTTP range - spill file - BlockCache - signature sweep - verification - regexes - hashes · typing · carving - container walk - whole-object checks - - - past --max-object-bytes - a request per block - a large member or view - 64 KiB blocks, 8 MiB held, - least recently used dropped; - bytes at any offset, or in - 1 MiB chunks in order - chunks in order, one pass - windows at any offset - lazy DFAs stepped through - one shared pass; start and end - headers and members, by offset - held whole only: PE, RAR, OLE - - - - - - - - - - - - +unless something else is found. The same holds for a PCRE with a backreference +whose trigger holds, and when a lazy DFA meets a construct it cannot follow (a +Unicode word boundary next to a non-ASCII byte). -What the cache holds, and which checks parse an object whole and so do not run -past `--max-object-bytes`, is in -[Streaming & memory](/concepts/streaming-memory/#past-it-through-a-block-cache). - -## Where the spill fits - - - The two uses of spill files - Before a scan, a stream (stdin, INSTREAM, ICAP) is held in - memory up to --spill-threshold-bytes and then in a temporary file up to - --max-spill-bytes, and becomes a seekable source for the scan. Inside a scan, - a streamed archive member that decodes past --max-object-bytes, and the normalised - text views of a large text file, are written to spill files and read back - through a block cache. A stream past its budget, or past the threshold with - --spill-dir off, has what was held scanned; with --spill-dir off, such a - member or view is not scanned. Either way a scan that finds nothing is - LIMITS-EXCEEDED. - - - - - - - BEFORE THE SCAN: A STREAM - INSIDE THE SCAN: WHAT IT MAKES - - - - - - - - - - - - - stdin · INSTREAM - RAM - temp file - the scan - member past the limit - text views, large file - spill file - scanned from there - - - ICAP bodies - --spill-threshold-bytes - --max-spill-bytes - a seekable source - decoded past --max-object-bytes - HTML, text, JS normalised - written as it is made - through a block cache - - - - - - - - - - - Past --max-spill-bytes, or past the threshold with --spill-dir off, what was held is - scanned and the stream is LIMITS-EXCEEDED unless that finds something. - --max-total-spill-bytes bounds every spill file of a process together. - - +## Memory and spill files -The library never writes to disk itself: the host hands a scan somewhere to -spill (the `exav` binary passes its `--spill-dir` files and budgets). Only a -member decoded as it is read can spill; one decoded whole is bounded by -`--max-object-bytes` (see [Archive members](/concepts/streaming-memory/#archive-members)). -Without a spill, a member that decodes past `--max-object-bytes` is not scanned, -the normalised views of a text file that large are skipped, and a scan that -found nothing is `LIMITS-EXCEEDED` rather than `OK`. +An object larger than `--max-object-bytes` is read through a block cache rather +than held, and a member or text view too large to hold is written to a spill +file the host provides. The library never writes to disk itself. +[Streaming & memory](/concepts/streaming-memory/) shows what reads through the +cache, which checks need an object whole, and where spill files fit. diff --git a/www/src/content/docs/concepts/archive-extraction.md b/www/src/content/docs/concepts/archive-extraction.md index 791d7e69..44b69b7c 100644 --- a/www/src/content/docs/concepts/archive-extraction.md +++ b/www/src/content/docs/concepts/archive-extraction.md @@ -26,9 +26,8 @@ and codec those extractors open, exav's status against each, and [the gap list](/reference/formats/#the-complete-gap-list) of what is still missing. -Every input is extracted through a seekable source: a file, an HTTP range reader -(for a format with an index, such as a ZIP's central directory, the walk fetches -only the index and the members it reads), or a stream buffered first (see +Every input is extracted through a seekable source: a file, an HTTP range +reader, or a stream buffered first (see [Streaming & memory](/concepts/streaming-memory/)). ## The shape of the unpacker @@ -91,7 +90,8 @@ of it. `walk()` then hands the members to a visitor one at a time. A format whos decoder reads forward (ZIP, tar, CAB, ISO, the single-stream compressors, …) produces each member as a reader that decodes as it is read, so neither the container nor the member is resident. A format whose decoder needs random access -over the whole container (7z, RAR, OLE, PDF, CHM, the disk images, …) is read +over the whole container (7z, RAR, OLE, PDF, CHM, the virtual disks and +filesystem images, …) is read whole, bounded before it is allocated, up to `--max-object-bytes`; past it the container is `LIMITS-EXCEEDED`. 7z's members still stream once the container is in memory. Which it is is a property of the decoder, not of how exav was @@ -108,24 +108,20 @@ unread costs nothing for it: a ZIP is listed from its central directory, and a tar's headers are read in turn with the data between them skipped, so listing decodes nothing. -**A ZIP has more members than its directory admits to.** The walk also scans the -bytes the central directory does not claim (see below), after the directory's -own members. - ## Bounded, budget before allocation Every extraction runs under one `Budget`, reserved before a member is read rather than checked after: -| Limit | Bounds | -|---|---| -| output bytes | what formats decoded whole produce across the tree | -| per-member size | a member decoded whole (a streamed member is bounded by the ratio and matcher budgets) | -| compression ratio | output ÷ input, the decompression-bomb guard | -| file count | members across the archive and everything nested | -| recursion depth | archives inside archives | -| matcher bytes | bytes fed to the matcher across the tree, a CPU bound | -| emulation steps | instructions the PE unpacking emulator runs across the tree | +| Limit | Bounds | Default | +|---|---|---| +| output bytes | what formats decoded whole produce across the tree | 1 GiB, within half of `--max-process-bytes` | +| per-member size | a member decoded whole (a streamed member is bounded by the ratio and matcher budgets) | `--max-object-bytes` 256M | +| compression ratio | output ÷ input, the decompression-bomb guard | 1000 | +| file count | members across the archive and everything nested | `--max-members` 100000 | +| recursion depth | archives inside archives | `--max-unpack-depth` 16 | +| matcher bytes | bytes fed to the matcher across the tree, a CPU bound | `--max-matcher-bytes` 10G | +| emulation steps | instructions the PE unpacking emulator runs across the tree | `--max-pe-emulation-steps` 1,000,000,000 | Reserving first keeps the peak bounded: a "1 GB from 4 KB" member never allocates 1 GB and then gets rejected. @@ -148,11 +144,17 @@ local header and data in the file. Plenty of extractors still write it out. exav does dual indexing: after the central-directory pass it scans the raw bytes for local headers the directory did not cover, and extracts those too. The same path is the salvage route when the central directory is corrupt, forged or -truncated. +truncated. On a ZIP read from its source rather than held in memory, the search +covers at most 16 MiB of unclaimed bytes, and a search cut short there is +reported, not taken as complete. `PK\x03\x04` is four bytes and occurs by chance in ordinary binaries, so a -credibility check (version, reserved flag bits, a real compression method, a -NUL-free path within the length limit) decides what counts as a member. +credibility check decides what counts as a member: version at most 6.3, no +reserved flag bits, a non-empty path within the length limit and free of NULs, +and a known compression method or, failing that, a readable UTF-8 name. A +header with an unknown method and a clean name is still a member, reported +unscannable, because a packer can stamp a nonsense method on a member to hide +it from readers that check the method. ISO images have the same shape: a CD image carries several volume descriptors, each with its own directory tree over the same sectors. A malicious ISO lists its @@ -200,16 +202,12 @@ is caught by name even when its content is unreadable. ## Encryption -exav decrypts what it can: ZIP (ZipCrypto and WinZip AES), 7z AES-256 including -encrypted headers, encrypted DMG, PDF, and Office (legacy XLS RC4 and XOR -obfuscation, OOXML standard and agile AES). - -Two defaults need no configuration. Office documents are tried with -`VelvetSweatshop`, Excel's built-in default password, which opens without a -prompt for the victim but looks opaque to a scanner that stops at "encrypted". -ZIPs are tried against a short built-in list of passwords malware distribution -uses (`infected`, `virus`, …). `--passwords` and ClamAV `.pwdb` databases add -your own. +exav decrypts ZIP, 7z, DMG, PDF, Office and garbled ARJ members; the schemes +are listed in [Supported formats](/reference/formats/#encryption-support). +Office documents are tried with `VelvetSweatshop`, Excel's built-in default +password, and ZIPs with a short list of passwords malware distribution uses +(`infected`, `virus`, …), so neither needs configuration. +`--passwords` and ClamAV `.pwdb` databases add your own. An archive that stays encrypted is still a signal: "this member is encrypted" can be matched by `.cdb` signatures, and `--partial-as password-protected=found` diff --git a/www/src/content/docs/concepts/bytecode-sandbox.md b/www/src/content/docs/concepts/bytecode-sandbox.md index 80c3b654..7537684c 100644 --- a/www/src/content/docs/concepts/bytecode-sandbox.md +++ b/www/src/content/docs/concepts/bytecode-sandbox.md @@ -36,12 +36,13 @@ LLVM JIT. An attacker needs to get a `.cbc` file loaded, from a third-party feed for instance: - **[CVE-2020-37167](https://www.cve.org/CVERecord?id=CVE-2020-37167):** weak - validation in the interpreter's function-name processing (CWE-94) lets a - crafted `.cbc` corrupt the VM's memory and chain to arbitrary code execution - in the scanner process, with a - [public ROP exploit](https://www.exploit-db.com/exploits/47687) (exploit-db - 47687, the `bytecode_vm` sandbox escape). CVSS 8.4; ClamAV before 0.103.0. - Also tracked by [Ubuntu](https://ubuntu.com/security/CVE-2020-37167). + validation of function names in the ClamBC bytecode interpreter lets a + crafted `.cbc` manipulate the interpreter; the advisory rates it CVSS 8.4 and + lists ClamAV before 0.103.0. A + [public exploit](https://www.exploit-db.com/exploits/47687) (exploit-db 47687, + `bytecode_vm`) reaches code execution through the bytecode VM, demonstrated + with the `clambc` tool. Also tracked by + [Ubuntu](https://ubuntu.com/security/CVE-2020-37167). - The optional LLVM JIT generated and ran native code from bytecode, a large attack surface Cisco has been moving away from. @@ -49,7 +50,7 @@ exav removes these failure modes by construction: | Risk in ClamAV's C VM | exav | |---|---| -| Memory corruption in the VM (the CVE-2020-37167 class) | Safe Rust with no `unsafe`: every access is a bounds-checked slice, and an out-of-range index panics into isolation instead of corrupting memory | +| Memory corruption in the VM (the CVE-2020-37167 class) | Safe Rust with no `unsafe`: every access to program memory is bounds-checked, so a bad index cannot reach memory outside it | | Native code generation from bytecode | No JIT: interpretation only | | A program escaping the sandbox (syscalls, host memory) | The program sees only bounded buffers and a fixed read-only file API: no syscalls, no host pointers | | A runaway program | An instruction budget, a scratch-memory cap and a call-depth limit | @@ -57,10 +58,24 @@ exav removes these failure modes by construction: | A malformed or hostile `.cbc` | A fallible parser with no panics; a program not fully understood is never executed | The worst a hostile or buggy `.cbc` can do to exav is be skipped or be stopped. -A program stopped before it finished (out of instructions, or a VM panic) makes -the scan `LIMITS-EXCEEDED` unless something is found, since what it would have -found is unknown. exav can run these programs with the code-execution risk -removed, for anyone who disables ClamAV bytecode for safety. +A program exav stops before it finished (out of instructions or call depth, an +opcode or API exav does not model, more than 256 MiB written to one extracted +file, a read of the PE header data `__clambc_pedata` on a PE that exav has no +header data for, or a VM panic) makes the scan `LIMITS-EXCEEDED` unless +something is found, since what it would have found is unknown. A program that +fails on its own (an out-of-bounds access, a division by zero, an abort) ends +there as it would in ClamAV, and its detection is discarded; so is that of one +that asks `disasm_x86` about bytes the decoder does not know, since exav cannot +tell an invalid instruction from one outside its decoder. Except after a VM +panic, what the program wrote before it stopped is still scanned, as ClamAV +scans the output of a run that failed. A `write` that would take one extracted +file past `--max-object-bytes`, or the output past what is left of +`--max-matcher-bytes`, fails with -1 as ClamAV's does at its own limits: the +program carries on, and the scan is `LIMITS-EXCEEDED` unless something is +found. On a file +that is not a PE, `__clambc_pedata` reads as zeros, as in ClamAV. exav can run +these programs with the code-execution risk removed, for anyone who disables +ClamAV bytecode for safety. ## Trigger-gated execution, and the API subset @@ -69,10 +84,10 @@ only when its logical-signature trigger matches. The few programs with no trigger (two PDF hooks in the current `bytecode.cvd`) run on every file of the type they hook. -exav implements about 40 of the 107 host API functions. The rest are fail-safe +exav implements 34 of the 107 host API functions. The rest are fail-safe stubs rather than a reason to skip a program: a call returns a conservative value and is recorded, so the program still runs to the end. A stub that fabricates a -result is recorded in the run's outcome and, with the `EXAV_BC_WARN` +result is recorded in the run's outcome and, with the `EXAV_DEBUG_BC_WARN` environment variable set, printed to stderr. On a live `bytecode.cvd`, every program parses and runs to completion with no unsupported opcode, and in practice programs rarely reach a stub: a `.cbc` header declares a @@ -84,5 +99,4 @@ opcode, and in practice programs rarely reach a stub: a `.cbc` header declares a Bytecode programs are a tiny fraction of the signatures, and uneven in value: the few unpackers matter most (missing one weakens detection across many packed samples), the polymorphic-family checks each cover a whole family, and a long -tail are legacy single-CVE checks. The detection difference is modest, which is -why the memory-safe sandbox is the main gain. +tail are legacy single-CVE checks. diff --git a/www/src/content/docs/concepts/design-principles.md b/www/src/content/docs/concepts/design-principles.md index 1d489958..6f55bb46 100644 --- a/www/src/content/docs/concepts/design-principles.md +++ b/www/src/content/docs/concepts/design-principles.md @@ -18,10 +18,7 @@ that makes the scanner stop early, for any reason, is reported instead of being folded into `OK`. ClamAV, for example, reads a file over about 2 GB, scans none of it, and reports -`OK`. exav closes that gap and every other way a scan can stop early. The one -exception is asked for by name: `--clamav-compat`, a preset for differential -testing, answers an incomplete scan `OK` as ClamAV does, and logs every such -object on stderr so the pass stays visible. +`OK`. exav closes that gap and every other way a scan can stop early. Three rules follow: @@ -35,7 +32,9 @@ Three rules follow: 3. **Not fully scanned is never clean, whatever the cause.** That covers external limits and exav's own work bounds: the budget that stops a pathological wildcard search, a PE emulation that runs out of instructions - (`--max-pe-emulation-steps`), a bytecode program that runs out of steps. + (`--max-pe-emulation-steps`), a bytecode program that runs out of steps or + reaches an opcode or API exav does not model. A bytecode program that fails + on its own (an out-of-bounds access) is discarded, as ClamAV discards it. A scan that did not complete resolves to `LIMITS-EXCEEDED`, `UNSCANNABLE` or `PASSWORD-PROTECTED`. See [Verdicts & exit codes](/reference/verdicts/) for what @@ -46,9 +45,11 @@ each means and how it is reported. `clamscan` returns `OK` after bounding its own work (its `alert-exceeds-max` and `alert-encrypted` heuristics are off by default); exav does not. It is the main behavioral difference a migrating user has to plan for, along with the flags -(see [Migrating from ClamAV](/guides/migrating-from-clamav/)). `--clamav-compat` -matches ClamAV's limit values and its `OK` for incomplete scans, for -differential testing, and logs each such object. +(see [Migrating from ClamAV](/guides/migrating-from-clamav/)). The one exception +is asked for by name: +[`--clamav-compat`](/reference/cli/#clamav-compatibility), a preset for +differential testing, answers an incomplete scan `OK` as ClamAV does and logs +each such object on stderr. ### The verdict is not the policy @@ -105,8 +106,9 @@ reported, never a silent truncation. See > database is built, not when it is loaded or when a file is scanned. A [prebuilt `.exavdb`](/guides/prebuilt-database/) is built once and loaded by -every CLI run, daemon start, reload and worker. So the build does the expensive -work (automatons, indexes, tables derived from the signatures) and stores the +every CLI run, daemon start and reload, then shared by every worker. So the +build does the expensive +work (the anchor index, tables derived from the signatures) and stores the result in a form that loads with little more than a copy. Load time and scan speed come first; a larger file is an acceptable price. diff --git a/www/src/content/docs/concepts/differential-testing.md b/www/src/content/docs/concepts/differential-testing.md index a6268219..4033cba3 100644 --- a/www/src/content/docs/concepts/differential-testing.md +++ b/www/src/content/docs/concepts/differential-testing.md @@ -15,7 +15,9 @@ against a cold page cache and are not performance numbers. The harness lives in the repository's `scripts/`; this page is the concept. The runbook is [`docs/DIFF_TESTING.md`](https://github.com/sylvinus/exav/blob/main/docs/DIFF_TESTING.md), -and the smallest run is `LIMIT=500 scripts/difftest.sh`. +and a quick run is `LIMIT=500 scripts/difftest.sh`. Bytecode programs and YARA +rules have their own comparisons (`scripts/bc-difftest.sh`, and the YARA +differential tests in `exav-core`). ## The setup @@ -63,13 +65,9 @@ sample. Agreement is what matters. ## `--clamav-compat`: matching boundaries, not quirks By default exav runs at full capability, and the harness runs it with -`--clamav-compat` (`COMPAT=0` measures what exav adds). For a like-for-like run, -`--clamav-compat` sets a stock ClamAV build's documented defaults: the limit -values (`--max-input-bytes 100M`, 400M held per file, `--max-unpack-depth 17`, -`--max-members 10000`, `--decode none`), the extractor -set, `.UNOFFICIAL` naming for unofficial-database signatures, and -`--partial-as ok`, so a file ClamAV would call clean is answered `OK` here too. -Each limit is also its own flag, and an explicit flag wins over the preset. +[`--clamav-compat`](/reference/cli/#clamav-compatibility) (`COMPAT=0` measures +what exav adds), which sets a stock ClamAV build's documented limits, extractor +set and naming, and answers a file ClamAV would call clean `OK`. It matches ClamAV's documented boundaries, not its implementation quirks: it does not reproduce each parser's internal caps and bail-outs (an RTF parser that stops diff --git a/www/src/content/docs/concepts/pe-emulation.md b/www/src/content/docs/concepts/pe-emulation.md index 8d38acca..23bcfe79 100644 --- a/www/src/content/docs/concepts/pe-emulation.md +++ b/www/src/content/docs/concepts/pe-emulation.md @@ -83,7 +83,8 @@ and nested content in the recovered image are reached too. Some stubs will not run to completion. The run stops on an instruction outside the implemented set, an export with no implementation, a fault nothing handled, -or a budget; the `exav-pe-emu` triage tool names which. +or a budget; the [`exav-pe-emu`](/subprojects/exav-pe-emu/) triage tool names +which. * **Nothing is fabricated.** A dump is emitted only when it reads back as a valid PE; a wrong guess is discarded rather than handed to the matcher. diff --git a/www/src/content/docs/concepts/quirks.md b/www/src/content/docs/concepts/quirks.md index 980b011c..79165a71 100644 --- a/www/src/content/docs/concepts/quirks.md +++ b/www/src/content/docs/concepts/quirks.md @@ -52,30 +52,17 @@ case needs no configuration. A file with a non-default password stays Most people assume a signature is a pattern. ClamAV's most expressive signature type is a program: a `.cbc` file in `bytecode.cvd` is C compiled to a custom VM bytecode, which runs against a candidate file and decides whether it is -malicious. A compatible scanner therefore has to embed an interpreter, with an -instruction budget, a scratch-memory cap, a call-depth limit, a bounds-checked -pointer model and a panic boundary per program. - -The format is line-oriented: a `ClamBC` header with a nibble-encoded integer -scheme (a byte `0x6N` carries nibble `N`), a trigger line that is itself a `.ldb` -logical signature (the program runs only when it matches), then records for -types, API declarations, globals, functions, basic blocks and strings. Programs -reach the outside world only through a fixed API of 107 functions -(`read`/`seek`/`file_find`, PE/PDF/JSON accessors, hashing, `disasm_x86`, and -`setvirusname` to report a hit). The most valuable programs are unpackers: part -of ClamAV's unpacking is implemented as bytecode, so a missing one weakens -detection across every sample of that packer. - -The second surprise: the signature database is code your process runs. ClamAV's -bytecode subsystem has a code-execution history -([CVE-2020-37167](https://www.cve.org/CVERecord?id=CVE-2020-37167), CVSS 8.4, the -`bytecode_vm` sandbox escape; and the optional LLVM JIT that generated native -code from database content). exav's interpreter is safe Rust with no JIT and a -panic boundary per program, and its budget is a step count, not wall-clock time: -a deadline would make detection depend on machine load, letting an attacker (or -a busy server) push a program past it. - -See [Bytecode sandbox](/concepts/bytecode-sandbox/). +malicious. Part of ClamAV's unpacking is written this way, so a compatible +scanner has to embed an interpreter, and the signature database becomes code +your process runs. Even the integers are odd: the `ClamBC` header encodes them +a nibble per byte (a byte `0x6N` carries nibble `N`). + +One design choice is not obvious: exav's budget for a program is a step count, +not wall-clock time. A deadline would make detection depend on machine load, +letting an attacker (or a busy server) push a program past it. + +The format, the sandbox and ClamAV's history with it are in +[Bytecode sandbox](/concepts/bytecode-sandbox/). ## The file extension is a lie, and so is the magic number @@ -116,20 +103,15 @@ the standard `Name="Project"` stream of benign macro documents. Every scanner has a maximum file size. The naive implementation ("over the limit, skip it, report OK") hands the attacker a bypass: append padding until the file -crosses the threshold. ClamAV's large-file behaviour is the known example: a file -over about 2 GB is read, scanned as zero bytes, and reported `OK`. - -exav's rule is that refusing by size still scans. An input over -`--max-input-bytes` has its first `--max-input-bytes` bytes scanned first, as -any smaller input would be, and only then gets a limit verdict (here with -`--max-input-bytes 100M`): +crosses the threshold. exav scans the first `--max-input-bytes` bytes before it +gives a limit verdict (here with `--max-input-bytes 100M`): ```text file size 5368709120 exceeds max-input-bytes 104857600; scanned first 104857600 bytes only ``` -A detection always beats a limit, and an incomplete scan is never `OK`. See -[Design principles](/concepts/design-principles/#never-a-silent-clean). +The rule, and ClamAV's large-file behaviour that motivates it, are in +[Never a silent clean](/concepts/design-principles/#never-a-silent-clean). ## Containers have two indexes, and malware uses the one you don't read @@ -137,28 +119,17 @@ A ZIP lists its members twice: a local file header before each member's data, and the central directory at the end. Every normal reader trusts the central directory, because APPNOTE says it is authoritative. So a member can be hidden: leave it out of the central directory and keep its local header and data. Many -extractors still write it out. - -exav does dual indexing: after the central-directory pass it scans the raw bytes -for local headers the directory did not claim and extracts those too, and the -same path is the salvage route when the directory is corrupt or missing. An -orphan it cannot read (encrypted, an unsupported codec, a size deferred to a -data descriptor, an extent past the end of the file) is reported rather than -skipped: its header is credible, so it is a member the target will extract. - -`PK\x03\x04` is four bytes and appears by chance in ordinary binaries, so a -local header only counts if the fields a real writer fills in are consistent -(version at most 6.3, no reserved flag bits, a defined method, a non-empty name -within the path limit and free of NULs). Orphans are charged to the archive-wide +extractors still write it out, so exav reads both +([dual indexing](/concepts/archive-extraction/#containers-have-two-indexes)). + +Members found only by their local header are charged to the archive-wide member budget rather than to a separate cap: a real JAR whose end-of-directory -record was gone had over 400 members reachable only as local headers. +record was gone had over 400 members reachable only that way. -**ISO images have the same shape**, and the trick is used more openly. A CD image -carries several volume descriptors, each with its own directory tree over the -same sectors: the primary ISO9660 tree and the Joliet tree with long Unicode -names. A malicious ISO lists its payload in only one of them, leaving the other -empty, so a reader that parses the other sees an empty disc. exav walks every -tree, Joliet first so the long names win. +**ISO images have the same shape**, and the trick is used more openly: a +malicious ISO lists its payload in only one of its directory trees (the primary +ISO9660 tree or the Joliet tree), so a reader that parses the other sees an +empty disc. exav walks every tree, Joliet first so the long names win. **gzip has the problem in miniature.** RFC 1952 allows a `.gz` to be several concatenated members, and `gzip`/`zcat` decompress all of them, while Rust's @@ -200,22 +171,24 @@ Two more places encryption is a signal: - `--partial-as password-protected=found` turns a password-protected member into a detection, `Heuristics.Encrypted.Zip` / `.RAR` / `.7Zip` / `.PDF` / `.OLE2`, or `.Archive` for other formats. -- exav tries a default password list on ZIPs: `infected`, `virus`, `malware`, - `password`, `123456`. "infected" is the standard password for sharing samples, - so a password-protected dropper opens with no configuration, as - `VelvetSweatshop` does for Office. `.pwdb` databases and `--passwords` extend - the pool. +- "infected" is the standard password for sharing samples, so exav tries it, + and a few others, on every encrypted ZIP: a password-protected dropper opens + with no configuration, as `VelvetSweatshop` does for Office (see + [Encryption support](/reference/formats/#encryption-support)). ## A wrong checksum must never stop the scan -This reads like a bug in every code review: exav extracts archive members without -verifying their CRCs, and verification is off by default. A wrong CRC must never -stop a member's bytes from being scanned, or an attacker could downgrade a -detection by flipping one checksum byte. (ClamAV also ignores CRCs when scanning. +This reads like a bug in every code review: for most formats exav scans a +member's bytes whether or not its CRC matches. A wrong CRC must not stop a +member from being scanned, or an attacker could downgrade a detection by +flipping one checksum byte. (ClamAV also ignores CRCs when scanning. Verification is for a library embedding extracting files, where a bad CRC is a real "corrupt file" signal: it needs the `checksums` feature of `exav-core` or `exav-unpack` and `ScanOptions::verify_checksums`. The command line has no -switch for it.) +switch for it.) The exceptions are formats where the checksum is the only check +on a complex decoder (RAR, WIM, ARC, EGG): there a mismatch means the decoder +produced garbage, and the member is reported `UNSCANNABLE` rather than scanned +as if it were right. The same instinct generalises. A CAB whose total-size field is overwritten with `0xFFFFFFFF` defeats a strict parser; exav clamps it and extracts the member. A @@ -226,14 +199,9 @@ recovered prefix, and `zcat` recovers it too. Leniency stops at structure. An OLE file a strict reader rejects (a broken red-black-tree ordering, common in real Office documents and malware) is walked as a flat directory instead, but a member that is present and unreadable is -still reported, not treated as clean. - -> When a sequential stream runs out of input, every byte that exists has been -> scanned: the missing tail is absent, not hidden, so a clean result is a real -> clean. The incomplete verdicts are for content that is present but unscanned: -> an encrypted member, an unsupported codec, a member over a limit. A stream -> damaged part way is one of those: the compressed data after the damage is -> present and was not decoded, so a clean result there is `UNSCANNABLE`. +still reported, not treated as clean. Where a truncated stream is a clean result +and a damaged one is not is explained in +[Archive extraction](/concepts/archive-extraction/#recovering-what-a-naive-reader-would-skip). ## Identity that survives the bytes changing @@ -429,26 +397,27 @@ The `clamd` protocol itself has oddities: escapes (or megabytes of text) in a member name that ends up in the scanner's output, so names are stripped of control characters and capped at 200 bytes. - **Decoder panics have to be a non-event.** Third-party decoders panic on crafted - input instead of returning an error. One `catch_unwind` boundary turns any - decoder panic into `UNSCANNABLE`, for every decoder, without patching each - dependency. The daemon adds a second layer for what `catch_unwind` cannot catch - (OOM, stack overflow): a prefork pool with per-job `RLIMIT_AS`, `RLIMIT_CPU` - and an alarm, after which the worker answers its client and is replaced. + input instead of returning an error, so one `catch_unwind` boundary covers them + all without patching each dependency + ([Hostile input](/concepts/archive-extraction/#hostile-input)). What it cannot + catch (OOM, stack overflow) is left to the daemon's prefork pool, whose + per-job limits replace the worker after it answers its client. - **A container type exav cannot model refuses the signature.** Leaving a `Container:` constraint unenforced to avoid a false negative would let a signature scoped to one container fire everywhere. Every type the official database uses is determined, so none is currently refused. - **PUA signatures are a separate alphabet.** The `.ndu`/`.ldu`/`.hdu`/`.hsu`/ - `.mdu` databases are ClamAV's potentially-unwanted-application sets, loaded + `.mdu`/`.msu` databases are ClamAV's potentially-unwanted-application sets, loaded only with PUA detection on. Hash-based PUA has no `PUA.` name gate, so loading them by default would flag adware. - **The signature ABI is 32-bit where the engine is not.** `.ldb` subsignature match offsets are handed to bytecode programs as `u32` values in which `u32::MAX` means "no match", so a real match at exactly 4 GiB is clamped to `0xFFFFFFFE` rather than read as a miss. -- **An unpacker that is not sure must emit nothing.** Every unpacked PE must - reconstruct a valid `MZ`/`PE\0\0` image, and a packer exav can only detect gets - no synthesised output. A wrong guess is discarded, never fed to the matcher. +- **An unpacker that is not sure must emit nothing.** A wrong guess fed to the + matcher is worse than none, so an unpacked PE that does not read back as a + valid image is discarded + ([PE stub emulation](/concepts/pe-emulation/#when-the-stub-wins)). - **Read one byte past the limit.** `take(cap)` returns end-of-file at the cap, so "exactly at the cap" and "over it" look the same; exav reads `take(cap + 1)` at every size gate to tell them apart, and reports the second. diff --git a/www/src/content/docs/concepts/streaming-memory.md b/www/src/content/docs/concepts/streaming-memory.md index 9ff4f72f..e130f02b 100644 --- a/www/src/content/docs/concepts/streaming-memory.md +++ b/www/src/content/docs/concepts/streaming-memory.md @@ -10,16 +10,17 @@ parse it whole (below): - `.ldb` logical signatures, with their regexes and byte comparisons; - the normalised views of HTML, text and scripts that those signatures are written against; -- YARA rules, bytecode programs, PE section hashes, whole-file hashes; -- structural analysis: unpacking, embedded files, heuristics. +- YARA rules, bytecode programs, whole-file hashes; +- archives decoded as they are read, embedded files, heuristics. What changes with size is where the file is while that runs. ## Up to `--max-object-bytes`: in memory -A file up to `--max-object-bytes` (256 MiB by default) is read into memory. -Memory is roughly the file's size, plus a lowercase copy for case-insensitive -signatures and, for text, one normalised copy at a time. The deobfuscated +A file up to `--max-object-bytes` (256 MiB by default) is read into memory, and +so is an HTTP source that size. Memory is roughly the file's size, plus a +lowercase copy when a case-insensitive signature has to be checked and, for +text, one normalised copy at a time. The deobfuscated JavaScript view stops at 32 MiB; a script whose view is longer is reported `LIMITS-EXCEEDED` unless something is found, though its raw bytes are still scanned in full. @@ -27,22 +28,91 @@ scanned in full. ## Past it: through a block cache A larger file is read through a cache of 64 KiB blocks, of which at most 8 MiB -are held, least recently used dropped first. So are an HTTP range source and a -spill file (below). Verifying a match can still look anywhere in the file: the -cache bounds what is held at once, not how far a check may reach, and a block -that was dropped is read again when it is needed. The scan is the same one, with -the same results, only slower, since the file is read a few times over: once for -what format detection searches it for, what carving looks for and its digests -(when a hash signature has its size), once for the signature sweep (every -automaton at once), and again in the parts a check needs. -[Architecture](/concepts/architecture/#reading-an-object-not-held-in-memory) -shows what reads through the cache. - -A few things parse a file whole and are not done past the limit: +are held, least recently used dropped first. So are an HTTP source of that size +and a spill file (below). Verifying a match can still look anywhere in the +file: the cache bounds what is held at once, not how far a check may reach, and +a block that was dropped is read again when it is needed. The scan is the same +one, with the same results, only slower, since the file is read a few times +over: once for format detection's search, carving and the digests (when a hash +signature has its size), once for the signature sweep (every anchor at once), +twice more for YARA's prefilter when rules are loaded, and again in the parts a +check needs. + + + One source, read through a block cache + A file or an HTTP source past --max-object-bytes, or a spill + file, is read through a block cache of 64 KiB blocks holding 8 MiB. Every + check asks it for bytes at an offset: the signature sweep a chunk at a time + in order, verification and the container walk at any offset, the regexes + stepped through, the hashes, typing's search and carving in one shared pass. + The checks that parse an object whole run only when it fits within + --max-object-bytes. + + + + + + + SOURCES + WHAT READS IT + + + + + + + + + + + + + + + file + HTTP range + spill file + BlockCache + signature sweep + verification + regexes + hashes · typing · carving + container walk + whole-object checks + + + past --max-object-bytes + past the limit too + a large member or view + 64 KiB blocks, 8 MiB held, + least recently used dropped; + bytes at any offset, or in + 1 MiB chunks in order + chunks in order, one pass + windows at any offset + lazy DFAs stepped through + one shared pass; start and end + headers and members, by offset + held whole only: PE, RAR, OLE + + + + + + + + + + + + + +A few things need the file whole and are not done past the limit: - the structure of a PE: entry-point and section offsets, section hashes, imports, icons, the Authenticode signature, UPX and packer unpacking; - YARA's `pe`, `elf` and `dotnet` modules; +- a PCRE subsignature with a backreference, and the `fuzzy_img` image hash; - containers whose format is read whole (7z, RAR, OLE, PDF, the virtual-disk and filesystem images and others; see [Supported formats](/reference/formats/#size-what-is-read-as-it-goes-and-what-is-read-whole)). @@ -51,10 +121,8 @@ A file for which one of these applied is reported `LIMITS-EXCEEDED`, naming `--max-object-bytes`, unless something is found. Raise the limit to have them run, at the cost of memory. -The normalised views of a large text file are as large as the file, so they are -written to the same temporary files as a buffered stream (below), under the same -budgets. With `--spill-dir off` they are skipped and the file is -`LIMITS-EXCEEDED`. +The normalised views of a large text file are as large as the file, so they go +to a [spill file](#spill-files). ## Archive members @@ -65,8 +133,9 @@ limit, and past it written to a temporary file and scanned from there. With `--spill-dir off`, or past `--max-spill-bytes`, it is not scanned and the file is `LIMITS-EXCEEDED`. 7z members stream the same way once the container is read. -A member that has to be decoded whole (an encrypted ZIP member, a member of a -format read whole) is `LIMITS-EXCEEDED` past `--max-object-bytes`. +A member that has to be decoded whole (an encrypted ZIP member, a file inside a +DMG, a member of a format read whole) is `LIMITS-EXCEEDED` past +`--max-object-bytes`. ## Stdin, `INSTREAM` and ICAP bodies @@ -79,17 +148,86 @@ buffered stream then goes through the same scan as a file. A stream past `--max-input-bytes` is. See [buffering a stream](/reference/cli/#buffering-a-stream-spill). -## The database, not the file, is what uses memory +## Spill files + + + The two uses of spill files + Before a scan, a stream (stdin, INSTREAM, ICAP) is held in + memory up to --spill-threshold-bytes and then in a temporary file up to + --max-spill-bytes, and becomes a seekable source for the scan. Inside a scan, + a streamed archive member that decodes past --max-object-bytes, and the normalised + text views of a large text file, are written to spill files and read back + through a block cache. A stream past its budget, or past the threshold with + --spill-dir off, has what was held scanned; with --spill-dir off, such a + member or view is not scanned. Either way a scan that finds nothing is + LIMITS-EXCEEDED. + + + + + + + BEFORE THE SCAN: A STREAM + INSIDE THE SCAN: WHAT IT MAKES + + + + + + + + + + + + + stdin · INSTREAM + RAM + temp file + the scan + member past the limit + text views, large file + spill file + scanned from there + + + ICAP bodies + --spill-threshold-bytes + --max-spill-bytes + a seekable source + decoded past --max-object-bytes + HTML, text, JS normalised + written as it is made + through a block cache + + + + + + + + + + + Past --max-spill-bytes, or past the threshold with --spill-dir off, what was held is + scanned and the stream is LIMITS-EXCEEDED unless that finds something. + --max-total-spill-bytes bounds every spill file of a process together. + + + +The library never writes to disk itself: the host hands a scan somewhere to +spill (the `exav` binary passes its `--spill-dir` files and budgets). Only a +member decoded as it is read can spill; one decoded whole is bounded by +`--max-object-bytes`. Without a spill, a member that decodes past +`--max-object-bytes` is not scanned, the normalised views of a text file that +large are skipped, and a scan that found nothing is `LIMITS-EXCEEDED` rather +than `OK`. -Loading raw ClamAV databases builds a double-array Aho-Corasick automaton, and -for the full `main` + `daily` set that construction needs several GB for a short -time, far more than the structures it produces. It is one allocation burst, so it -cannot be reduced by freeing things between steps. +## The database, not the file, is what uses memory -The [prebuilt database](/guides/prebuilt-database/) avoids it: build the -automaton once on a capable host, serialize it to a `.exavdb`, and load that -everywhere else. Loading allocates about the final size and skips the -construction. +Building the signature index from raw ClamAV databases briefly takes more memory +than the index itself. A [prebuilt `.exavdb`](/guides/prebuilt-database/#what-it-costs) +is built once on a capable host and loads at about its final size. ## Tuning @@ -101,8 +239,9 @@ Memory and CPU budgets are separate flags: once across nesting levels, so it does not bound the total. - **`--max-process-bytes`** (2G per worker in the pool): the memory a scan may use. What formats decoded whole hold across one top-level file is charged - cumulatively against 1G, lowered to half of `--max-process-bytes`, so reaching - it is reported as a limit instead of the scan being killed. Streamed members + cumulatively against 1G. That total and `--max-object-bytes` are both kept + within half of `--max-process-bytes`, so reaching them is reported as a limit + instead of the scan being killed. Streamed members are not charged there: they are bounded by `--max-object-bytes` and the spill budgets. - **`--max-matcher-bytes`** (10G): the most bytes fed to the matcher across one diff --git a/www/src/content/docs/getting-started/installation.mdx b/www/src/content/docs/getting-started/installation.mdx index 97d560f2..3fb6c525 100644 --- a/www/src/content/docs/getting-started/installation.mdx +++ b/www/src/content/docs/getting-started/installation.mdx @@ -7,8 +7,9 @@ import { Steps } from '@astrojs/starlight/components'; exav runs on Linux, macOS and Windows and ships as a single binary. The Linux release builds are static (musl); the macOS and Windows builds link the system -libraries. On Windows there is no prefork worker pool and no `--auto-update`, -which are Unix only. +libraries. On Windows there is no prefork worker pool, no per-job limits +(`--max-scan-secs`, `--max-process-bytes`) and no `--auto-update`, which are +Unix only. ## Prebuilt binaries @@ -16,9 +17,27 @@ Each release on [GitHub](https://github.com/sylvinus/exav/releases) carries binaries for `{x86_64,aarch64}-unknown-linux-musl` (static, any distro), `{x86_64,aarch64}-apple-darwin` and `x86_64-pc-windows-msvc`, and the WASI module `exav-wasm--wasm32-wasip1.wasm` (see the -[WASM sandbox guide](/guides/wasm-sandbox/)). They are the default build, -without HTTP support: to fetch signatures with `--auto-update` or scan URLs, -build with `--features http` below. +[WASM sandbox guide](/guides/wasm-sandbox/)). Each archive has a `.sha256` +checksum beside it. For example, on 64-bit Linux: + +```sh +tag=v0.0.2 target=x86_64-unknown-linux-musl +curl -fsSLO https://github.com/sylvinus/exav/releases/download/$tag/exav-$tag-$target.tar.gz +tar xzf exav-$tag-$target.tar.gz +sudo install -m755 exav /usr/local/bin/exav +exav --version +``` + +They are the default build, without HTTP support: to fetch signatures with +`--auto-update` or scan URLs, build with `--features http` below. + +The standalone tools come as archives of their own, `exav-unpack--` +(an extractor with `unzip`'s command line, see +[exav-unpack](/subprojects/exav-unpack/#cli)), `exav-grep--` +(see [exav-grep](/subprojects/exav-grep/)) and `exav-pe-emu--` (a +PE unpacker that runs a packer's own stub in an emulator, see +[exav-pe-emu](/subprojects/exav-pe-emu/#cli)). Every archive carries the +`LICENSE` and the `NOTICE` of what the binary links. ## From crates.io @@ -87,6 +106,7 @@ Build a `.deb` locally with `cargo-deb`: cargo install cargo-deb cargo build --release -p exav cargo deb -p exav --no-build # the binary, and the systemd unit exav.service +sudo apt install ./target/debian/exav_*.deb ``` The unit runs as the `clamav` user and reads `/var/lib/clamav`, as diff --git a/www/src/content/docs/getting-started/introduction.md b/www/src/content/docs/getting-started/introduction.md index 98f83a2c..b5c47394 100644 --- a/www/src/content/docs/getting-started/introduction.md +++ b/www/src/content/docs/getting-started/introduction.md @@ -41,7 +41,7 @@ suspicious.bin: Win.Trojan.Agent-1234 FOUND A stream on stdin is scanned like a file: ```console -$ curl -fsS https://example.com/download.zip | exav -d /var/lib/exav - +$ curl -fsS https://example.com/download.zip | exav -d ~/.cvdupdate/database - stdin: OK ``` @@ -50,7 +50,7 @@ protocol authenticates its clients, so bind to an interface only trusted clients reach): ```console -$ exav --listen clamd://127.0.0.1:3310 --listen icap://127.0.0.1:1344 -d /var/lib/exav +$ exav --listen clamd://127.0.0.1:3310 --listen icap://127.0.0.1:1344 -d /var/lib/clamav exav: prefork daemon: 4 workers; per-job limits: wall 120s, mem 2048 MiB, cpu 120s; recycle every 1000 jobs exav: serving ICAP on tcp:127.0.0.1:1344 (services: avscan, srv_clamav, virus_scan; preview 4096 B) ``` @@ -74,6 +74,7 @@ malware as clean). Load signatures with -d/--database or --sig-dir, or pass container. - [Quick start](/getting-started/quick-start/): signatures, a first scan, the output. +- [Scanning](/guides/scanning/) and [Daemon mode](/guides/daemon/): everyday use. - [Comparison with ClamAV](/project/comparison-with-clamav/): compatibility, performance, and where the two differ. - [Migrating from ClamAV](/guides/migrating-from-clamav/): replace `clamscan` and diff --git a/www/src/content/docs/getting-started/quick-start.mdx b/www/src/content/docs/getting-started/quick-start.mdx index 762df7b9..9ee95183 100644 --- a/www/src/content/docs/getting-started/quick-start.mdx +++ b/www/src/content/docs/getting-started/quick-start.mdx @@ -18,81 +18,87 @@ signatures loaded it refuses to run. Fetching signatures is the first step below 1. **Get ClamAV signatures** with Cisco's own updater: ```sh - pipx install cvdupdate # or `pip install --user cvdupdate` + pipx install cvdupdate cvd update ``` This populates `~/.cvdupdate/database` with `main.cvd`, `daily.cvd` and - `bytecode.cvd`. If `freshclam` already keeps a database current, use its - directory instead (usually `/var/lib/clamav`) in the commands below. See - [Signatures](/guides/signatures/) for the licensing background and - third-party feeds. + `bytecode.cvd`. See [Signatures](/guides/signatures/) for the licensing + background and third-party feeds. -2. **Check that detection works** with the EICAR test file, which every scanner - detects and which is harmless: +2. **Tell exav where they are.** exav loads `--sig-dir` (default + `/var/lib/exav`) when `-d` is not given; point it at the new directory once + for this shell: + + ```sh + export EXAV_SIG_DIR=~/.cvdupdate/database + ``` + + If `freshclam` already keeps a database current, use its directory instead + (usually `/var/lib/clamav`). + +3. **Check that detection works** with the EICAR test file, which is harmless: ```sh printf '%s%s' 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-' \ 'TEST-FILE!$H+H*' > eicar.com - exav -d ~/.cvdupdate/database eicar.com + exav eicar.com ``` ```console eicar.com: Eicar-Test-Signature FOUND ``` -3. **Scan a file.** The output follows `clamscan`'s format: + A `SCAN SUMMARY` block follows the result lines. This first run parses the raw signatures, which takes a while and needs + memory to spare (see [what it costs](/guides/prebuilt-database/#what-it-costs)). + If the process dies with no message, it ran out of memory. exav always knows + EICAR, so the check that the ClamAV signatures loaded is `Known viruses` in + the summary: it should be in the millions. + +4. **Scan a file.** The output follows `clamscan`'s format: ```sh - exav -d ~/.cvdupdate/database suspicious.bin + exav suspicious.bin ``` ```console suspicious.bin: OK ``` -4. **Scan a directory.** It is recursive by default; `--quiet` prints only +5. **Scan a directory.** It is recursive by default; `--quiet` prints only detections, partial results and errors: ```sh - exav -d ~/.cvdupdate/database --quiet /var/www + exav --quiet /var/www ``` -5. **Scan stdin** with `-`: +6. **Scan stdin** with `-`: ```sh - cat backup.tar.gz | exav -d ~/.cvdupdate/database - + cat backup.tar.gz | exav - ``` ## Exit codes -The status is the last word on each result line, and each status has an exit -code: `0` for `OK`, `1` for `FOUND`, `2` for `ERROR` (exav could not do its job), -and `3` for `PARTIAL` (something could not be fully examined). - - +The status is the last word on each result line: `0` `OK`, `1` `FOUND`, `2` +`ERROR` (exav could not do its job), `3` `PARTIAL` (something could not be fully +examined, where `clamscan` would say `OK`). A CI job should not treat `3` as a +pass. See [which code a run exits with](/reference/verdicts/#process-exit-code). ## Faster startup: build a database once -Loading Cisco's raw database (`main.cvd` + `daily.cvd`) builds a large in-memory -automaton, which takes time and several GB of transient RAM. Do it once and reuse -the result: +Loading the raw signatures parses millions of them on every start. Do it once +and reuse the result: ```sh -exav -d ~/.cvdupdate/database --build-db exav.exavdb # heavy, run daily in CI -exav -d exav.exavdb /data # fast cold start +exav --build-db exav.exavdb # run daily, e.g. in CI +exav -d exav.exavdb /data # fast cold start ``` -A `.exavdb` is tied to the exav version that built it: after upgrading exav, -rebuild it. See the [prebuilt database guide](/guides/prebuilt-database/). +Rebuild the `.exavdb` after upgrading exav if its format changed (the load says +so). See the [prebuilt database guide](/guides/prebuilt-database/). ## Run it as a daemon @@ -101,8 +107,8 @@ load cost. The daemon stays in the foreground, so run the client from a second terminal: ```sh -exav --listen "$XDG_RUNTIME_DIR/exav.sock" -d exav.exavdb # load once, serve -exav --connect "$XDG_RUNTIME_DIR/exav.sock" /data # client +exav --listen /tmp/exav.sock -d exav.exavdb # load once, serve +exav --connect /tmp/exav.sock /data # client ``` A daemon can also keep its signatures current itself, fetching from a mirror diff --git a/www/src/content/docs/guides/daemon.mdx b/www/src/content/docs/guides/daemon.mdx index 8717c2f8..e0e4d53c 100644 --- a/www/src/content/docs/guides/daemon.mdx +++ b/www/src/content/docs/guides/daemon.mdx @@ -10,6 +10,10 @@ socket, so callers pay no per-scan load cost. The wire protocol is a subset of ClamAV's `clamd` protocol, so existing tooling (`clamdscan`, milters, `clamd` client libraries) talks to exav unchanged. +The daemon refuses to start without signatures, so fetch them first (see +[Signatures](/guides/signatures/)). A prebuilt `.exavdb` starts much faster than +raw `.cvd` files: see [what it costs](/guides/prebuilt-database/#what-it-costs). + ## Start a daemon ```sh @@ -20,7 +24,8 @@ exav --listen clamd://0.0.0.0:3310 -d /var/lib/exav # the same, spelled ou The protocol travels in the address, and with no scheme it is `clamd`. A leading `/` is a socket path; anything else is `host:port`. Nothing listens without -`--listen`. +`--listen`. Creating a socket under `/run/clamav` needs root or the `clamav` +user; as yourself, use a path such as `$XDG_RUNTIME_DIR/exav.sock`. Then scan through it: @@ -32,7 +37,8 @@ clamdscan --stream file.bin # the official ClamAV client clamdscan --fdpass file.bin # fd-passing also works ``` -`--listen` accepts connections and `--connect` makes one. +`--listen` accepts connections and `--connect` makes one. `clamdscan` takes the +address from its `clamd.conf` (`LocalSocket`, or `TCPSocket` and `TCPAddr`). By default the client sends paths, which the daemon opens itself, so the daemon has to be able to see them. `--send-as contents` sends the bytes (`INSTREAM`) and @@ -42,10 +48,12 @@ or a daemon running as another user. `-` (stdin) is always sent as contents. A directory names its whole tree in all three modes, as it does for `clamdscan`. The client walks it and sends one request per file, so `--exclude` / `--include` -apply to the tree and each request has one reply. A directory holding the parts -of a byte-split archive (`x.001`, `x.002`, ...) is sent whole instead, as one -`CONTSCAN` by path, so the daemon can rejoin them; the client's filters and -`--no-recursive` do not apply to that directory. +apply to the tree and each request has one reply. The parts of a byte-split +archive (`x.001`, `x.002`, ...) are sent together so the daemon can rejoin them +(see [split archives](#split-archives)). By path, a directory holding such parts +goes as one `CONTSCAN`, and the client's filters and `--no-recursive` do not +apply to it; with `contents` or `fd`, the parts of each directory go in one +`EXINSTREAM MULTI` request. ## Who may connect @@ -64,6 +72,8 @@ inherited, and a value that is not a workable mode is refused. The mode belongs to the address because it is a property of that socket. A `host:port` has no file to set permissions on, so asking for one there is refused; bind a TCP listener to the interface that should reach it and firewall the rest. +The other address options are listed in +[what belongs to one listener](/reference/cli/#what-belongs-to-one-listener).