diff --git a/CHANGELOG.md b/CHANGELOG.md index 58b3be2..a7c5a51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,17 @@ records, not how the code changed. ### Fixed +- **Opening a folder on an HFS+ volume showed the volume's top level again, + and search, hashing, indexing and tagging skipped HFS+ folders.** Each + reader names its entries by its own key (the MFT record for NTFS, the inode + for ext, the catalog node ID for HFS+), and most of the code looked for + every key but HFS+'s. Expanding a folder in the tree listed the volume root + instead of its contents (nested without end), a search, file-type check, + hash or index run never went below the top folder, and an HFS+ file had no + key to tag or hash it under. The key is now chosen in one place. A tagged + HFS+ file can also be exported, which it could not be before: it is looked + up again from its key. Checked in a browser on a synthetic HFS+ volume + built with nested folders; the volume itself is unchanged. - **A cut-off name in the evidence tree can be read in full.** The tree is narrow, so a long line such as a partition's "volume label ..." detail was shown with an ellipsis and the rest was unreachable. Hovering over a row now diff --git a/engine/artifacts.py b/engine/artifacts.py index b5c4826..bbfef24 100644 --- a/engine/artifacts.py +++ b/engine/artifacts.py @@ -1,8 +1,4 @@ -def _node_of(e): - return (e.get("mft") if e.get("mft") is not None - else e.get("inode") if e.get("inode") is not None - else e.get("oid") if e.get("oid") is not None - else e.get("start_cluster")) +from .nodes import node_of as _node_of def _safe_listdir(fs, node, path): try: diff --git a/engine/casedb.py b/engine/casedb.py index e207454..91caf3a 100644 --- a/engine/casedb.py +++ b/engine/casedb.py @@ -14,6 +14,7 @@ from . import fuzzyhash from . import version as version_mod from .text import t as _t +from .nodes import node_of SCHEMA = """ CREATE TABLE IF NOT EXISTS meta ( @@ -1633,10 +1634,7 @@ def remove_bookmark(self, bid): @_writes def tag_item(self, evidence_id, item, tag, note="", part=0): - node = str(item.get("mft") if item.get("mft") is not None - else item.get("inode") if item.get("inode") is not None - else item.get("oid") if item.get("oid") is not None - else item.get("start_cluster")) + node = str(node_of(item)) cur = self.db.execute( "INSERT INTO tagged_items (evidence_id,part,node,path,name,size," "is_dir,deleted,tag,note,created_at,examiner,modified,accessed," @@ -1782,10 +1780,7 @@ def save_search(self, evidence_id, name, query, hits, part=None): "deleted,where_found,term,context,file_offset,encoding,modified) " "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", [(sid, h.get("name"), h.get("path"), - str(h.get("mft") if h.get("mft") is not None - else h.get("inode") if h.get("inode") is not None - else h.get("oid") if h.get("oid") is not None - else h.get("start_cluster")), + str(node_of(h)), h.get("size"), int(bool(h.get("is_dir"))), int(bool(h.get("deleted"))), h.get("where"), h.get("term"), h.get("context"), h.get("file_offset"), h.get("encoding"), diff --git a/engine/filesearch.py b/engine/filesearch.py index b0415b9..1512f1c 100644 --- a/engine/filesearch.py +++ b/engine/filesearch.py @@ -1,6 +1,7 @@ import re from . import profile +from .nodes import node_of from .text import t as _t DEFAULT_SCAN_BYTES = 64 << 20 @@ -45,10 +46,7 @@ def _walk(fs, root_node, path, out, depth, max_depth, seen, progress=None, if progress is not None and len(out) % PROGRESS_EVERY == 1: progress(len(out)) if e.get("is_dir"): - node = (e.get("mft") if e.get("mft") is not None - else e.get("inode") if e.get("inode") is not None - else e.get("oid") if e.get("oid") is not None - else e.get("start_cluster")) + node = node_of(e) if node is None or node in seen: continue seen.add(node) @@ -325,7 +323,7 @@ def _hit(e, where, context, term=None, offset=None, encoding=None): "modified": e.get("modified"), "created": e.get("created"), "accessed": e.get("accessed"), "mft": e.get("mft"), "inode": e.get("inode"), "oid": e.get("oid"), - "start_cluster": e.get("start_cluster"), + "cnid": e.get("cnid"), "start_cluster": e.get("start_cluster"), "where": where, "term": term, "context": context, "file_offset": offset, "encoding": encoding, "entry": e, diff --git a/engine/filetype.py b/engine/filetype.py index b7cc5ad..21d3fda 100644 --- a/engine/filetype.py +++ b/engine/filetype.py @@ -1,5 +1,7 @@ import struct +from .nodes import node_of + SIGNATURES = [ (0, b"\xFF\xD8\xFF", "jpeg"), (0, b"\x89PNG\r\n\x1a\n", "png"), @@ -280,10 +282,7 @@ def scan(fs, entries, read=None, progress=None, limit=None): "name": e.get("name"), "path": e.get("path"), "size": size, "deleted": bool(e.get("deleted")), "modified": e.get("modified"), - "node": (e.get("mft") if e.get("mft") is not None - else e.get("inode") if e.get("inode") is not None - else e.get("oid") if e.get("oid") is not None - else e.get("start_cluster")), + "node": node_of(e), **{k: got[k] for k in ("extension", "extension_says", "detected", "content_is", "why", "severity")}, diff --git a/engine/fs/hfsplus.py b/engine/fs/hfsplus.py index 46fea14..24c2e67 100644 --- a/engine/fs/hfsplus.py +++ b/engine/fs/hfsplus.py @@ -224,8 +224,36 @@ def _index(self): self._by_parent, self._threads = by_parent, threads self._resolve_links(by_parent) self._resolve_compression(by_parent) + self._by_cnid = {rec["cnid"]: rec for recs in by_parent.values() + for rec in recs} return by_parent, threads + def _path_of(self, cnid): + """The path of the file or folder with this catalog ID, built from + the parent links of its thread records; None if the chain is broken + or loops.""" + _by_parent, threads = self._index() + parts, seen = [], set() + while cnid != CNID_ROOT_FOLDER: + if cnid in seen or cnid not in self._by_cnid: + return None + seen.add(cnid) + rec = self._by_cnid[cnid] + parts.append(rec["name"]) + cnid = rec["parent"] + return "/" + "/".join(reversed(parts)) + + def entry_by_node(self, cnid): + """The entry listdir() would give for the file or folder with this + catalog ID, or None. For a caller that kept only the handle.""" + self._index() + rec = self._by_cnid.get(int(cnid)) + if rec is None: + return None + out = dict(rec) + out["path"] = self._path_of(rec["cnid"]) or "/" + rec["name"] + return out + def _private_children(self, by_parent, name, prefix): """{number: record} of the entries in the private folder `name` (at the volume root) whose names are `prefix` and a number.""" diff --git a/engine/hashing.py b/engine/hashing.py index aa896a5..5a10b92 100644 --- a/engine/hashing.py +++ b/engine/hashing.py @@ -11,16 +11,7 @@ ALGO_BY_LEN = {32: "md5", 40: "sha1", 64: "sha256"} -def node_of(entry): - for key in ("mft", "inode", "oid", "start_cluster"): - v = entry.get(key) - if v is not None: - return v - return None - -def node_key(entry): - v = node_of(entry) - return None if v is None else str(v) +from .nodes import node_of, node_key # noqa: F401 (re-exported) CHUNK = 4 << 20 diff --git a/engine/nodes.py b/engine/nodes.py new file mode 100644 index 0000000..ed325c1 --- /dev/null +++ b/engine/nodes.py @@ -0,0 +1,27 @@ +"""The handle a filesystem reader gives each entry it lists. + +Every reader names an entry by one field of its own: the MFT record number +(NTFS), the inode (ext), the object ID (APFS, and logical evidence), the +catalog node ID (HFS+) or the first cluster (FAT and exFAT). Anything that +goes on to list a folder, open a file by handle, key a tag or a hash, or +cross-reference an entry has to pick whichever one is there. That choice was +once written out at each place that made it, and HFS+ was left out of most of +them: its folders were never descended into and nothing keyed on its entries +could be kept. It is made here, once.""" + +KEYS = ("mft", "inode", "oid", "cnid", "start_cluster") + + +def node_of(entry): + """The entry's handle, or None if it has none.""" + for key in KEYS: + value = entry.get(key) + if value is not None: + return value + return None + + +def node_key(entry): + """The handle as a string, for keying; None if the entry has none.""" + value = node_of(entry) + return None if value is None else str(value) diff --git a/engine/recyclebin.py b/engine/recyclebin.py index 56f4aab..b26f5c3 100644 --- a/engine/recyclebin.py +++ b/engine/recyclebin.py @@ -74,11 +74,7 @@ def parse_info2(data): }) return out -def _node_of(e): - return (e.get("mft") if e.get("mft") is not None - else e.get("inode") if e.get("inode") is not None - else e.get("oid") if e.get("oid") is not None - else e.get("start_cluster")) +from .nodes import node_of as _node_of def scan(fs, listdir, root_node, progress=None): out = [] diff --git a/engine/server.py b/engine/server.py index 7ee1820..9be15c6 100644 --- a/engine/server.py +++ b/engine/server.py @@ -15,6 +15,7 @@ from urllib.parse import urlparse, parse_qs from . import carve as carve_mod +from .nodes import node_of from . import volmap as volmap_mod from . import nativedialog from . import attack as attack_mod @@ -196,10 +197,7 @@ def __init__(self, fs, entry, size, stream="", cache=None): def _key(self): e = self.entry - node = (e.get("oid") if e.get("oid") is not None else - e.get("mft") if e.get("mft") is not None else - e.get("inode") if e.get("inode") is not None else - e.get("start_cluster")) + node = node_of(e) return (id(self.fs), str(node), e.get("path") or e.get("name") or "", self.stream) @@ -595,8 +593,7 @@ def region(self, offset, size=None, ev=None): return OffsetReader(ev.image, offset, size or (ev.image.size - offset)) def hive(self, offset, entry, ev=None): - key = (offset, entry.get("mft") or entry.get("inode") - or entry.get("oid") or entry.get("start_cluster"), entry.get("name")) + key = (offset, node_of(entry), entry.get("name")) ev = ev or self.current if key in ev.hive_cache: return ev.hive_cache[key] @@ -3677,7 +3674,7 @@ def run(progress): "name": e.get("name"), "path": e.get("path"), "size": e.get("size"), "deleted": e.get("deleted"), "mft": e.get("mft"), "inode": e.get("inode"), - "oid": e.get("oid"), + "oid": e.get("oid"), "cnid": e.get("cnid"), "start_cluster": e.get("start_cluster")} for m in r["messages"]: m["store"] = e.get("path") @@ -4590,6 +4587,14 @@ def _entry_from_body(fs, body): walking the FAT (#94).""" node = body.get("node") n = None if node in (None, "", "null") else int(node) + lookup = getattr(fs, "entry_by_node", None) + if lookup is not None and n is not None: + # A reader whose entries carry what is needed to read them (HFS+ + # keeps a file's extents in its catalog record) is asked for the + # entry itself, rather than given a bare handle it cannot read. + found = lookup(n) + if found is not None: + return found entry = {"name": body.get("name"), "path": body.get("path"), "size": body.get("size"), "is_dir": False, "deleted": bool(body.get("deleted")), @@ -4604,6 +4609,8 @@ def _entry_from_body(fs, body): entry["inode"] = n elif fsname.startswith("APFS") or fsname.startswith("LOGICAL"): entry["oid"] = n + elif fsname.startswith("HFS"): + entry["cnid"] = n else: entry["start_cluster"] = n return entry @@ -4894,11 +4901,7 @@ def _resolve_path(fs, path): return hit if not hit.get("is_dir"): return None - node = (hit.get("mft") if hit.get("mft") is not None - else hit.get("inode") if hit.get("inode") is not None - else hit.get("oid") if hit.get("oid") is not None - else hit.get("cnid") if hit.get("cnid") is not None - else hit.get("start_cluster")) + node = node_of(hit) here = hit.get("path") or (here + "/" + want) return None diff --git a/engine/textindex.py b/engine/textindex.py index 42bbb9c..9889eaf 100644 --- a/engine/textindex.py +++ b/engine/textindex.py @@ -5,6 +5,7 @@ import zlib from . import filesearch +from .nodes import node_of from . import entropy as entropy_mod from . import officedoc as officedoc_mod @@ -464,10 +465,7 @@ def on_entry(e): return if len(body) >= max_text: counts["text_capped"] += 1 - node = (e.get("mft") if e.get("mft") is not None - else e.get("inode") if e.get("inode") is not None - else e.get("oid") if e.get("oid") is not None - else e.get("start_cluster")) + node = node_of(e) rows.append((e.get("name") or "", e.get("path") or "", body, str(node), part, e.get("size") or 0, int(bool(e.get("deleted"))), e.get("modified") or "", diff --git a/engine/timeline.py b/engine/timeline.py index 93fc52d..26345e9 100644 --- a/engine/timeline.py +++ b/engine/timeline.py @@ -45,11 +45,7 @@ def _parse(ts): return None return t if t.tzinfo else t.replace(tzinfo=_UTC) -def _node_id(entry): - for key in ("mft", "inode", "oid", "cnid", "start_cluster"): - if entry.get(key) is not None: - return entry[key] - return None +from .nodes import node_of as _node_id def root_of(fs): root = getattr(fs, "root_node", None) diff --git a/tests/test_nodes.py b/tests/test_nodes.py new file mode 100644 index 0000000..33b8411 --- /dev/null +++ b/tests/test_nodes.py @@ -0,0 +1,135 @@ +"""The handle a filesystem reader gives an entry (engine.nodes), and that +everything which walks or keys entries uses the one rule. HFS+ names its +entries by catalog node ID ("cnid"); most of the code once looked only for +the other readers' keys, so an HFS+ folder was never descended into and an +HFS+ file had no key to tag or hash it under.""" + +import os +import re +import sys +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import imagebuild_hfsplus as hb # noqa: E402 +from engine import filesearch, hashing, nodes, timeline # noqa: E402 +from engine.ewf import OffsetReader # noqa: E402 +from engine.fs.ntfs import open_fs # noqa: E402 + + +class Mem(object): + bytes_per_sector = 512 + + def __init__(self, data): + self.data, self.size = data, len(data) + + def read_at(self, offset, length): + if offset < 0 or offset >= self.size: + return b"" + return self.data[offset:offset + length] + + +def hfs(): + data = hb.build([ + hb.Dir("Top", [hb.Dir("Inner", [hb.File("deep.txt", b"deep")]), + hb.File("a.txt", b"aa")]), + hb.File("root.txt", b"rr")]) + return open_fs(OffsetReader(Mem(data), 0, len(data))) + + +class Rule(unittest.TestCase): + + def test_each_readers_key_is_found(self): + for key in ("mft", "inode", "oid", "cnid", "start_cluster"): + self.assertEqual(nodes.node_of({key: 7, "name": "x"}), 7, key) + self.assertEqual(nodes.node_key({key: 7}), "7", key) + + def test_zero_is_a_handle(self): + self.assertEqual(nodes.node_of({"start_cluster": 0}), 0) + self.assertEqual(nodes.node_key({"mft": 0}), "0") + + def test_an_entry_with_no_handle_has_none(self): + self.assertIsNone(nodes.node_of({"name": "x"})) + self.assertIsNone(nodes.node_key({"name": "x"})) + + def test_the_older_names_for_it_are_the_same_function(self): + entry = {"cnid": 21} + self.assertEqual(hashing.node_of(entry), 21) + self.assertEqual(hashing.node_key(entry), "21") + self.assertEqual(timeline._node_id(entry), 21) + + +class Hfsplus(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.fs = hfs() + + def test_the_shared_walker_descends_into_folders(self): + got = filesearch.collect(self.fs, self.fs.root_node) + self.assertEqual(sorted(e["path"] for e in got), [ + "/Top", "/Top/Inner", "/Top/Inner/deep.txt", "/Top/a.txt", + "/root.txt"]) + + def test_every_entry_has_a_handle_to_key_it_by(self): + for e in filesearch.collect(self.fs, self.fs.root_node): + self.assertIsNotNone(hashing.node_key(e), e["path"]) + + def test_a_folder_lists_by_its_handle(self): + top = next(e for e in self.fs.listdir(self.fs.root_node) + if e["name"] == "Top") + kids = self.fs.listdir(nodes.node_of(top), top["path"]) + self.assertEqual(sorted(e["name"] for e in kids), ["Inner", "a.txt"]) + + def test_an_entry_can_be_found_again_from_its_handle(self): + deep = next(e for e in filesearch.collect(self.fs, self.fs.root_node) + if e["name"] == "deep.txt") + again = self.fs.entry_by_node(nodes.node_of(deep)) + self.assertEqual(again["path"], "/Top/Inner/deep.txt") + self.assertEqual(self.fs.read_file(again), b"deep") + self.assertIsNone(self.fs.entry_by_node(999999)) + + def test_the_server_rebuilds_a_readable_entry_from_a_bare_handle(self): + from engine import server + deep = next(e for e in filesearch.collect(self.fs, self.fs.root_node) + if e["name"] == "deep.txt") + entry = server._entry_from_body( + self.fs, {"node": str(nodes.node_of(deep)), "name": "wrong"}) + self.assertEqual(entry["name"], "deep.txt") + self.assertEqual(self.fs.read_file(entry), b"deep") + + +class Drift(unittest.TestCase): + """The rule is written in one place. A copy of it elsewhere is how HFS+ + was left out before, so a copy fails here rather than quietly going + stale.""" + + def source(self, path): + with open(os.path.join(ROOT, path), encoding="utf-8") as fh: + return fh.read() + + def test_no_python_module_spells_the_chain_out_again(self): + chain = re.compile( + r"get\(\"mft\"\)\s+if\s+\w+\.get\(\"mft\"\)\s+is\s+not\s+None" + r"|get\(\"oid\"\)\s+if\s+\w+\.get\(\"oid\"\)\s+is\s+not\s+None") + for name in sorted(os.listdir(os.path.join(ROOT, "engine"))): + if name.endswith(".py") and name != "nodes.py": + self.assertIsNone(chain.search(self.source("engine/" + name)), + name) + + def test_the_page_script_does_not_either(self): + js = self.source("web/app.js") + self.assertIsNone(re.search(r"\.mft\s*\?\?\s*\w+\.inode", js)) + self.assertIsNone(re.search(r"\.oid\s*\?\?\s*\w+\.mft", js)) + self.assertIn("'cnid'", js) + + def test_the_two_lists_of_keys_agree(self): + js = self.source("web/app.js") + found = re.search(r"const NODE_KEYS = \[([^\]]*)\]", js).group(1) + self.assertEqual(re.findall(r"'(\w+)'", found), list(nodes.KEYS)) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/app.js b/web/app.js index 12800c5..9540b2f 100644 --- a/web/app.js +++ b/web/app.js @@ -968,7 +968,7 @@ function marksHere() { if (!S.scope.file) return S.marks.filter(m => (m.frame || 'media') !== 'file'); const e = S.scope.entry; if (!e) return []; - const node = String(e.oid ?? e.mft ?? e.inode ?? e.start_cluster ?? e.path ?? ''); + const node = String(nodeValue(e) ?? e.path ?? ''); const stream = S.scope.stream || null; return S.marks.filter(m => m.frame === 'file' && String(m.node ?? '') === node @@ -1525,7 +1525,7 @@ async function listInto(holder, part, nodeId, depth, path, dirName) { const frag = document.createDocumentFragment(); for (const e of r.entries) { - const childId = e.mft ?? e.inode ?? e.oid ?? e.start_cluster; + const childId = nodeValue(e); frag.appendChild(node({ label: e.tree_label || e.name, meta: e.tree_label @@ -2177,7 +2177,7 @@ async function exportEntry(e, part, dest = null, stream = '', } const r = await api.post('export/file', { part: partOffset(part), entry: e, - node: e.mft ?? e.inode ?? e.oid ?? e.start_cluster, + node: nodeValue(e), name: e.name, path: e.path, size: e.size, dest, stream, add_exhibit: addExhibit, snap: part && part.snap != null ? part.snap : undefined }); @@ -2578,7 +2578,16 @@ async function loadHashMap(part, force = false) { hashMap = { ev: S.activeId, part, map: (r && r.map) || {} }; } -const nodeOf = e => String(e.mft ?? e.inode ?? e.oid ?? e.start_cluster ?? ''); +// The handle a filesystem reader gives an entry: the MFT record (NTFS), inode +// (ext), object ID (APFS, logical evidence), catalog node ID (HFS+) or first +// cluster (FAT, exFAT). Kept in step with engine/nodes.py. +const NODE_KEYS = ['mft', 'inode', 'oid', 'cnid', 'start_cluster']; +function nodeValue(e) { + for (const k of NODE_KEYS) if (e[k] != null) return e[k]; + return undefined; +} + +const nodeOf = e => String(nodeValue(e) ?? ''); const PAGE_ROWS = 400; @@ -2744,7 +2753,7 @@ async function dirUp() { async function listAllBelow(part, e = null) { if (!await useOwner(part)) return; - const node = e ? (e.mft ?? e.inode ?? e.oid ?? e.start_cluster) : null; + const node = e ? nodeValue(e) : null; const root = e ? (e.path || '/') : '/'; const label = e ? (e.name || root) : txt('ui.dir.at_root'); const scopeName = txt('ui.dir.scope_below', { name: label }); @@ -3174,7 +3183,7 @@ async function previewEntry(e, part, from = null, stream = null) { if (e.is_dir) { pvSet(e.name || txt('ui.preview.preview_title'), '', `
${txt('messages.folder_has_a_listing_not_a_preview')}
`); - const nodeId = e.mft ?? e.inode ?? e.oid ?? e.start_cluster; + const nodeId = nodeValue(e); if (!dirCache.has(dirKey(part.offset, nodeId, part.ev_id, part.snap))) { dirSet(e.name, 'reading…', `${txt('ui.reading_directory')}
`); @@ -4850,7 +4859,7 @@ $('#btn-attack-import')?.addEventListener('click', async () => { toast(txt('help.techniques_techniques_imported_attributions_already_made_unchanged', { techniques: r.techniques.toLocaleString() })); }); -const nodeIdOf = e => String(e.mft ?? e.inode ?? e.oid ?? e.start_cluster); +const nodeIdOf = e => String(nodeValue(e)); function tagsFor(entry) { const id = nodeIdOf(entry); @@ -5197,7 +5206,7 @@ const profileKey = (part, scope = S.scope) => { const base = `${S.activeId ?? '?'}:${part ?? 'image'}`; if (!scope || !scope.file || !scope.entry) return base; const e = scope.entry; - const node = e.oid ?? e.mft ?? e.inode ?? e.start_cluster ?? e.path; + const node = nodeValue(e) ?? e.path; return `${base}:file:${node}${scope.stream ? ':' + scope.stream : ''}`; }; let profileToken = null; @@ -9117,7 +9126,7 @@ function markFrame() { return { frame: 'file', part: S.scope.part, - node: String(e.oid ?? e.mft ?? e.inode ?? e.start_cluster ?? e.path ?? ''), + node: String(nodeValue(e) ?? e.path ?? ''), stream: S.scope.stream || null, name: e.name || '', }; @@ -11039,8 +11048,7 @@ $('#dlg-tag').addEventListener('close', async () => { const a = await api.post('attack/tag', { part: +$('#dlg-tag').dataset.part, target_kind: 'file', - target_ref: String(entry.mft ?? entry.inode ?? entry.oid - ?? entry.start_cluster ?? entry.path ?? entry.name), + target_ref: String(nodeValue(entry) ?? entry.path ?? entry.name), technique: tech, technique_name: t?.name, tactic: t?.tactic, note: $('#tag-note').value.trim(), asserted: true, catalogue: attackState.catalogue?.version,