Pick an entry's handle in one place, so HFS+ folders can be opened, searched and tagged

CHANGELOG.md

@@ -44,6 +44,17 @@ records, not how the code changed.
 
 ### Fixed
 
+- **Opening a folder on an HFS+ volume showed the volume's top level again,
+  and search, hashing, indexing and tagging skipped HFS+ folders.** Each
+  reader names its entries by its own key (the MFT record for NTFS, the inode
+  for ext, the catalog node ID for HFS+), and most of the code looked for
+  every key but HFS+'s. Expanding a folder in the tree listed the volume root
+  instead of its contents (nested without end), a search, file-type check,
+  hash or index run never went below the top folder, and an HFS+ file had no
+  key to tag or hash it under. The key is now chosen in one place. A tagged
+  HFS+ file can also be exported, which it could not be before: it is looked
+  up again from its key. Checked in a browser on a synthetic HFS+ volume
+  built with nested folders; the volume itself is unchanged.
 - **A cut-off name in the evidence tree can be read in full.** The tree is
   narrow, so a long line such as a partition's "volume label ..." detail was
   shown with an ellipsis and the rest was unreachable. Hovering over a row now

engine/artifacts.py

@@ -1,8 +1,4 @@
-def _node_of(e):
-    return (e.get("mft") if e.get("mft") is not None
-            else e.get("inode") if e.get("inode") is not None
-            else e.get("oid") if e.get("oid") is not None
-            else e.get("start_cluster"))
+from .nodes import node_of as _node_of
 
 def _safe_listdir(fs, node, path):
     try:

engine/casedb.py

@@ -14,6 +14,7 @@
 from . import fuzzyhash
 from . import version as version_mod
 from .text import t as _t
+from .nodes import node_of
 
 SCHEMA = """
 CREATE TABLE IF NOT EXISTS meta (
@@ -1633,10 +1634,7 @@ def remove_bookmark(self, bid):
 
     @_writes
     def tag_item(self, evidence_id, item, tag, note="", part=0):
-        node = str(item.get("mft") if item.get("mft") is not None
-                   else item.get("inode") if item.get("inode") is not None
-                   else item.get("oid") if item.get("oid") is not None
-                   else item.get("start_cluster"))
+        node = str(node_of(item))
         cur = self.db.execute(
             "INSERT INTO tagged_items (evidence_id,part,node,path,name,size,"
             "is_dir,deleted,tag,note,created_at,examiner,modified,accessed,"
@@ -1782,10 +1780,7 @@ def save_search(self, evidence_id, name, query, hits, part=None):
             "deleted,where_found,term,context,file_offset,encoding,modified) "
             "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
             [(sid, h.get("name"), h.get("path"),
-              str(h.get("mft") if h.get("mft") is not None
-                  else h.get("inode") if h.get("inode") is not None
-                  else h.get("oid") if h.get("oid") is not None
-                  else h.get("start_cluster")),
+              str(node_of(h)),
               h.get("size"), int(bool(h.get("is_dir"))),
               int(bool(h.get("deleted"))), h.get("where"), h.get("term"),
               h.get("context"), h.get("file_offset"), h.get("encoding"),

engine/filesearch.py

@@ -1,6 +1,7 @@
 import re
 
 from . import profile
+from .nodes import node_of
 from .text import t as _t
 
 DEFAULT_SCAN_BYTES = 64 << 20
@@ -45,10 +46,7 @@ def _walk(fs, root_node, path, out, depth, max_depth, seen, progress=None,
         if progress is not None and len(out) % PROGRESS_EVERY == 1:
             progress(len(out))
         if e.get("is_dir"):
-            node = (e.get("mft") if e.get("mft") is not None
-                    else e.get("inode") if e.get("inode") is not None
-                    else e.get("oid") if e.get("oid") is not None
-                    else e.get("start_cluster"))
+            node = node_of(e)
             if node is None or node in seen:
                 continue
             seen.add(node)
@@ -325,7 +323,7 @@ def _hit(e, where, context, term=None, offset=None, encoding=None):
         "modified": e.get("modified"), "created": e.get("created"),
         "accessed": e.get("accessed"),
         "mft": e.get("mft"), "inode": e.get("inode"), "oid": e.get("oid"),
-        "start_cluster": e.get("start_cluster"),
+        "cnid": e.get("cnid"), "start_cluster": e.get("start_cluster"),
         "where": where, "term": term, "context": context,
         "file_offset": offset, "encoding": encoding,
         "entry": e,

engine/filetype.py

@@ -1,5 +1,7 @@
 import struct
 
+from .nodes import node_of
+
 SIGNATURES = [
     (0, b"\xFF\xD8\xFF", "jpeg"),
     (0, b"\x89PNG\r\n\x1a\n", "png"),
@@ -280,10 +282,7 @@ def scan(fs, entries, read=None, progress=None, limit=None):
                 "name": e.get("name"), "path": e.get("path"),
                 "size": size, "deleted": bool(e.get("deleted")),
                 "modified": e.get("modified"),
-                "node": (e.get("mft") if e.get("mft") is not None
-                         else e.get("inode") if e.get("inode") is not None
-                         else e.get("oid") if e.get("oid") is not None
-                         else e.get("start_cluster")),
+                "node": node_of(e),
                 **{k: got[k] for k in ("extension", "extension_says",
                                        "detected", "content_is", "why",
                                        "severity")},

engine/fs/hfsplus.py

@@ -224,8 +224,36 @@ def _index(self):
         self._by_parent, self._threads = by_parent, threads
         self._resolve_links(by_parent)
         self._resolve_compression(by_parent)
+        self._by_cnid = {rec["cnid"]: rec for recs in by_parent.values()
+                         for rec in recs}
         return by_parent, threads
 
+    def _path_of(self, cnid):
+        """The path of the file or folder with this catalog ID, built from
+        the parent links of its thread records; None if the chain is broken
+        or loops."""
+        _by_parent, threads = self._index()
+        parts, seen = [], set()
+        while cnid != CNID_ROOT_FOLDER:
+            if cnid in seen or cnid not in self._by_cnid:
+                return None
+            seen.add(cnid)
+            rec = self._by_cnid[cnid]
+            parts.append(rec["name"])
+            cnid = rec["parent"]
+        return "/" + "/".join(reversed(parts))
+
+    def entry_by_node(self, cnid):
+        """The entry listdir() would give for the file or folder with this
+        catalog ID, or None. For a caller that kept only the handle."""
+        self._index()
+        rec = self._by_cnid.get(int(cnid))
+        if rec is None:
+            return None
+        out = dict(rec)
+        out["path"] = self._path_of(rec["cnid"]) or "/" + rec["name"]
+        return out
+
     def _private_children(self, by_parent, name, prefix):
         """{number: record} of the entries in the private folder `name`
         (at the volume root) whose names are `prefix` and a number."""

engine/hashing.py

@@ -11,16 +11,7 @@
 
 ALGO_BY_LEN = {32: "md5", 40: "sha1", 64: "sha256"}
 
-def node_of(entry):
-    for key in ("mft", "inode", "oid", "start_cluster"):
-        v = entry.get(key)
-        if v is not None:
-            return v
-    return None
-
-def node_key(entry):
-    v = node_of(entry)
-    return None if v is None else str(v)
+from .nodes import node_of, node_key  # noqa: F401  (re-exported)
 
 CHUNK = 4 << 20
 

engine/nodes.py

@@ -0,0 +1,27 @@
+"""The handle a filesystem reader gives each entry it lists.
+
+Every reader names an entry by one field of its own: the MFT record number
+(NTFS), the inode (ext), the object ID (APFS, and logical evidence), the
+catalog node ID (HFS+) or the first cluster (FAT and exFAT). Anything that
+goes on to list a folder, open a file by handle, key a tag or a hash, or
+cross-reference an entry has to pick whichever one is there. That choice was
+once written out at each place that made it, and HFS+ was left out of most of
+them: its folders were never descended into and nothing keyed on its entries
+could be kept. It is made here, once."""
+
+KEYS = ("mft", "inode", "oid", "cnid", "start_cluster")
+
+
+def node_of(entry):
+    """The entry's handle, or None if it has none."""
+    for key in KEYS:
+        value = entry.get(key)
+        if value is not None:
+            return value
+    return None
+
+
+def node_key(entry):
+    """The handle as a string, for keying; None if the entry has none."""
+    value = node_of(entry)
+    return None if value is None else str(value)

engine/recyclebin.py

@@ -74,11 +74,7 @@ def parse_info2(data):
         })
     return out
 
-def _node_of(e):
-    return (e.get("mft") if e.get("mft") is not None
-            else e.get("inode") if e.get("inode") is not None
-            else e.get("oid") if e.get("oid") is not None
-            else e.get("start_cluster"))
+from .nodes import node_of as _node_of
 
 def scan(fs, listdir, root_node, progress=None):
     out = []

engine/server.py

@@ -15,6 +15,7 @@
 from urllib.parse import urlparse, parse_qs
 
 from . import carve as carve_mod
+from .nodes import node_of
 from . import volmap as volmap_mod
 from . import nativedialog
 from . import attack as attack_mod
@@ -196,10 +197,7 @@ def __init__(self, fs, entry, size, stream="", cache=None):
 
     def _key(self):
         e = self.entry
-        node = (e.get("oid") if e.get("oid") is not None else
-                e.get("mft") if e.get("mft") is not None else
-                e.get("inode") if e.get("inode") is not None else
-                e.get("start_cluster"))
+        node = node_of(e)
         return (id(self.fs), str(node),
                 e.get("path") or e.get("name") or "", self.stream)
 
@@ -595,8 +593,7 @@ def region(self, offset, size=None, ev=None):
         return OffsetReader(ev.image, offset, size or (ev.image.size - offset))
 
     def hive(self, offset, entry, ev=None):
-        key = (offset, entry.get("mft") or entry.get("inode")
-               or entry.get("oid") or entry.get("start_cluster"), entry.get("name"))
+        key = (offset, node_of(entry), entry.get("name"))
         ev = ev or self.current
         if key in ev.hive_cache:
             return ev.hive_cache[key]
@@ -3677,7 +3674,7 @@ def run(progress):
                         "name": e.get("name"), "path": e.get("path"),
                         "size": e.get("size"), "deleted": e.get("deleted"),
                         "mft": e.get("mft"), "inode": e.get("inode"),
-                        "oid": e.get("oid"),
+                        "oid": e.get("oid"), "cnid": e.get("cnid"),
                         "start_cluster": e.get("start_cluster")}
                     for m in r["messages"]:
                         m["store"] = e.get("path")
@@ -4590,6 +4587,14 @@ def _entry_from_body(fs, body):
     walking the FAT (#94)."""
     node = body.get("node")
     n = None if node in (None, "", "null") else int(node)
+    lookup = getattr(fs, "entry_by_node", None)
+    if lookup is not None and n is not None:
+        # A reader whose entries carry what is needed to read them (HFS+
+        # keeps a file's extents in its catalog record) is asked for the
+        # entry itself, rather than given a bare handle it cannot read.
+        found = lookup(n)
+        if found is not None:
+            return found
     entry = {"name": body.get("name"), "path": body.get("path"),
              "size": body.get("size"), "is_dir": False,
              "deleted": bool(body.get("deleted")),
@@ -4604,6 +4609,8 @@ def _entry_from_body(fs, body):
         entry["inode"] = n
     elif fsname.startswith("APFS") or fsname.startswith("LOGICAL"):
         entry["oid"] = n
+    elif fsname.startswith("HFS"):
+        entry["cnid"] = n
     else:
         entry["start_cluster"] = n
     return entry
@@ -4894,11 +4901,7 @@ def _resolve_path(fs, path):
             return hit
         if not hit.get("is_dir"):
             return None
-        node = (hit.get("mft") if hit.get("mft") is not None
-                else hit.get("inode") if hit.get("inode") is not None
-                else hit.get("oid") if hit.get("oid") is not None
-                else hit.get("cnid") if hit.get("cnid") is not None
-                else hit.get("start_cluster"))
+        node = node_of(hit)
         here = hit.get("path") or (here + "/" + want)
     return None
 

engine/textindex.py

@@ -5,6 +5,7 @@
 import zlib
 
 from . import filesearch
+from .nodes import node_of
 from . import entropy as entropy_mod
 from . import officedoc as officedoc_mod
 
@@ -464,10 +465,7 @@ def on_entry(e):
             return
         if len(body) >= max_text:
             counts["text_capped"] += 1
-        node = (e.get("mft") if e.get("mft") is not None
-                else e.get("inode") if e.get("inode") is not None
-                else e.get("oid") if e.get("oid") is not None
-                else e.get("start_cluster"))
+        node = node_of(e)
         rows.append((e.get("name") or "", e.get("path") or "", body,
                      str(node), part, e.get("size") or 0,
                      int(bool(e.get("deleted"))), e.get("modified") or "",

engine/timeline.py

@@ -45,11 +45,7 @@ def _parse(ts):
         return None
     return t if t.tzinfo else t.replace(tzinfo=_UTC)
 
-def _node_id(entry):
-    for key in ("mft", "inode", "oid", "cnid", "start_cluster"):
-        if entry.get(key) is not None:
-            return entry[key]
-    return None
+from .nodes import node_of as _node_id
 
 def root_of(fs):
     root = getattr(fs, "root_node", None)