Read an HFS+ catalog and attributes file that continue in the extents overflow tree

CHANGELOG.md

@@ -31,6 +31,15 @@ records, not how the code changed.
 
 ### Fixed
 
+- **A large HFS+ volume could list as empty.** The volume header holds only
+  the first eight pieces of the catalog (the file that lists every folder and
+  file) and of the attributes file; on a volume where either is in more
+  pieces, the rest are recorded in the extents overflow file, which was not
+  consulted. A folder whose entries lay in a later piece showed "Empty
+  directory", the volume name and extended attributes could be missing, and
+  nothing said so. Both files are now read in full. Checked against a
+  synthetic volume built to the published layout and read by an independent
+  reader (libfshfs); not yet against a large volume written by macOS.
 - **Pressing Enter in a dialog cancelled it instead of submitting it.** In
   the BitLocker and LUKS unlock dialog, the examiner-name dialogs, New case,
   Open case, tagging, and every other dialog with a Cancel button, Enter in a

engine/fs/hfsplus.py

@@ -10,6 +10,8 @@
 SIG_HFSX = b"HX"
 VOLUME_HEADER_OFFSET = 1024
 CNID_ROOT_FOLDER = 2
+CNID_CATALOG_FILE = 4
+CNID_ATTRIBUTES_FILE = 8
 
 REC_FOLDER = 0x0001
 REC_FILE = 0x0002
@@ -163,11 +165,17 @@ def __init__(self, source):
             self.findings.append(
                 "HFSX volume — filename comparison may be case-sensitive.")
 
-        self.catalog = BTree(self, self.catalog_fork, "catalog")
         try:
             self.extents = BTree(self, self.extents_fork, "extents overflow")
         except ValueError:
             self.extents = None
+        # The volume header holds only a system file's first eight extents; a
+        # catalog or attributes file in more pieces than that continues in the
+        # extents overflow tree. Without the rest, nodes beyond the first
+        # pieces read as nothing and a listing comes up empty.
+        self._complete_fork(self.catalog_fork, CNID_CATALOG_FILE)
+        self._complete_fork(self.attributes_fork, CNID_ATTRIBUTES_FILE)
+        self.catalog = BTree(self, self.catalog_fork, "catalog")
         try:
             self.attrs = BTree(self, self.attributes_fork, "attributes")
         except ValueError:
@@ -176,6 +184,13 @@ def __init__(self, source):
         self._threads = None
         self._xattrs = None
 
+    def _complete_fork(self, fork, cnid):
+        """Add to `fork` the extents the volume header could not hold."""
+        if fork.total_blocks > sum(c for _s, c in fork.extents):
+            fork.extents = self._all_extents({
+                "cnid": cnid, "_data_extents": fork.extents,
+                "_data_blocks": fork.total_blocks})
+
     def _index(self):
         if self._by_parent is not None:
             return self._by_parent, self._threads

tests/imagebuild_hfsplus.py

@@ -105,8 +105,10 @@ def _pad2(n):
     return n + (n & 1)
 
 
-def btree(records, node_size, max_key_len, attributes):
-    """A B-tree file. `records` is [(sort key, key bytes, data bytes)]."""
+def btree(records, node_size, max_key_len, attributes, reverse_leaves=False):
+    """A B-tree file. `records` is [(sort key, key bytes, data bytes)]. With
+    `reverse_leaves` the leaves are numbered from the highest node down, so the
+    first leaf is the last node in the file, as it can be in a real tree."""
     records = sorted(records, key=lambda r: r[0])
     cap = node_size - 14
 
@@ -150,10 +152,15 @@ def fit(items):
     first_keys = []
     i = 0
     for g, group in enumerate(leaf_groups):
-        num = 1 + g
-        nodes[num] = pack(KIND_LEAF, 1, group,
-                          num + 1 if g + 1 < n_leaves else 0,
-                          num - 1 if g else 0)
+        if reverse_leaves:
+            num = n_leaves - g
+            flink = num - 1 if g + 1 < n_leaves else 0
+            blink = num + 1 if g else 0
+        else:
+            num = 1 + g
+            flink = num + 1 if g + 1 < n_leaves else 0
+            blink = num - 1 if g else 0
+        nodes[num] = pack(KIND_LEAF, 1, group, flink, blink)
         first_keys.append((records[i][1] if records else b"", num))
         i += len(group)
     depth, level = 1, first_keys
@@ -179,8 +186,10 @@ def fit(items):
     header[8] = KIND_HEADER
     struct.pack_into(">H", header, 10, 3)
     struct.pack_into(">HIIIIHHII", header, 14, depth, root if records else 0,
-                     len(records), 1 if records else 0,
-                     n_leaves if records else 0, node_size, max_key_len,
+                     len(records),
+                     (n_leaves if reverse_leaves else 1) if records else 0,
+                     (1 if reverse_leaves else n_leaves) if records else 0,
+                     node_size, max_key_len,
                      total, 0)
     header[14 + 39] = 0xCF                      # key compare: case folding
     struct.pack_into(">I", header, 14 + 40, attributes)
@@ -209,12 +218,19 @@ def _fork(size, start, blocks):
     return struct.pack(">QII", size, 0, blocks) + ext.ljust(64, b"\x00")
 
 
+def _fork_extents(size, extents, blocks):
+    """A fork with up to eight extents (start, count) held inline."""
+    ext = b"".join(struct.pack(">II", a, n) for a, n in extents[:8])
+    return struct.pack(">QII", size, 0, blocks) + ext.ljust(64, b"\x00")
+
+
 def _perms(flags, special):
     return struct.pack(">IIBBHI", 501, 20, 0, flags, 0o100644, special)
 
 
 def build(entries, block_size=4096, node_size=4096, name="TESTVOL",
-          private_files=(), private_dirs=(), unmounted=True):
+          private_files=(), private_dirs=(), unmounted=True, fragment=(),
+          reverse_leaves=False):
     """The volume image (bytes). `entries` are the root folder's children;
     `private_files` are File objects whose catalog ID is their inode number
     (kept in the private data folder, with their link counts set from the
@@ -341,10 +357,24 @@ def take(nbytes):
         cat.append((cat_sort(parent, f.name), cat_key(parent, f.name), data))
         cat.append((cat_sort(f.cnid, ""), cat_key(f.cnid, ""),
                     struct.pack(">HHI", 4, 0, parent) + uname(f.name)))
-    cat_tree = btree(cat, node_size, 516, 0x6)
+    cat_tree = btree(cat, node_size, 516, 0x6, reverse_leaves=reverse_leaves)
     attr_tree = btree(attrs, node_size, 266, 0x6) if attrs else None
-    cat_ext = take(len(cat_tree))
-    attr_ext = take(len(attr_tree)) if attr_tree else (0, 0)
+
+    def place(tree, scatter):
+        """[(start block, count)] for a system file of `tree`'s size."""
+        if not tree:
+            return []
+        if not scatter:
+            start, count = take(len(tree))
+            return [(start, count)]
+        out = []
+        for _ in range(blocks_for(len(tree))):
+            out.append((take(block_size)[0], 1))
+            take(block_size)              # a gap, so no two are adjacent
+        return out
+
+    cat_ext = place(cat_tree, "catalog" in fragment)
+    attr_ext = place(attr_tree, "attributes" in fragment)
     take(1024)                  # room for the alternate volume header
     total_blocks = cursor[0]
 
@@ -355,10 +385,34 @@ def take(nbytes):
         bitmap[n >> 3] |= 0x80 >> (n & 7)
     img[alloc[0] * block_size:alloc[0] * block_size + len(bitmap)] = bitmap
     img[ext[0] * block_size:ext[0] * block_size + len(extents_tree)] = extents_tree
-    img[cat_ext[0] * block_size:cat_ext[0] * block_size + len(cat_tree)] = cat_tree
+    def write(tree, extents):
+        at = 0
+        for start, count in extents:
+            piece = tree[at:at + count * block_size]
+            img[start * block_size:start * block_size + len(piece)] = piece
+            at += count * block_size
+
+    write(cat_tree, cat_ext)
     if attr_tree:
-        at = attr_ext[0] * block_size
-        img[at:at + len(attr_tree)] = attr_tree
+        write(attr_tree, attr_ext)
+
+    # Extents beyond the eighth go in the extents overflow tree, keyed by the
+    # system file's catalog ID (catalog 4, attributes 8) and the file block
+    # each run starts at.
+    overflow = []
+    for cnid, extents in ((4, cat_ext), (8, attr_ext)):
+        done = sum(n for _a, n in extents[:8])
+        for i in range(8, len(extents), 8):
+            group = extents[i:i + 8]
+            key = struct.pack(">HBBII", 10, 0, 0, cnid, done)
+            rec = b"".join(struct.pack(">II", a, n) for a, n in group)
+            overflow.append(((cnid, 0, done), key, rec.ljust(64, b"\x00")))
+            done += sum(n for _a, n in group)
+    if overflow:
+        extents_tree = btree(overflow, node_size, 10, 0)
+        assert len(extents_tree) <= ext[1] * block_size
+        img[ext[0] * block_size:ext[0] * block_size
+            + len(extents_tree)] = extents_tree
     for f in files:
         d, r = plan[id(f)]
         if f.data:
@@ -376,8 +430,11 @@ def take(nbytes):
     struct.pack_into(">I", vh, 64, max(taken) + 1)
     vh[112:192] = _fork(alloc[1] * block_size, *alloc)
     vh[192:272] = _fork(len(extents_tree), *ext)
-    vh[272:352] = _fork(len(cat_tree), *cat_ext)
-    vh[352:432] = _fork(len(attr_tree) if attr_tree else 0, *attr_ext)
+    vh[272:352] = _fork_extents(
+        len(cat_tree), cat_ext, sum(n for _a, n in cat_ext))
+    vh[352:432] = _fork_extents(
+        len(attr_tree) if attr_tree else 0, attr_ext,
+        sum(n for _a, n in attr_ext))
     img[1024:1536] = vh
     img[len(img) - 1024:len(img) - 512] = vh
     return bytes(img)

tests/test_hfsplus_fragmented.py

@@ -0,0 +1,101 @@
+"""An HFS+ catalog or attributes file in more pieces than the volume header
+can hold: its first eight extents are in the header and the rest in the
+extents overflow tree. The builder scatters the file a block at a time and
+numbers the catalog's leaves from the top down, so the first leaf lies in the
+last piece. libfshfs, an independent reader, lists the same volume."""
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import imagebuild_hfsplus as hb                               # noqa: E402
+from engine import volume                                     # noqa: E402
+from engine.ewf import OffsetReader                           # noqa: E402
+from engine.fs import hfsplus                                 # noqa: E402
+
+
+class Mem(object):
+    bytes_per_sector = 512
+
+    def __init__(self, data):
+        self.data, self.size = data, len(data)
+
+    def read_at(self, offset, length):
+        if offset < 0 or offset >= self.size:
+            return b""
+        return self.data[offset:offset + length]
+
+
+def entries():
+    return [hb.Dir("Top", [hb.File("f%d.txt" % i, b"data%d" % i,
+                                   xattrs={"user.k": b"value%d" % i})
+                           for i in range(250)])]
+
+
+def open_fs(data):
+    return hfsplus.HfsPlus(OffsetReader(Mem(data), 0, len(data)))
+
+
+class Fragmented(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.data = hb.build(entries(), name="Frag Vol",
+                            fragment=("catalog", "attributes"),
+                            reverse_leaves=True)
+        cls.fs = open_fs(cls.data)
+
+    def test_the_catalog_really_has_more_extents_than_the_header_holds(self):
+        header_only = hfsplus.Fork(self.data, 1024 + 272)
+        self.assertEqual(len(header_only.extents), 8)
+        self.assertGreater(header_only.total_blocks, 8)
+        self.assertGreater(len(self.fs.catalog_fork.extents), 8)
+
+    def test_the_root_and_a_folder_list_in_full(self):
+        root = self.fs.listdir(hfsplus.CNID_ROOT_FOLDER)
+        self.assertEqual([e["name"] for e in root], ["Top"])
+        top = self.fs.listdir(root[0]["cnid"], "/Top")
+        self.assertEqual(len(top), 250)
+
+    def test_a_file_in_it_reads_back(self):
+        root = self.fs.listdir(hfsplus.CNID_ROOT_FOLDER)
+        top = {e["name"]: e for e in self.fs.listdir(root[0]["cnid"], "/Top")}
+        self.assertEqual(self.fs.read_file(top["f17.txt"]), b"data17")
+
+    def test_attributes_in_a_scattered_attributes_file_are_found(self):
+        root = self.fs.listdir(hfsplus.CNID_ROOT_FOLDER)
+        top = {e["name"]: e for e in self.fs.listdir(root[0]["cnid"], "/Top")}
+        info = self.fs.stat(top["f200.txt"])
+        self.assertEqual(info["xattrs"][0]["name"], "user.k")
+
+    def test_the_volume_name_is_read(self):
+        self.assertEqual(self.fs.volume_name(), "Frag Vol")
+        self.assertEqual(volume.volume_label(Mem(self.data), 0, "HFS+"),
+                         "Frag Vol")
+
+    def test_a_contiguous_volume_is_unchanged(self):
+        plain = open_fs(hb.build(entries(), name="Frag Vol"))
+        self.assertEqual(len(plain.catalog_fork.extents), 1)
+        root = plain.listdir(hfsplus.CNID_ROOT_FOLDER)
+        self.assertEqual(len(plain.listdir(root[0]["cnid"], "/Top")), 250)
+
+
+class Damaged(unittest.TestCase):
+
+    def test_missing_overflow_records_give_a_partial_listing_not_an_error(self):
+        data = bytearray(hb.build(entries(), fragment=("catalog",)))
+        import struct
+        # Blank the extents overflow file's header-held extent: its records
+        # are then unreadable, as if the tree were lost.
+        data[1024 + 192 + 16:1024 + 192 + 80] = bytes(64)
+        fs = open_fs(bytes(data))
+        root = fs.listdir(hfsplus.CNID_ROOT_FOLDER)
+        self.assertIsInstance(root, list)
+        self.assertLessEqual(len(fs.catalog_fork.extents), 8)
+
+
+if __name__ == "__main__":
+    unittest.main()