diff --git a/CHANGELOG.md b/CHANGELOG.md index 448c8a3..9e0c6ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,15 @@ records, not how the code changed. ### Fixed +- **A large HFS+ volume could list as empty.** The volume header holds only + the first eight pieces of the catalog (the file that lists every folder and + file) and of the attributes file; on a volume where either is in more + pieces, the rest are recorded in the extents overflow file, which was not + consulted. A folder whose entries lay in a later piece showed "Empty + directory", the volume name and extended attributes could be missing, and + nothing said so. Both files are now read in full. Checked against a + synthetic volume built to the published layout and read by an independent + reader (libfshfs); not yet against a large volume written by macOS. - **Pressing Enter in a dialog cancelled it instead of submitting it.** In the BitLocker and LUKS unlock dialog, the examiner-name dialogs, New case, Open case, tagging, and every other dialog with a Cancel button, Enter in a diff --git a/engine/fs/hfsplus.py b/engine/fs/hfsplus.py index 443744c..46fea14 100644 --- a/engine/fs/hfsplus.py +++ b/engine/fs/hfsplus.py @@ -10,6 +10,8 @@ SIG_HFSX = b"HX" VOLUME_HEADER_OFFSET = 1024 CNID_ROOT_FOLDER = 2 +CNID_CATALOG_FILE = 4 +CNID_ATTRIBUTES_FILE = 8 REC_FOLDER = 0x0001 REC_FILE = 0x0002 @@ -163,11 +165,17 @@ def __init__(self, source): self.findings.append( "HFSX volume — filename comparison may be case-sensitive.") - self.catalog = BTree(self, self.catalog_fork, "catalog") try: self.extents = BTree(self, self.extents_fork, "extents overflow") except ValueError: self.extents = None + # The volume header holds only a system file's first eight extents; a + # catalog or attributes file in more pieces than that continues in the + # extents overflow tree. Without the rest, nodes beyond the first + # pieces read as nothing and a listing comes up empty. + self._complete_fork(self.catalog_fork, CNID_CATALOG_FILE) + self._complete_fork(self.attributes_fork, CNID_ATTRIBUTES_FILE) + self.catalog = BTree(self, self.catalog_fork, "catalog") try: self.attrs = BTree(self, self.attributes_fork, "attributes") except ValueError: @@ -176,6 +184,13 @@ def __init__(self, source): self._threads = None self._xattrs = None + def _complete_fork(self, fork, cnid): + """Add to `fork` the extents the volume header could not hold.""" + if fork.total_blocks > sum(c for _s, c in fork.extents): + fork.extents = self._all_extents({ + "cnid": cnid, "_data_extents": fork.extents, + "_data_blocks": fork.total_blocks}) + def _index(self): if self._by_parent is not None: return self._by_parent, self._threads diff --git a/tests/imagebuild_hfsplus.py b/tests/imagebuild_hfsplus.py index 111897b..ad09ae8 100644 --- a/tests/imagebuild_hfsplus.py +++ b/tests/imagebuild_hfsplus.py @@ -105,8 +105,10 @@ def _pad2(n): return n + (n & 1) -def btree(records, node_size, max_key_len, attributes): - """A B-tree file. `records` is [(sort key, key bytes, data bytes)].""" +def btree(records, node_size, max_key_len, attributes, reverse_leaves=False): + """A B-tree file. `records` is [(sort key, key bytes, data bytes)]. With + `reverse_leaves` the leaves are numbered from the highest node down, so the + first leaf is the last node in the file, as it can be in a real tree.""" records = sorted(records, key=lambda r: r[0]) cap = node_size - 14 @@ -150,10 +152,15 @@ def fit(items): first_keys = [] i = 0 for g, group in enumerate(leaf_groups): - num = 1 + g - nodes[num] = pack(KIND_LEAF, 1, group, - num + 1 if g + 1 < n_leaves else 0, - num - 1 if g else 0) + if reverse_leaves: + num = n_leaves - g + flink = num - 1 if g + 1 < n_leaves else 0 + blink = num + 1 if g else 0 + else: + num = 1 + g + flink = num + 1 if g + 1 < n_leaves else 0 + blink = num - 1 if g else 0 + nodes[num] = pack(KIND_LEAF, 1, group, flink, blink) first_keys.append((records[i][1] if records else b"", num)) i += len(group) depth, level = 1, first_keys @@ -179,8 +186,10 @@ def fit(items): header[8] = KIND_HEADER struct.pack_into(">H", header, 10, 3) struct.pack_into(">HIIIIHHII", header, 14, depth, root if records else 0, - len(records), 1 if records else 0, - n_leaves if records else 0, node_size, max_key_len, + len(records), + (n_leaves if reverse_leaves else 1) if records else 0, + (1 if reverse_leaves else n_leaves) if records else 0, + node_size, max_key_len, total, 0) header[14 + 39] = 0xCF # key compare: case folding struct.pack_into(">I", header, 14 + 40, attributes) @@ -209,12 +218,19 @@ def _fork(size, start, blocks): return struct.pack(">QII", size, 0, blocks) + ext.ljust(64, b"\x00") +def _fork_extents(size, extents, blocks): + """A fork with up to eight extents (start, count) held inline.""" + ext = b"".join(struct.pack(">II", a, n) for a, n in extents[:8]) + return struct.pack(">QII", size, 0, blocks) + ext.ljust(64, b"\x00") + + def _perms(flags, special): return struct.pack(">IIBBHI", 501, 20, 0, flags, 0o100644, special) def build(entries, block_size=4096, node_size=4096, name="TESTVOL", - private_files=(), private_dirs=(), unmounted=True): + private_files=(), private_dirs=(), unmounted=True, fragment=(), + reverse_leaves=False): """The volume image (bytes). `entries` are the root folder's children; `private_files` are File objects whose catalog ID is their inode number (kept in the private data folder, with their link counts set from the @@ -341,10 +357,24 @@ def take(nbytes): cat.append((cat_sort(parent, f.name), cat_key(parent, f.name), data)) cat.append((cat_sort(f.cnid, ""), cat_key(f.cnid, ""), struct.pack(">HHI", 4, 0, parent) + uname(f.name))) - cat_tree = btree(cat, node_size, 516, 0x6) + cat_tree = btree(cat, node_size, 516, 0x6, reverse_leaves=reverse_leaves) attr_tree = btree(attrs, node_size, 266, 0x6) if attrs else None - cat_ext = take(len(cat_tree)) - attr_ext = take(len(attr_tree)) if attr_tree else (0, 0) + + def place(tree, scatter): + """[(start block, count)] for a system file of `tree`'s size.""" + if not tree: + return [] + if not scatter: + start, count = take(len(tree)) + return [(start, count)] + out = [] + for _ in range(blocks_for(len(tree))): + out.append((take(block_size)[0], 1)) + take(block_size) # a gap, so no two are adjacent + return out + + cat_ext = place(cat_tree, "catalog" in fragment) + attr_ext = place(attr_tree, "attributes" in fragment) take(1024) # room for the alternate volume header total_blocks = cursor[0] @@ -355,10 +385,34 @@ def take(nbytes): bitmap[n >> 3] |= 0x80 >> (n & 7) img[alloc[0] * block_size:alloc[0] * block_size + len(bitmap)] = bitmap img[ext[0] * block_size:ext[0] * block_size + len(extents_tree)] = extents_tree - img[cat_ext[0] * block_size:cat_ext[0] * block_size + len(cat_tree)] = cat_tree + def write(tree, extents): + at = 0 + for start, count in extents: + piece = tree[at:at + count * block_size] + img[start * block_size:start * block_size + len(piece)] = piece + at += count * block_size + + write(cat_tree, cat_ext) if attr_tree: - at = attr_ext[0] * block_size - img[at:at + len(attr_tree)] = attr_tree + write(attr_tree, attr_ext) + + # Extents beyond the eighth go in the extents overflow tree, keyed by the + # system file's catalog ID (catalog 4, attributes 8) and the file block + # each run starts at. + overflow = [] + for cnid, extents in ((4, cat_ext), (8, attr_ext)): + done = sum(n for _a, n in extents[:8]) + for i in range(8, len(extents), 8): + group = extents[i:i + 8] + key = struct.pack(">HBBII", 10, 0, 0, cnid, done) + rec = b"".join(struct.pack(">II", a, n) for a, n in group) + overflow.append(((cnid, 0, done), key, rec.ljust(64, b"\x00"))) + done += sum(n for _a, n in group) + if overflow: + extents_tree = btree(overflow, node_size, 10, 0) + assert len(extents_tree) <= ext[1] * block_size + img[ext[0] * block_size:ext[0] * block_size + + len(extents_tree)] = extents_tree for f in files: d, r = plan[id(f)] if f.data: @@ -376,8 +430,11 @@ def take(nbytes): struct.pack_into(">I", vh, 64, max(taken) + 1) vh[112:192] = _fork(alloc[1] * block_size, *alloc) vh[192:272] = _fork(len(extents_tree), *ext) - vh[272:352] = _fork(len(cat_tree), *cat_ext) - vh[352:432] = _fork(len(attr_tree) if attr_tree else 0, *attr_ext) + vh[272:352] = _fork_extents( + len(cat_tree), cat_ext, sum(n for _a, n in cat_ext)) + vh[352:432] = _fork_extents( + len(attr_tree) if attr_tree else 0, attr_ext, + sum(n for _a, n in attr_ext)) img[1024:1536] = vh img[len(img) - 1024:len(img) - 512] = vh return bytes(img) diff --git a/tests/test_hfsplus_fragmented.py b/tests/test_hfsplus_fragmented.py new file mode 100644 index 0000000..5de6e8d --- /dev/null +++ b/tests/test_hfsplus_fragmented.py @@ -0,0 +1,101 @@ +"""An HFS+ catalog or attributes file in more pieces than the volume header +can hold: its first eight extents are in the header and the rest in the +extents overflow tree. The builder scatters the file a block at a time and +numbers the catalog's leaves from the top down, so the first leaf lies in the +last piece. libfshfs, an independent reader, lists the same volume.""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import imagebuild_hfsplus as hb # noqa: E402 +from engine import volume # noqa: E402 +from engine.ewf import OffsetReader # noqa: E402 +from engine.fs import hfsplus # noqa: E402 + + +class Mem(object): + bytes_per_sector = 512 + + def __init__(self, data): + self.data, self.size = data, len(data) + + def read_at(self, offset, length): + if offset < 0 or offset >= self.size: + return b"" + return self.data[offset:offset + length] + + +def entries(): + return [hb.Dir("Top", [hb.File("f%d.txt" % i, b"data%d" % i, + xattrs={"user.k": b"value%d" % i}) + for i in range(250)])] + + +def open_fs(data): + return hfsplus.HfsPlus(OffsetReader(Mem(data), 0, len(data))) + + +class Fragmented(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.data = hb.build(entries(), name="Frag Vol", + fragment=("catalog", "attributes"), + reverse_leaves=True) + cls.fs = open_fs(cls.data) + + def test_the_catalog_really_has_more_extents_than_the_header_holds(self): + header_only = hfsplus.Fork(self.data, 1024 + 272) + self.assertEqual(len(header_only.extents), 8) + self.assertGreater(header_only.total_blocks, 8) + self.assertGreater(len(self.fs.catalog_fork.extents), 8) + + def test_the_root_and_a_folder_list_in_full(self): + root = self.fs.listdir(hfsplus.CNID_ROOT_FOLDER) + self.assertEqual([e["name"] for e in root], ["Top"]) + top = self.fs.listdir(root[0]["cnid"], "/Top") + self.assertEqual(len(top), 250) + + def test_a_file_in_it_reads_back(self): + root = self.fs.listdir(hfsplus.CNID_ROOT_FOLDER) + top = {e["name"]: e for e in self.fs.listdir(root[0]["cnid"], "/Top")} + self.assertEqual(self.fs.read_file(top["f17.txt"]), b"data17") + + def test_attributes_in_a_scattered_attributes_file_are_found(self): + root = self.fs.listdir(hfsplus.CNID_ROOT_FOLDER) + top = {e["name"]: e for e in self.fs.listdir(root[0]["cnid"], "/Top")} + info = self.fs.stat(top["f200.txt"]) + self.assertEqual(info["xattrs"][0]["name"], "user.k") + + def test_the_volume_name_is_read(self): + self.assertEqual(self.fs.volume_name(), "Frag Vol") + self.assertEqual(volume.volume_label(Mem(self.data), 0, "HFS+"), + "Frag Vol") + + def test_a_contiguous_volume_is_unchanged(self): + plain = open_fs(hb.build(entries(), name="Frag Vol")) + self.assertEqual(len(plain.catalog_fork.extents), 1) + root = plain.listdir(hfsplus.CNID_ROOT_FOLDER) + self.assertEqual(len(plain.listdir(root[0]["cnid"], "/Top")), 250) + + +class Damaged(unittest.TestCase): + + def test_missing_overflow_records_give_a_partial_listing_not_an_error(self): + data = bytearray(hb.build(entries(), fragment=("catalog",))) + import struct + # Blank the extents overflow file's header-held extent: its records + # are then unreadable, as if the tree were lost. + data[1024 + 192 + 16:1024 + 192 + 80] = bytes(64) + fs = open_fs(bytes(data)) + root = fs.listdir(hfsplus.CNID_ROOT_FOLDER) + self.assertIsInstance(root, list) + self.assertLessEqual(len(fs.catalog_fork.extents), 8) + + +if __name__ == "__main__": + unittest.main()