diff --git a/CHANGELOG.md b/CHANGELOG.md index dbb305a..527f379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ records, not how the code changed. ## [Unreleased] +### Added + +- **LZFSE-compressed files are read on HFS+ and APFS.** Files macOS + compresses with LZFSE (the method newer macOS versions prefer) were listed + with a note and a volume finding but their content was not shown. They are + now decompressed like zlib and LZVN ones. The decoder was checked against + streams written by Apple's own reference encoder; only files stored raw + (compression types 9 and 10) are still listed without content. A cached + HFS+ or APFS filesystem tree is rebuilt the next time it is opened, so + these files show their size. Tested on synthetic volumes only. + ## [0.8.0] - 2026-10-01 Adds RAID 0, 1 and 5 assembly, QCOW2 and VirtualBox VDI disk images, diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d8138f6..946d4d6 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -84,9 +84,10 @@ recognised and refused by name today; these entries are about *reading* it. fork) are decompressed on HFS+ and APFS; HFS+ file hard links read as the file they stand for, and directory hard links are listed with their target. Tested on synthetic volumes (an independent reader, - libfshfs, confirmed the builder's hard-link layout). Not done: LZFSE - compressed files (listed, contents not shown), APFS hard links, and - **HFS** (the original, not HFS+). + libfshfs, confirmed the builder's hard-link layout). LZFSE is read too, + checked against Apple's reference encoder. Not done: files stored raw + (types 9 and 10), APFS hard links, and **HFS** (the original, not + HFS+). - [~] **APFS snapshots** — snapshots are listed and can be opened as a read-only view of the volume. Tested on synthetic images only so far. diff --git a/engine/decmpfs.py b/engine/decmpfs.py index 1d1f3f4..a1cf61e 100644 --- a/engine/decmpfs.py +++ b/engine/decmpfs.py @@ -17,12 +17,12 @@ 7 LZVN attribute 8 LZVN resource fork 9/10 stored raw (attribute / resource fork; not read here) - 11/12 LZFSE (attribute / resource fork; not read here) + 11/12 LZFSE (attribute / resource fork) The numbering and the block tables follow libfshfs and Apple's published -LZFSE sources; only zlib and LZVN are decoded. A block that is stored -uncompressed starts with a marker byte (0xFF for zlib, 0x06 for LZVN) and -carries the data after it.""" +LZFSE sources; zlib, LZVN and LZFSE are decoded. A block that is stored +uncompressed starts with a marker byte (0xFF for zlib and LZFSE, 0x06 for +LZVN) and carries the data after it.""" import struct import zlib @@ -45,7 +45,7 @@ 9: (CODEC_RAW, False), 10: (CODEC_RAW, True), 11: (CODEC_LZFSE, False), 12: (CODEC_LZFSE, True), } -SUPPORTED = (CODEC_ZLIB, CODEC_LZVN) +SUPPORTED = (CODEC_ZLIB, CODEC_LZVN, CODEC_LZFSE) class DecmpfsError(ValueError): @@ -212,7 +212,18 @@ def _lzvn_block(data, expected): return lzvn_decode(data, expected) -_BLOCK_DECODERS = {CODEC_ZLIB: _zlib_block, CODEC_LZVN: _lzvn_block} +def _lzfse_block(data, expected): + if data[:1] == b"\xff": # stored: 0xFF, then the data + return bytes(data[1:1 + expected]) + from . import lzfse # (it uses this module's LZVN) + try: + return lzfse.decode(data, max_size=max(expected, 1))[:expected] + except lzfse.LzfseError as exc: + raise DecmpfsError(str(exc)) + + +_BLOCK_DECODERS = {CODEC_ZLIB: _zlib_block, CODEC_LZVN: _lzvn_block, + CODEC_LZFSE: _lzfse_block} class Compressed: diff --git a/engine/lzfse.py b/engine/lzfse.py new file mode 100644 index 0000000..563e978 --- /dev/null +++ b/engine/lzfse.py @@ -0,0 +1,367 @@ +"""LZFSE decompression, as used by Apple for compressed files (decmpfs types +11 and 12) and for disk images (DMG "ULFO"). + +An LZFSE stream is a series of blocks, each starting with a four-byte magic: + + bvx$ end of the stream + bvx- raw bytes follow (a 32-bit count, then the bytes) + bvxn LZVN-compressed bytes (raw size, payload size, then the payload) + bvx1 LZFSE block, frequency tables stored in full + bvx2 LZFSE block, frequency tables packed + +An LZFSE block holds literals, coded four at a time with finite state +entropy (FSE), and a list of (literal length, match length, distance) +triplets, coded with three more FSE streams. Both are read backwards from the +end of their payload. A match may reach back into any earlier block. + +The layouts and tables follow Apple's published reference implementation +(github.com/lzfse/lzfse); the decoder here was checked against its output. +Output is capped (`max_size`) so a damaged or hostile stream cannot grow +without bound.""" + +import struct + +from . import decmpfs +from .text import t as _t + +MAGIC_END = b"bvx$" +MAGIC_RAW = b"bvx-" +MAGIC_V1 = b"bvx1" +MAGIC_V2 = b"bvx2" +MAGIC_LZVN = b"bvxn" + +L_SYMBOLS, M_SYMBOLS, D_SYMBOLS, LIT_SYMBOLS = 20, 20, 64, 256 +L_STATES, M_STATES, D_STATES, LIT_STATES = 64, 64, 256, 1024 +MATCHES_PER_BLOCK = 10000 +LITERALS_PER_BLOCK = 4 * MATCHES_PER_BLOCK +V1_HEADER = 770 +V2_FIXED = 32 + +L_EXTRA = (0,) * 16 + (2, 3, 5, 8) +L_BASE = tuple(range(16)) + (16, 20, 28, 60) +M_EXTRA = (0,) * 16 + (3, 5, 8, 11) +M_BASE = tuple(range(16)) + (16, 24, 56, 312) +D_EXTRA = tuple(b for b in range(16) for _ in range(4)) +D_BASE = ( + 0, 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 36, 44, 52, 60, 76, 92, 108, + 124, 156, 188, 220, 252, 316, 380, 444, 508, 636, 764, 892, 1020, 1276, + 1532, 1788, 2044, 2556, 3068, 3580, 4092, 5116, 6140, 7164, 8188, 10236, + 12284, 14332, 16380, 20476, 24572, 28668, 32764, 40956, 49148, 57340, + 65532, 81916, 98300, 114684, 131068, 163836, 196604, 229372) + +_FREQ_BITS = (2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14, + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14) +_FREQ_VALUE = (0, 2, 1, 4, 0, 3, 1, -1, 0, 2, 1, 5, 0, 3, 1, -1, + 0, 2, 1, 6, 0, 3, 1, -1, 0, 2, 1, 7, 0, 3, 1, -1) + + +class LzfseError(ValueError): + pass + + +def _bad(key, *args): + return LzfseError(_t(key) % args if args else _t(key)) + + +# -- header --------------------------------------------------------------------- + +def _freq_value(bits): + """(value, bits consumed) for one packed frequency.""" + low = bits & 31 + n = _FREQ_BITS[low] + if n == 8: + return 8 + ((bits >> 4) & 0xF), n + if n == 14: + return 24 + ((bits >> 4) & 0x3FF), n + return _FREQ_VALUE[low], n + + +def _unpack_v2(data, at): + """The fields of a v2 header at `at`, as a dict like the v1 header, and + the header's size.""" + if at + V2_FIXED > len(data): + raise _bad("lzfse.truncated") + n_raw = struct.unpack_from(" len(data): + raise _bad("lzfse.truncated") + h = { + "n_raw": n_raw, + "n_literals": v0 & 0xFFFFF, + "n_literal_payload": (v0 >> 20) & 0xFFFFF, + "n_matches": (v0 >> 40) & 0xFFFFF, + "literal_bits": ((v0 >> 60) & 7) - 7, + "literal_state": [(v1 >> s) & 0x3FF for s in (0, 10, 20, 30)], + "n_lmd_payload": (v1 >> 40) & 0xFFFFF, + "lmd_bits": ((v1 >> 60) & 7) - 7, + "l_state": (v2 >> 32) & 0x3FF, + "m_state": (v2 >> 42) & 0x3FF, + "d_state": (v2 >> 52) & 0x3FF, + } + total = L_SYMBOLS + M_SYMBOLS + D_SYMBOLS + LIT_SYMBOLS + freq = [0] * total + src, end = at + V2_FIXED, at + size + if src != end: + accum = nbits_in = 0 + for i in range(total): + while src < end and nbits_in + 8 <= 32: + accum |= data[src] << nbits_in + nbits_in += 8 + src += 1 + value, used = _freq_value(accum) + if used > nbits_in: + raise _bad("lzfse.header_damaged") + freq[i] = value + accum >>= used + nbits_in -= used + if nbits_in >= 8 or src != end: + raise _bad("lzfse.header_damaged") + _split_freq(h, freq) + return h, size + + +def _split_freq(h, freq): + a, b, c = L_SYMBOLS, L_SYMBOLS + M_SYMBOLS, L_SYMBOLS + M_SYMBOLS + D_SYMBOLS + h["l_freq"], h["m_freq"] = freq[:a], freq[a:b] + h["d_freq"], h["literal_freq"] = freq[b:c], freq[c:] + + +def _unpack_v1(data, at): + if at + V1_HEADER > len(data): + raise _bad("lzfse.truncated") + (n_raw, _n_payload, n_literals, n_matches, n_lit_payload, n_lmd_payload, + literal_bits) = struct.unpack_from(" LITERALS_PER_BLOCK or h["n_matches"] > MATCHES_PER_BLOCK: + raise _bad("lzfse.header_damaged") + if any(s >= LIT_STATES for s in h["literal_state"]) \ + or h["l_state"] >= L_STATES or h["m_state"] >= M_STATES \ + or h["d_state"] >= D_STATES: + raise _bad("lzfse.header_damaged") + for key, states in (("l_freq", L_STATES), ("m_freq", M_STATES), + ("d_freq", D_STATES), ("literal_freq", LIT_STATES)): + if sum(h[key]) > states: + raise _bad("lzfse.header_damaged") + if h["n_literals"] % 4: + raise _bad("lzfse.header_damaged") + + +# -- finite state entropy -------------------------------------------------------- + +def _decoder_table(nstates, freq): + """[(bits to read, symbol, delta)] per state.""" + n_clz = 32 - nstates.bit_length() + table = [] + for sym, f in enumerate(freq): + if not f: + continue + k = (32 - f.bit_length()) - n_clz + j0 = ((2 * nstates) >> k) - f + for j in range(f): + if j < j0: + table.append((k, sym, ((f + j) << k) - nstates)) + else: + table.append((k - 1, sym, (j - j0) << (k - 1))) + table.extend([(0, 0, 0)] * (nstates - len(table))) + return table + + +def _value_table(nstates, freq, vbits, vbase): + """[(total bits, value bits, delta, base)] per state.""" + n_clz = 32 - nstates.bit_length() + table = [] + for sym, f in enumerate(freq): + if not f: + continue + k = (32 - f.bit_length()) - n_clz + j0 = ((2 * nstates) >> k) - f + for j in range(f): + if j < j0: + table.append((k + vbits[sym], vbits[sym], + ((f + j) << k) - nstates, vbase[sym])) + else: + table.append((k - 1 + vbits[sym], vbits[sym], + (j - j0) << (k - 1), vbase[sym])) + table.extend([(0, 0, 0, 0)] * (nstates - len(table))) + return table + + +class _Reader: + """Bits taken from the end of data[start:end], going backwards.""" + + def __init__(self, data, end, start, bits): + self.data, self.pos, self.start = data, end, start + if bits: + if self.pos < start + 4: + raise _bad("lzfse.truncated") + self.pos -= 4 + self.accum = int.from_bytes(data[self.pos:self.pos + 4], "little") + self.nbits = bits + 32 + else: + if self.pos < start + 3: + raise _bad("lzfse.truncated") + self.pos -= 3 + self.accum = int.from_bytes(data[self.pos:self.pos + 3], "little") + self.nbits = 24 + if self.nbits < 24 or self.nbits >= 32 or self.accum >> self.nbits: + raise _bad("lzfse.header_damaged") + + def flush(self): + take = (63 - self.nbits) & -8 + if not take: + return + pos = self.pos - (take >> 3) + if pos < self.start: + raise _bad("lzfse.truncated") + self.pos = pos + incoming = int.from_bytes(self.data[pos:pos + (take >> 3)], "little") + self.accum = (self.accum << take) | incoming + self.nbits += take + + def pull(self, n): + self.nbits -= n + value = self.accum >> self.nbits + self.accum &= (1 << self.nbits) - 1 + return value + + +def _literals(data, at, h, start): + """The block's literals, decoded from the payload at data[at:...].""" + table = _decoder_table(LIT_STATES, h["literal_freq"]) + end = at + h["n_literal_payload"] + if end > len(data): + raise _bad("lzfse.truncated") + rd = _Reader(data, end, start, h["literal_bits"]) + states = list(h["literal_state"]) + out = bytearray(h["n_literals"]) + for i in range(0, h["n_literals"], 4): + rd.flush() + for j in range(4): + k, sym, delta = table[states[j]] + states[j] = delta + rd.pull(k) + if states[j] >= LIT_STATES: + raise _bad("lzfse.stream_damaged") + out[i + j] = sym + return bytes(out) + + +def _block(data, at, h, header_size, out, limit): + """Decode one LZFSE block into `out`; returns the offset after it.""" + _check_header(h) + lit_at = at + header_size + lmd_at = lit_at + h["n_literal_payload"] + end = lmd_at + h["n_lmd_payload"] + if end > len(data): + raise _bad("lzfse.truncated") + literals = _literals(data, lit_at, h, 0) + l_t = _value_table(L_STATES, h["l_freq"], L_EXTRA, L_BASE) + m_t = _value_table(M_STATES, h["m_freq"], M_EXTRA, M_BASE) + d_t = _value_table(D_STATES, h["d_freq"], D_EXTRA, D_BASE) + rd = _Reader(data, end, lmd_at, h["lmd_bits"]) + ls, ms, ds = h["l_state"], h["m_state"], h["d_state"] + lit = 0 + d_prev = -1 + block_start = len(out) + for _ in range(h["n_matches"]): + rd.flush() + tb, vb, delta, base = l_t[ls] + sv = rd.pull(tb) + ls = delta + (sv >> vb) + length = base + (sv & ((1 << vb) - 1)) + tb, vb, delta, base = m_t[ms] + sv = rd.pull(tb) + ms = delta + (sv >> vb) + match = base + (sv & ((1 << vb) - 1)) + tb, vb, delta, base = d_t[ds] + sv = rd.pull(tb) + ds = delta + (sv >> vb) + dist = base + (sv & ((1 << vb) - 1)) + if dist: + d_prev = dist + if ls >= L_STATES or ms >= M_STATES or ds >= D_STATES: + raise _bad("lzfse.stream_damaged") + if lit + length > len(literals): + raise _bad("lzfse.stream_damaged") + out += literals[lit:lit + length] + lit += length + if match: + if d_prev <= 0 or d_prev > len(out): + raise _bad("lzfse.stream_damaged") + start = len(out) - d_prev + if d_prev >= match: + out += out[start:start + match] + else: + pattern = bytes(out[start:]) + out += (pattern * -(-match // d_prev))[:match] + if len(out) > limit or len(out) - block_start > h["n_raw"]: + raise _bad("lzfse.too_large") + if len(out) - block_start != h["n_raw"]: + raise _bad("lzfse.stream_damaged") + return end + + +def decode(data, max_size=1 << 30): + """The bytes an LZFSE stream decodes to. Raises LzfseError for a stream + that is damaged, truncated, lacks its end marker or would decode to more + than `max_size` bytes.""" + data = bytes(data) + out = bytearray() + at = 0 + while True: + if at + 4 > len(data): + raise _bad("lzfse.truncated") + magic = data[at:at + 4] + if magic == MAGIC_END: + return bytes(out) + if magic == MAGIC_RAW: + if at + 8 > len(data): + raise _bad("lzfse.truncated") + n = struct.unpack_from(" len(data): + raise _bad("lzfse.truncated") + if len(out) + n > max_size: + raise _bad("lzfse.too_large") + out += data[at + 8:at + 8 + n] + at += 8 + n + elif magic == MAGIC_LZVN: + if at + 12 > len(data): + raise _bad("lzfse.truncated") + n_raw, n_payload = struct.unpack_from(" len(data): + raise _bad("lzfse.truncated") + if len(out) + n_raw > max_size: + raise _bad("lzfse.too_large") + try: + got = decmpfs.lzvn_decode(data[at + 12:at + 12 + n_payload], + n_raw) + except decmpfs.DecmpfsError as exc: + raise LzfseError(str(exc)) + if len(got) != n_raw: + raise _bad("lzfse.stream_damaged") + out += got + at += 12 + n_payload + elif magic in (MAGIC_V1, MAGIC_V2): + if magic == MAGIC_V1: + h, size = _unpack_v1(data, at) + else: + h, size = _unpack_v2(data, at) + if len(out) + h["n_raw"] > max_size: + raise _bad("lzfse.too_large") + at = _block(data, at, h, size, out, max_size) + else: + raise _bad("lzfse.bad_magic") diff --git a/engine/treecache.py b/engine/treecache.py index 08b2ac6..121bd44 100644 --- a/engine/treecache.py +++ b/engine/treecache.py @@ -2,7 +2,7 @@ import os import sqlite3 -VERSION = 3 +VERSION = 4 SCHEMA = """ CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT); diff --git a/tests/fuzz.py b/tests/fuzz.py index fa2ba3a..b3fb8e8 100644 --- a/tests/fuzz.py +++ b/tests/fuzz.py @@ -177,6 +177,19 @@ def _logfile_seed(): return b.logfile(specs)[0] +def run_lzfse(data): + from engine import lzfse + try: + lzfse.decode(data, max_size=1 << 22) + except lzfse.LzfseError: + pass + + +def _lzfse_seed(): + import lzfse_vectors + return lzfse_vectors.unpack(lzfse_vectors.CHAIN)[0] + + def run_ewf(data, _dir=[], name="case.E01"): from engine import ewf if not _dir: @@ -405,6 +418,7 @@ def targets(): run_reglog) out["blockfile"] = (_blockfile_seed, run_blockfile) out["logfile"] = (_logfile_seed, run_logfile) + out["lzfse"] = (_lzfse_seed, run_lzfse) out["pst"] = (lambda: _pst_seed(False, attachment=b"z" * 9000), run_pst) out["pst-ansi"] = (lambda: _pst_seed(True, attachment=b"z" * 9000), run_pst) diff --git a/tests/lzfse_vectors.py b/tests/lzfse_vectors.py new file mode 100644 index 0000000..48fa15c --- /dev/null +++ b/tests/lzfse_vectors.py @@ -0,0 +1,195 @@ +"""LZFSE streams written by Apple's reference encoder (liblzfse), embedded so +the tests need no dependency. Each is a pair of base64 strings: the compressed +stream and its plain text (zlib-packed to keep this file small). V2 is a +`bvx2` block with literals and matches, LZVN a `bvxn` block, RAW a `bvx-` +block, and CHAIN the three blocks in one stream.""" + +import base64 +import zlib + +V2 = (( + 'YnZ4MtUXAADYAfANABgDMPdrHz3+3QVQpgAAAD0UUABvwAcwUBARABBwnp0ezSoIEAFrE1fb' + 'k7nnzT44uLrZN2dHJyd3NzcHR7OrAgAAAADAAAAAAAAAAAAuAQAAgOMv4ml4MR6KV+Kv+CPe' + 'iZcCAAAAAAAAAAAAuBzkhozAHsmA3AAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + 'AAAAAAAAAADAN+jkcA1wbLo0LPOR6cTjPS7lEvY9gUd/qbB75Mlbq+6wyhBhrp8IBXcTNlNx' + 'IwA8NrWoUI4cFXSixEvjWHuZ0heZDRKkWMSxeXWXmhW2W6dwqa/MkWdLa4TPN8ZPeSWtwhS2' + '61WjsptKIdu+JvRxbVi23MWRASlx2PL/qFnPREHrrFRvIhRHro8J+de8FyvvVu2wGFqWuSrb' + 'grscUqsLOQHyXc+XxdAwK8S+i+31iQOTnj3cCbzWy4ravM07R+XTwW3TY08h8dS8le+ypBPI' + 'V3niNYgieiQHUgcAAAAAAAAAAAAAUBIAkrskEn6N/Dd0ixa8HBxB/mbFM53Xg1m0g1zw1/Kq' + 'xp0e4khRNEDZEe48c4ogQ0nVPEwYpJoEnI2YJ6OfUWVR3lJIeDHVF5luHc2M03npDZTRBiGk' + '1I1CKkI8KKIIssA/yxtOxtZzlma8lMz18qqvw5DwlDuI/+pklgbLU06Suu5gXouOSTxM+eRR' + 'gsNKbVnCnHX2sIPY7UbPhmJPXQyJtcC4mBwl2B6S8OgjazevgG+h2k8nUwVPm6N7+Y3kewy8' + 'Y4DbOSNNxEIUHuTx78mo73LVSglJXsdeG4V1so/tE2jYaG5qcvRHK7pT05fHi8y+xKLBqTqM' + '/AjJjgvcUVxvVJkK5a0lt+M8xJQr0JFaVOZttFHVuHNUhNDC8SJMqalXI0hO4FgMNO1G3YVP' + 'yS2Ee28plaKDvpty89Qez511LdffyfbGGQADjK1nd4R61X00n1ZwKXYhqgA2/K2DNq7Zj1GO' + '0OqvEC0ZessT/9CUn2H3x46Gix5Fbe5W9QnqFqWdMt/UrTN9vnIVWqoHNc9zZUgKSB3LAo+a' + 'xRjmAS0wIRBCOaXkK8It9L8Pk2hMcRx63vbeo736EhtXBxQJL685Gls6lyqPMB13dOnIknAX' + 'VJZxBgDTCDk+iXsGfoBcC0MdlO0FRbNsBhQdKaEYO0Meucr2LfglssrD7vEN5LAeatwqw+yo' + 'hPfghJpNHAe8WnG36uOKZWIfVcEiqE1wGgaU9yyXuyqdxaig/Cl06Gj1viPQtNcF9jkplwFE' + 'VsLOHHUjt5HWvCdHoEQydNTCP4I1KjaPPCdtpTDcMVOCuEMDtNbpdcNy9kYg86cYFDvxYZHD' + '5ORBud0tpk2XO0j5hyHzyI1qdayW6A5CSQTk8nta20u7M3xBo51A/XXXLpWqZ/oeuJlCXNaW' + 'fbKhEbgikqpbOipdMQg/wJpiBuXMDioMzhLG5UgfwubJvJ9xNCOApj1HwLrMF8pTYOH7plvl' + '8W0kTQn/3pMQWGtOEjrFMD//ZZZaGgCGcN5hj5HMJwRjTUMHCkdO4PxMLLuTKoY3MoAUggQN' + '1nWEN/Jtb2X+Q1YPGyaOhYXbYGRbsGCwIlfYCWLC4G1Tl7k/uesqOyWMVkF/tmrsS3RDDu/y' + 'oPffmMlwgCMTyXFomfeEbdgpIWew8+1KhhjPHtOd59QyCfaS34vh5mBXnRbtKyx2MAiISVht' + 'hFzLh1UDIim0Zvddh+NJRTV4HiE87vxGhn5WnUJMBSz4CMlbmASrTrfwBRkoWrqhtWvhJ7cZ' + '0nAjGJFcTskTlNLUUPR5CgZ6H2sr94uOJGF+68767hQ1ucgWf0PQRAVbAcCVhshrdmKGPj98' + 'U2EPP7DO6wRw1ytHeikRQMVRjmFzunUyHSvMnNkXFDpLat3PXADeXVGEo97iM2D6SV0Jzvqo' + '3bsY1+/upVhWGrQi5iWF+PS9XLrUPMSRAV9PhP/I4T7RljCamWJLXQjewQTIpqPnH9VqDfKw' + 'wfIpy6h5BW4tsGRSXUWBCFnjSF4kK/tqdTOaaTxVJCgqauGXfgo6rrs8u0BW7SVYy2Rxzwnw' + '8uXowBNiYgv8jiX9Vxh+EXHDX+laIxYSisbuM8HeljcUoqGbrGP8hFLa4N3xLYjFR57v5kgv' + 'u5jYwjCkUMYAbFj0XPoZprJ9m1aIMcEWbVdkf9ZKys7NZ7Byl29DXWqcT8wRIMDLKEeV2fBb' + 'T3ZQmXoBwpeIA3g0RejCG67jhmgEkpwH+KsmqmM8gslQGfxTvUWxnRYEDCENkZiv+JavniM2' + 'NIFnXc1uOc78mQg8ZafexLgkAZ4u2ykKIqniOAfTTdasOjhXXR4uRel2liIAVBaCS+yBeqW7' + 'OLNjB39h0L9OXDHimFwdC6+SyWK1I3hqwn5X3n1pKdSV4j+pOyY4RZLDeuw0qp8C7w6tLUb7' + 'Lsm4yXTJQI2U7IDHZpBwd9IYYLSFlR6rVXWZz4LiTLgPqxNXqpqZGkDVCpBqAjg7QB3AvqU+' + 'YnZ4JA=='), +( + 'eNptWFmuIzcMvIqP0Nql4zwjRibATDJA3lefPm3VIrWRL7s3iSySxaK+f7y+v1p4fL9/S3v8' + '8fr5/RUTrvvx+PPr16+vEHGdOq/D4+vn7x9frT7w1bzbBu7W9ni+F8VVzFjzendej4G1Mvds' + 'Bc9rxW8Pj9fvf//6+c/fNeGLnud6petBifPGOHQjH1yj6E6CCXixhPmT2tvazkVlHl2utKpP' + 'j/z5fHaZrI3wII3Hef2E97sD3gaucxk1L7F6obORKwngkQDZtdH75cxN5zN71exNdFjeVmHD' + 'ghUu+xEagjX6NA2+DtgSGAFGIsw3Mj8M9LzRpDzwq9hfvuLrTtNlDDHvc6cKiAutCMApOxzH' + '+6UUp/3cMDFtBoIUGH+ulJgWkWmzBx+fC2KaWadXtTlDT0QFYY1M0cKo87YRzk7o5/LmitJK' + 'MNgKL9tcfPCj6xfIJex5YPExXTa4WH/ezMxBRjIgZTodnyANrJWZP4WYVRhQCUoaMhde8ZKb' + 'XivN21ek6UborjW4A9CuAsPqzMA8E7uv9CvTJAYozoBH5FnkSgZamDiKeKwdnNxY5UICRAND' + 'RjA33NKnsViv13QH+CRSTmd8CxeI2dWwMnC+URXBJ/htbkvHinEDOoR1CHyGTcWcMlKOBVNY' + 'EAG0MMOo4ioE/toQIQF6BTzSiXtAJYiS8tjJRgWeM7EBUpMSlps1mtamn9kmz41YHVVgVAf+' + 'CSoDSxlm8VT1nTQzWGSghlDHFhhEYObMBdnMyOyEnzWBNjGbhzIarneux7QoE8RE3qqHGWuC' + 'x3BVVGMgiKKgRNM7U0TkN0SCKm0wRWLmBNo/AMnqKIMs2icA07A+Iy3KqSvGMFedhfkREceI' + 'zG2wWt1xDKcA6EyEw4aDPhHZJ7DeRFgZFZiUDZHOVS3yBP7YxT4SElx2t5fnVtLRtdqxWGMU' + 'VeuIHMiiTFQutBgZtfxz9cJILhCBZ4ZR3alZa8C44Ca/OnFFyQ0z2kUw863i+gcVA4XR2aH1' + 'UB2MZH5C0wAF7VtMpnK/bO0fIc40SoF/blzdoxOFRbU1gIDqHQxIIhZFVMOc6WAPx97OztSZ' + 'kNSqrqFdn3gGL4a7OcoxuIGgazL2xYvXxuYsX/2vulaa//XV2s20Ehlg5OBov7dFOMr2n9Wa' + 'zbm4BjzxnplslFCbXVmqcCFtmHpM3l4eN8mIUs18KumRxCj2hgmWHHIx3ZKl7oPZOSkhBM9V' + 'kUYobawnmlqMiTTpNK0dZpETrV1KBT0B7FHThx4xbwNlhlBJQHd7kjZEaOBro7weUi/PjVx7' + 's/JHmLf+TcJean0uLvbokhYbHVnzR6s5Z1mhoOiUP6DDQ58j2NPBNqsvZikkKTvsG1V1sJtf' + 'RtovMl4WgMntBUktSr7MK2CSTFDoF4hjFCOhnmgwwFBQpcjIFIMZUvDtLOfwUaDsanrVIwLy' + 'lsWFrFZXz8G6AW1p0QaWPLa2/tzGuSFQyBVMuVtbZduSNpGIb9WpB8tm3UWj2w+PJyI6pgWF' + 'K74OVi1iLZWfujVMatb+r9UxhApHn7CCWRdMvgM46fFhUYDQInrJ1oEdjo85TJNap8giBpmw' + 'A8R9OFWKg9AXixZmG6lcHXDyRSfCAznYCSDEJS1La1aMmpFRptFDmwjvtXMOZGPzxCH9u4RP' + 'jlv6wBBka2lM1yew5uyG4kv3eNI/aZNsqoa2e1rAVlaveueQDuwcLk98AecSJei5jYEpSU9x' + 'LpDSQa8g7z1fGyb2UJyrXtLS/UiiRTO7xNo+8ldUIfiTzLG6ZzFHq+XccyDCuVYNDvyhzcjI' + 'yILnfKowS+707FkdDqKaaK4FF8kIAVShnq81D6Z0E5HVyXQbrjQkH4oKhsM1gaBz4CSl7Acx' + 'JDVYIKGeFcHG9HtZOa5VG9ZJ4dbwKkrOG3KPjwYLtUbgOVjXsSs4NB0a6RFb8Ziru9LS6usI' + 'TRQJaBpBXZwwiarGnyASxzZ9SOmANphyqVk0n2j/pGYRGnzsliJI0NvxFLh4LEksxfdERmGj' + 'oZakmVaHMkaYpueduXVqpxbSdV1sIUUFwWFoppn2vq5TnSyrPlSWWp2aWNQxFUlpKgI2Gckc' + 'QKaesUk71ZDOJJAW+Pb4HPTUsuqwsr2dtEVa5ImXxz/Dsy7wXvR5O5sM2UoEyw8dUCxIEFRE' + 'sy3BqJM7RbZvuXnTG00VqoFLR5RogtUYAhDmoJrkog8dqd2Iqx44YtRxqvJWZ03FzVLWgmTr' + '5zS1Zjcdp51bbWKy9HGRdALOmiRN4PVSEzeNsqZFnSBUTW5xcYUr438mIJ9s69xsJVXZx1MK' + 'h7xUxrkfDfN453ayFyaKeeU9XUydEkcqU4fpMLyoClTErzVL1fEftMWzwQ==')) + +LZVN = (( + 'YnZ4bugDAAAsAAAA4AR0aGUgcXVpY2sgYnJvd24gZm94IDgU8P/w//D/8Irjb3ggBgAAAAAA' + 'AABidngk'), +( + 'eNoryUhVKCzNTM5WSCrKL89TSMuvUCgZFRsVGxUbVmIAhXRuRg==')) + +RAW = (( + 'YnZ4LSwBAAB5Qr3yIQbwhHdi8PPLTXZNxwcgURWaD4nyxtrK40S7MRJF/W+E35rXxbPQdqwO' + 'j1OnNWyIkT8g9vctsCLSTQqW2tQ8FhfBqY54Ep4DJzcQZdCVhk8VraC4RsHA68U0itx5mt+E' + 'm60F1KEKwEQequ60tI76Cx8KvYDpmKNaul6gvYeZwTUNQ55xiXqnX94xNKSqcuBWKKxv5opz' + 'PRFhoV2OriuwQteViu2x1ZTW0RLTT2YC9N5xEOmTrnQikj19FxFl3BkG9j1XmXoK0xs6rkCB' + '9B+0cWU+PVd6jEED+cwZin+J2BrypQAcQBc/GSP3ECz6oVChJLPFx5u4h2Go2z9BAcIoWxW/' + '68IW3Bu+/qHX1usJfW+KJNly2kIOpr+GPu0/wDejNALySXjHFi9idngk'), +( + 'eNoBLAHT/nlCvfIhBvCEd2Lw88tNdk3HByBRFZoPifLG2srjRLsxEkX9b4TfmtfFs9B2rA6P' + 'U6c1bIiRPyD29y2wItJNCpba1DwWF8GpjngSngMnNxBl0JWGTxWtoLhGwcDrxTSK3Hma34Sb' + 'rQXUoQrARB6q7rS0jvoLHwq9gOmYo1q6XqC9h5nBNQ1DnnGJeqdf3jE0pKpy4FYorG/minM9' + 'EWGhXY6uK7BC15WK7bHVlNbREtNPZgL03nEQ6ZOudCKSPX0XEWXcGQb2PVeZegrTGzquQIH0' + 'H7RxZT49V3qMQQP5zBmKf4nYGvKlABxAFz8ZI/cQLPqhUKEks8XHm7iHYajbP0EBwihbFb/r' + 'whbcG77+odfW6wl9b4ok2XLaQg6mv4Y+7T/AN6M0AvJJeMcWL18mk70=')) + +CHAIN = (( + 'YnZ4MtUXAADYAfANABgDMPdrHz3+3QVQpgAAAD0UUABvwAcwUBARABBwnp0ezSoIEAFrE1fb' + 'k7nnzT44uLrZN2dHJyd3NzcHR7OrAgAAAADAAAAAAAAAAAAuAQAAgOMv4ml4MR6KV+Kv+CPe' + 'iZcCAAAAAAAAAAAAuBzkhozAHsmA3AAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + 'AAAAAAAAAADAN+jkcA1wbLo0LPOR6cTjPS7lEvY9gUd/qbB75Mlbq+6wyhBhrp8IBXcTNlNx' + 'IwA8NrWoUI4cFXSixEvjWHuZ0heZDRKkWMSxeXWXmhW2W6dwqa/MkWdLa4TPN8ZPeSWtwhS2' + '61WjsptKIdu+JvRxbVi23MWRASlx2PL/qFnPREHrrFRvIhRHro8J+de8FyvvVu2wGFqWuSrb' + 'grscUqsLOQHyXc+XxdAwK8S+i+31iQOTnj3cCbzWy4ravM07R+XTwW3TY08h8dS8le+ypBPI' + 'V3niNYgieiQHUgcAAAAAAAAAAAAAUBIAkrskEn6N/Dd0ixa8HBxB/mbFM53Xg1m0g1zw1/Kq' + 'xp0e4khRNEDZEe48c4ogQ0nVPEwYpJoEnI2YJ6OfUWVR3lJIeDHVF5luHc2M03npDZTRBiGk' + '1I1CKkI8KKIIssA/yxtOxtZzlma8lMz18qqvw5DwlDuI/+pklgbLU06Suu5gXouOSTxM+eRR' + 'gsNKbVnCnHX2sIPY7UbPhmJPXQyJtcC4mBwl2B6S8OgjazevgG+h2k8nUwVPm6N7+Y3kewy8' + 'Y4DbOSNNxEIUHuTx78mo73LVSglJXsdeG4V1so/tE2jYaG5qcvRHK7pT05fHi8y+xKLBqTqM' + '/AjJjgvcUVxvVJkK5a0lt+M8xJQr0JFaVOZttFHVuHNUhNDC8SJMqalXI0hO4FgMNO1G3YVP' + 'yS2Ee28plaKDvpty89Qez511LdffyfbGGQADjK1nd4R61X00n1ZwKXYhqgA2/K2DNq7Zj1GO' + '0OqvEC0ZessT/9CUn2H3x46Gix5Fbe5W9QnqFqWdMt/UrTN9vnIVWqoHNc9zZUgKSB3LAo+a' + 'xRjmAS0wIRBCOaXkK8It9L8Pk2hMcRx63vbeo736EhtXBxQJL685Gls6lyqPMB13dOnIknAX' + 'VJZxBgDTCDk+iXsGfoBcC0MdlO0FRbNsBhQdKaEYO0Meucr2LfglssrD7vEN5LAeatwqw+yo' + 'hPfghJpNHAe8WnG36uOKZWIfVcEiqE1wGgaU9yyXuyqdxaig/Cl06Gj1viPQtNcF9jkplwFE' + 'VsLOHHUjt5HWvCdHoEQydNTCP4I1KjaPPCdtpTDcMVOCuEMDtNbpdcNy9kYg86cYFDvxYZHD' + '5ORBud0tpk2XO0j5hyHzyI1qdayW6A5CSQTk8nta20u7M3xBo51A/XXXLpWqZ/oeuJlCXNaW' + 'fbKhEbgikqpbOipdMQg/wJpiBuXMDioMzhLG5UgfwubJvJ9xNCOApj1HwLrMF8pTYOH7plvl' + '8W0kTQn/3pMQWGtOEjrFMD//ZZZaGgCGcN5hj5HMJwRjTUMHCkdO4PxMLLuTKoY3MoAUggQN' + '1nWEN/Jtb2X+Q1YPGyaOhYXbYGRbsGCwIlfYCWLC4G1Tl7k/uesqOyWMVkF/tmrsS3RDDu/y' + 'oPffmMlwgCMTyXFomfeEbdgpIWew8+1KhhjPHtOd59QyCfaS34vh5mBXnRbtKyx2MAiISVht' + 'hFzLh1UDIim0Zvddh+NJRTV4HiE87vxGhn5WnUJMBSz4CMlbmASrTrfwBRkoWrqhtWvhJ7cZ' + '0nAjGJFcTskTlNLUUPR5CgZ6H2sr94uOJGF+68767hQ1ucgWf0PQRAVbAcCVhshrdmKGPj98' + 'U2EPP7DO6wRw1ytHeikRQMVRjmFzunUyHSvMnNkXFDpLat3PXADeXVGEo97iM2D6SV0Jzvqo' + '3bsY1+/upVhWGrQi5iWF+PS9XLrUPMSRAV9PhP/I4T7RljCamWJLXQjewQTIpqPnH9VqDfKw' + 'wfIpy6h5BW4tsGRSXUWBCFnjSF4kK/tqdTOaaTxVJCgqauGXfgo6rrs8u0BW7SVYy2Rxzwnw' + '8uXowBNiYgv8jiX9Vxh+EXHDX+laIxYSisbuM8HeljcUoqGbrGP8hFLa4N3xLYjFR57v5kgv' + 'u5jYwjCkUMYAbFj0XPoZprJ9m1aIMcEWbVdkf9ZKys7NZ7Byl29DXWqcT8wRIMDLKEeV2fBb' + 'T3ZQmXoBwpeIA3g0RejCG67jhmgEkpwH+KsmqmM8gslQGfxTvUWxnRYEDCENkZiv+JavniM2' + 'NIFnXc1uOc78mQg8ZafexLgkAZ4u2ykKIqniOAfTTdasOjhXXR4uRel2liIAVBaCS+yBeqW7' + 'OLNjB39h0L9OXDHimFwdC6+SyWK1I3hqwn5X3n1pKdSV4j+pOyY4RZLDeuw0qp8C7w6tLUb7' + 'Lsm4yXTJQI2U7IDHZpBwd9IYYLSFlR6rVXWZz4LiTLgPqxNXqpqZGkDVCpBqAjg7QB3AvqU+' + 'YnZ4bugDAAAsAAAA4AR0aGUgcXVpY2sgYnJvd24gZm94IDgU8P/w//D/8Irjb3ggBgAAAAAA' + 'AABidngtLAEAAHlCvfIhBvCEd2Lw88tNdk3HByBRFZoPifLG2srjRLsxEkX9b4TfmtfFs9B2' + 'rA6PU6c1bIiRPyD29y2wItJNCpba1DwWF8GpjngSngMnNxBl0JWGTxWtoLhGwcDrxTSK3Hma' + '34SbrQXUoQrARB6q7rS0jvoLHwq9gOmYo1q6XqC9h5nBNQ1DnnGJeqdf3jE0pKpy4FYorG/m' + 'inM9EWGhXY6uK7BC15WK7bHVlNbREtNPZgL03nEQ6ZOudCKSPX0XEWXcGQb2PVeZegrTGzqu' + 'QIH0H7RxZT49V3qMQQP5zBmKf4nYGvKlABxAFz8ZI/cQLPqhUKEks8XHm7iHYajbP0EBwihb' + 'Fb/rwhbcG77+odfW6wl9b4ok2XLaQg6mv4Y+7T/AN6M0AvJJeMcWL2J2eCQ='), +( + 'eNrtWP/LnlUZLyWjRTYH21hGPpmQUNFzn3Puc58TzemyoB+GRZBQaLyPPaY0fd180/mAVDC0' + 'vcSiL0ZOEF+sLNFhuB/EmG7CHFQ/WJqrNTQIAhmGLFESiu73fD6f69zPS39B7Kfnub+dc12f' + '67o+1+c6SzdMlxa6ZrS0+tt2o69Ody4tOI/rNB59beGmmxYah2ufeN2MFnbecsNCF0f4qtzt' + 'Mu7GbjRZXRRXLmDN/t1ynTPWCtyza/E8RvymZjS95dYbdy7eHD2+SKGs1yY9aF25kce6EcZc' + 'o9UdDxPwYtuUH9+tWpu4qMyjy5FWpeKRfV6e9SZrIzzweTTrf5rVdzO8bbhOb1S5xOotnXVc' + 'SQBnD8j6jVZfDty0PDOvOvPGWVhWrcKGLVbo7UdoCFZOxTT4mmFLwwgwEk15I/DDhp53NClk' + '/Cr2va/4OtF0GUPMU9kpAuKWVjTAKVg4xqsveVfs54aeaZMRpIbx50qeaeGYNsPg43NBTDNj' + '8Sp2lqEzRAVhdUzRllHnbUM4WEJPqjd9lGqCwVZ42ZXFMz/qf4Gcx55jLJ6LywYu1i83A3OQ' + 'kWyQMomOF5Ay1grMn5aYRRgQCYrPMhde8ZKb9iuV232k6UaTrNbgDkDrCwyrMwNDSexU068t' + 'JjFArgTcIc8cVzKghYlFEY+1gyU3VumRANHAkNwYN8ylT8di7V/THeDjSTmJ8W25gAtWDTUD' + 'yxtREZyA38q2dKw13IAOYc0Cn2FTMfuAlGPBtCyIBrRQwqjiagl8vyFCAvRa8Egi7g0qQZQU' + '8pBsVOAhEBsgVSihuhmd0VrxM5jJZSNWRxQY0QI/AZWBpQxm8VS0O75ksMhADSHmQWAQgZIz' + 'PWQlI4MlfKkJtInSPJTRcD1xPaZFW0D05K04NsYq4DFcEdXYEERRkKfpiSki8ssiQZU2mMIz' + 'cxranwFJ7SiZLJoKAMWwVCItyok1xjBXnYX54RBHh8ztYLW6Y86WAqAzEQ4bDvqEY5/AegVh' + 'ZVTDpOwQ6RDVImfAH7uYj4QEl8nay2RQ0s5qNWGxjlFUrSNyIIu2oNKjxcio5c9qL3TkAhF4' + 'YBjVnTrTGjCusSZfO3FEyWVjtJ5gylut1T+oGCjkxA6th+pgJPMZNA1Q0L6tkancbwftHyEO' + 'NEqBnwy4OjlLFBbVoAE0qN7MgHhi0YpqmDMJ7GGxN2dL6hRIYlTX0K4TPIMX2bo5yrGxBoKu' + 'ydi3tnjs2Jzlq/2LViud/Uu1tRvTSmSAkRuL9uq2CEc7+M9qDca5uAY8bj4z2SihNpOyVOFC' + '2jD1mLypHc1JRpRq4FNJDy9GMW+YYN5CLqarstT6YLCclBCC56pIQ8gPWE80VRkTaZJoWjc2' + 'FpmhtUupoCeAPaJfo0eMt4EyQ6gkoLvJSxsiNPC1o7zOUi+TAbmmzpQ/wjzo3yTsqtbL4mKP' + 'JGkxoCPT/M7UnGVZS0GRKH9Ah2N9jmAXB7tSfS5IIUnZYV+nqoPd/NLRfpFxtQBMbl6Q1Jzk' + 'S7kCJt4ICv0CcXRiJNQTDQYYCqoUGZkiM0NafFvKuVlToOxqetVGBOQtiwtZra4eGtMNaEuV' + 'NrDkeNDWJ4NxLgsUcgVTbq6tsm1Jm0jEd9FSD5aVunOGbhrbeCKiY1pQuOLrxlSLWEvlp24N' + 'kzrT/tPaMYQKR5+mBjNWmOwO4KTHYxMFCC2i5806sMN4zRymSS1RZBGDQNgB4nA4VYqD0CuL' + 'tsw2Urk6YOGLRIQzcjARQIhLWubrrOg0I6NMnQ1tIrzpkHMgGzubOKR/q/AJbpA+MATZ2nZM' + '1wmw5uyG4vPz8aR/0ibBqBrabmICNrJ61TuzdGDicDnDF3DOU4LOBmOg99JTnAukdNAryHuT' + '6QAT81Ccq17S+fkjic4Zs0usDUf+iCoEf5I5avdsjaPVcuZzwMG5Lho48Ic2IyMdC57zqcIs' + 'uZOCzepwENVEc01wkYwQQBXqbFrnQe/nRGS0ZJobrjQkjxUVDId1AkHnwElKOzyIIanBAgn1' + 'oAh2TL+pKce6aod1fDPX8CJKzjbkHmsaLNQagedgHfNQwaHp0EgbsRWPsrpVmq99HaFxIgFN' + 'I6iLGUyiqrFPEInxYPqQ0gFtMOV8Z6J5hvZPahahwcdkUgQJOnc8BS7OVRJL8U2QUdgoqyVp' + 'ptWhjCFM08OQuXVqpxaSdN2ahRQVBIehKWaa97Ge6gRZtUZlqdWpiTkdU5GUiiJgk5HMAWTq' + 'GQNppxrSmQTSAt+O1w56alkxm7KdO2lztMgmXh7/ZJt1gXelz7mzySaYEsHyWQcUFRIEFdHs' + 'qmDUyZ0imwa5Oac3OlWoBi4dUaIJRsMQgDAH1SQrfehIbY644hhHjDpOVd7qrKm1ZilrQbJx' + '7TRVZzcdp80GtYnJ0o6LpBNw1iRpAq+rmpjTKHVa1AlC1OTmKldYZfyPCchOtnVuVpOqHY6n' + 'FA6hqozZ8GiYxztzJ3tNQTHUvKeLPlHiSGXqMB2Gt6oCFfG0zlIx9z+jXd+48bqvjya7F2+/' + 'eXT94p7R2Xtn75299/91747th8988LzX9t4+ee2fv91x247j7xx9fuOB9+478+zJ3/ztyl83' + 'Gz7978W9Lx84cezx52575PzvfeGhdud3vr9t9MabH3vs4j/sWPfjky98ctPmo7/cv2fD/ed+' + 'uFs/fe6eu67a+OgDT3zm6JHTx8LyqTsOvLz3vkff8cLKuiNXfuDhfxw6tP+td1+07vC3XvnJ' + 'g1968toHDt9979H2PZ+6f9e+2UNfeakJP31491+/eOkji39fvnXrBQsr1+w/+JHHtp+4Z/nV' + 'X/3xRy/+fsPzV11/zusv7Vr/yg8PLl38g613br5gemrLeW9svfre2brnL/zEwcu//fpFh3ZN' + 'L9t69ey7V5z7r99tWf7mvj+978zP3vb+yzdv2/KhN9d/9K2Vz61c8vix4/c9cffCL/6y7Yq3' + 'P3Pplzc+ffqZTacufOo/KydePP2uOxeXL/nz7pPbz//503dd9uq2I92D4Zwzn91zfNPH/wva' + 'WLXR')) + + +def unpack(pair): + """(compressed bytes, plain bytes).""" + return (base64.b64decode(pair[0]), + zlib.decompress(base64.b64decode(pair[1]))) diff --git a/tests/test_apfs_decmpfs.py b/tests/test_apfs_decmpfs.py index be01e9a..219c64e 100644 --- a/tests/test_apfs_decmpfs.py +++ b/tests/test_apfs_decmpfs.py @@ -18,6 +18,10 @@ from engine.ewf import OffsetReader # noqa: E402 from engine.fs.ntfs import open_fs # noqa: E402 +import lzfse_vectors # noqa: E402 + +LZ_STREAM, LZ_PLAIN = lzfse_vectors.unpack(lzfse_vectors.LZVN) +LZ_BIG_STREAM, LZ_BIG = lzfse_vectors.unpack(lzfse_vectors.V2) BIG = bytes((i * 7 + i // 251) & 0xFF for i in range(150000)) SMALL = b"held in the attribute " * 8 UF_COMPRESSED = 0x20 @@ -97,16 +101,19 @@ def read_at(self, offset, length): return self.data[offset:offset + length] -def compressed_file(kind, data, name, stream_obj=None): +def compressed_file(kind, data, name, stream_obj=None, blocks=None): """(name, flags, xattrs, fork) for `data` held as decmpfs `kind`.""" if kind in (3, 7): codec = dc.zlib_block if kind == 3 else dc.lzvn_literals value = dc.inline_value(kind, len(data), codec(data)) return (name, UF_COMPRESSED, [lambda oid: embedded(oid, "com.apple.decmpfs", value)], None) - fork = (dc.zlib_fork([dc.zlib_block(b) for b in dc.split(data)]) - if kind == 4 else - dc.lzvn_fork([dc.lzvn_literals(b) for b in dc.split(data)])) + if blocks is not None: + fork = dc.lzvn_fork(blocks) # an offset table, whatever the codec + else: + fork = (dc.zlib_fork([dc.zlib_block(b) for b in dc.split(data)]) + if kind == 4 else + dc.lzvn_fork([dc.lzvn_literals(b) for b in dc.split(data)])) value = dc.header(kind, len(data)) return (name, UF_COMPRESSED, [lambda oid: embedded(oid, "com.apple.decmpfs", value), @@ -123,9 +130,14 @@ def setUpClass(cls): compressed_file(7, SMALL, "l-inline"), compressed_file(4, BIG, "z-fork"), compressed_file(8, BIG, "l-fork"), + ("raw", UF_COMPRESSED, + [lambda oid: embedded(oid, "com.apple.decmpfs", + dc.header(9, 900) + b"\xcc")], None), ("lzfse", UF_COMPRESSED, [lambda oid: embedded(oid, "com.apple.decmpfs", - dc.header(11, 900) + b"bvx2")], None), + dc.header(11, len(LZ_PLAIN)) + LZ_STREAM)], + None), + compressed_file(12, LZ_BIG, "lzfse-fork", blocks=[LZ_BIG_STREAM]), ("flagged", UF_COMPRESSED, [], None), ("unflagged", 0, [lambda oid: embedded(oid, "com.apple.decmpfs", @@ -152,11 +164,21 @@ def test_stat_describes_the_compression(self): self.assertEqual(info["compression"]["uncompressed_size"], len(BIG)) self.assertIn("LZVN", info["note"]) - def test_lzfse_is_reported_not_faked(self): + def test_lzfse_is_decompressed(self): e = self.v.entry("lzfse") + self.assertEqual(e["size"], len(LZ_PLAIN)) + self.assertEqual(self.fs.read_file(e), LZ_PLAIN) + + def test_lzfse_in_the_resource_fork(self): + e = self.v.entry("lzfse-fork") + self.assertEqual(e["size"], len(LZ_BIG)) + self.assertEqual(self.fs.read_file(e), LZ_BIG) + + def test_a_method_that_is_not_read_is_reported_not_faked(self): + e = self.v.entry("raw") self.assertTrue(e["compression_unsupported"]) self.assertEqual(self.fs.read_file(e), b"") - self.assertIn("LZFSE", self.fs.stat(e)["note"]) + self.assertIn("raw", self.fs.stat(e)["note"]) self.assertTrue(any("does not decompress" in f for f in self.fs.info()["findings"])) diff --git a/tests/test_decmpfs.py b/tests/test_decmpfs.py index d3d7321..86c3f96 100644 --- a/tests/test_decmpfs.py +++ b/tests/test_decmpfs.py @@ -143,7 +143,7 @@ def test_every_type_maps_to_a_codec_and_a_place(self): for kind, (codec, fork) in want.items(): h = decmpfs.parse_header(build.header(kind, 1)) self.assertEqual((h["codec"], h["in_resource_fork"]), (codec, fork)) - self.assertEqual(h["supported"], codec in ("zlib", "LZVN")) + self.assertEqual(h["supported"], codec in ("zlib", "LZVN", "LZFSE")) def test_other_values_are_not_headers(self): for value in (None, b"", b"fpmc", b"fpmc" + bytes(11), @@ -157,8 +157,7 @@ def test_an_unknown_type_is_a_header_that_is_not_supported(self): self.assertIn("type 99", decmpfs.describe(h)) def test_an_unsupported_codec_is_refused_by_name(self): - for kind, text in ((11, "LZFSE"), (12, "LZFSE"), (9, "raw"), - (5, "type 5")): + for kind, text in ((9, "raw"), (10, "raw"), (5, "type 5")): h = decmpfs.parse_header(build.header(kind, 10)) with self.assertRaises(decmpfs.DecmpfsError) as cm: decmpfs.Compressed(h) @@ -314,6 +313,35 @@ def test_a_huge_declared_size_is_refused_for_an_attribute(self): decmpfs.Compressed(h) +class Lzfse(unittest.TestCase): + """Types 11 and 12, with streams from Apple's encoder.""" + + @classmethod + def setUpClass(cls): + import lzfse_vectors as v + cls.stream, cls.plain = v.unpack(v.V2) + + def test_attribute_type(self): + c = compressed(build.header(11, len(self.plain)) + self.stream) + self.assertEqual(c.read_at(0, len(self.plain)), self.plain) + self.assertEqual(c.read_at(100, 50), self.plain[100:150]) + + def test_resource_fork_type_with_a_stored_block_and_a_compressed_one(self): + first = bytes(range(256)) * 256 # a whole 64 KiB block + fork = build.lzvn_fork([b"\xff" + first, self.stream]) + c = compressed(build.header(12, len(first) + len(self.plain)), fork) + data = first + self.plain + self.assertEqual(c.read_at(0, len(data)), data) + self.assertEqual(c.read_at(65530, 20), data[65530:65550]) + self.assertEqual(c.findings, []) + + def test_a_damaged_block_reads_as_zeros_and_says_so(self): + c = compressed(build.header(11, len(self.plain)) + + self.stream[:len(self.stream) // 2]) + self.assertEqual(c.read_at(0, len(self.plain)), bytes(len(self.plain))) + self.assertTrue(c.findings) + + class ResourceFork(unittest.TestCase): """Types 4 and 8: 64 KiB blocks in the resource fork.""" diff --git a/tests/test_hfsplus_links.py b/tests/test_hfsplus_links.py index ad5bbb0..efb5dbd 100644 --- a/tests/test_hfsplus_links.py +++ b/tests/test_hfsplus_links.py @@ -63,6 +63,9 @@ def compressed(name, kind, data, **kw): owner_flags=0x20, **kw) +import lzfse_vectors # noqa: E402 + +LZ_STREAM, LZ_PLAIN = lzfse_vectors.unpack(lzfse_vectors.V2) BIG = bytes((i * 7 + i // 251) & 0xFF for i in range(150000)) @@ -156,8 +159,10 @@ def setUpClass(cls): compressed("empty", 3, b""), hb.File("stored", rsrc=dc.zlib_fork([dc.zlib_block(b"raw!", True)]), xattrs={XATTR: dc.header(4, 4)}, owner_flags=0x20), - hb.File("lzfse", xattrs={XATTR: dc.header(11, 500) + b"bvx2"}, + hb.File("raw", xattrs={XATTR: dc.header(9, 500) + b"\xcc"}, owner_flags=0x20), + hb.File("lzfse", xattrs={XATTR: dc.header( + 11, len(LZ_PLAIN)) + LZ_STREAM}, owner_flags=0x20), hb.File("noattr", b"stored text", owner_flags=0x20), hb.Link("linked", 200)], private_files=[node])) @@ -202,14 +207,21 @@ def test_stat_says_how_it_is_compressed(self): self.assertEqual(info["compression"]["uncompressed_size"], len(BIG)) self.assertIn("LZVN", info["note"]) - def test_lzfse_is_reported_and_not_faked(self): + def test_lzfse_is_decompressed(self): e = find(self.fs, "lzfse") + self.assertEqual(e["size"], len(LZ_PLAIN)) + self.assertEqual(self.fs.read_file(e), LZ_PLAIN) + self.assertIn("LZFSE", self.fs.stat(e)["note"]) + + def test_a_method_that_is_not_read_is_reported_and_not_faked(self): + e = find(self.fs, "raw") self.assertTrue(e["compression_unsupported"]) self.assertEqual(e["size"], 0) self.assertEqual(self.fs.read_file(e), b"") self.assertEqual(e["uncompressed_size"], 500) - self.assertIn("LZFSE", self.fs.stat(e)["note"]) - self.assertTrue(any("LZFSE" in f for f in self.fs.info()["findings"])) + self.assertIn("raw", self.fs.stat(e)["note"]) + self.assertTrue(any("does not decompress" in f + for f in self.fs.info()["findings"])) def test_flag_without_attribute_is_shown_as_stored(self): e = find(self.fs, "noattr") diff --git a/tests/test_lzfse.py b/tests/test_lzfse.py new file mode 100644 index 0000000..fee9d01 --- /dev/null +++ b/tests/test_lzfse.py @@ -0,0 +1,111 @@ +"""The LZFSE decoder (engine.lzfse), against streams written by Apple's +reference encoder (embedded in lzfse_vectors) and against damage.""" + +import os +import random +import struct +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import lzfse_vectors as vectors # noqa: E402 +from engine import lzfse # noqa: E402 + + +class Reference(unittest.TestCase): + + def test_each_block_kind_from_the_reference_encoder(self): + kinds = {"V2": b"bvx2", "LZVN": b"bvxn", "RAW": b"bvx-"} + for name, magic in kinds.items(): + stream, plain = vectors.unpack(getattr(vectors, name)) + self.assertEqual(stream[:4], magic, name) + self.assertEqual(lzfse.decode(stream), plain, name) + + def test_blocks_of_every_kind_in_one_stream(self): + stream, plain = vectors.unpack(vectors.CHAIN) + self.assertEqual(lzfse.decode(stream), plain) + + def test_the_v2_block_really_holds_literals_and_matches(self): + stream, _ = vectors.unpack(vectors.V2) + h, _size = lzfse._unpack_v2(stream, 0) + self.assertGreater(h["n_literals"], 100) + self.assertGreater(h["n_matches"], 500) + + def test_a_v1_block_is_read_too(self): + """The same block with its tables written out in full (bvx1), as + older encoders wrote it.""" + stream, plain = vectors.unpack(vectors.V2) + h, size = lzfse._unpack_v2(stream, 0) + payload = stream[size:size + h["n_literal_payload"] + + h["n_lmd_payload"]] + head = b"bvx1" + struct.pack( + "