diff --git a/.github/workflows/crosscheck.yml b/.github/workflows/crosscheck.yml new file mode 100644 index 0000000..b65123c --- /dev/null +++ b/.github/workflows/crosscheck.yml @@ -0,0 +1,44 @@ +name: Cross-check + +# Reads the synthetic corpus with Strata and with independent readers (the +# libyal libraries, qemu-img) and fails on any difference that +# tests/crosscheck_baseline.json does not explain. See docs/CROSSCHECK.md. + +on: + push: + branches: [main] + pull_request: + paths: + - "engine/**" + - "tests/crosscheck*" + - "tests/imagebuild_*.py" + - ".github/workflows/crosscheck.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: crosscheck-${{ github.ref }} + cancel-in-progress: true + +jobs: + crosscheck: + name: Against reference readers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install the reference readers + run: | + sudo apt-get update + sudo apt-get install -y qemu-utils + python3 -m pip install libewf-python libvhdi-python libqcow-python \ + libvmdk-python libvsgpt-python libfsntfs-python libfshfs-python \ + libfsext-python libfsfat-python + - name: List what is installed + run: python3 tests/crosscheck.py --list + - name: Compare + run: python3 tests/crosscheck.py --no-save diff --git a/.gitignore b/.gitignore index 7385b52..bc4e3fe 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ desktop.ini # Native sidecar build output and locally installed test binaries native/target/ engine/native/*/ + +# Output of tests/crosscheck.py; may describe real evidence. +/crosscheck-out/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a7c5a51..42160d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,14 @@ records, not how the code changed. ### Added +- A repeatable comparison of Strata against independent readers + (`tests/crosscheck.py`; libewf, libvhdi, libqcow, libvmdk, libvsgpt, + libfsntfs, libfshfs, libfsext, libfsfat and qemu-img where installed). It + reads the same image both ways, layer by layer, and lists every difference; + the ones that are understood are recorded with their reasons and a new one + fails the run. It can also be pointed at real images, with file and folder + names withheld from its output. Synthetic images only so far; see + `docs/CROSSCHECK.md`. - **AFF4 disk images can be opened.** Both the single-stream and the more common Map-over-ImageStream shape are read, with zlib/Deflate-compressed or stored chunks; a gap in a Map (unallocated space) reads as zeros, a diff --git a/docs/CROSSCHECK.md b/docs/CROSSCHECK.md new file mode 100644 index 0000000..57fb7b3 --- /dev/null +++ b/docs/CROSSCHECK.md @@ -0,0 +1,70 @@ +# Cross-checking Strata against other readers + +`tests/crosscheck.py` reads the same image with Strata and with independent +readers and reports every place they disagree. It exists because passing +Strata's own tests shows that Strata reads what Strata's authors expected, not +that it reads what other tools read. A disagreement is a lead, not a verdict: +either reader may be the one that is wrong. + +## What is compared + +Layer by layer, each side given the same input: + +| Layer | Strata | Reference | +|---|---|---| +| Disk container (EWF, VHD, QCOW2, VMDK, VDI, DMG) | the engine's reader | libewf, libvhdi, libqcow, libvmdk, qemu-img | +| Partition table (GPT) | `volume.scan` | libvsgpt | +| Filesystem tree (NTFS, FAT, ext, HFS+) | the walker search and hashing share | libfsntfs, libfsfat, libfsext, libfshfs | + +Containers are compared by size and by a digest of each 1 MiB chunk (all of +them, or with `--chunks N` the first and last few and a seeded random sample, +so a run can be repeated). Filesystems are compared by path, kind, size, a +digest of the first `--hash-bytes` of content, and extra streams (NTFS +alternate data streams, the HFS+ resource fork). Filesystem readers are given +Strata's own view of the partition, so a container difference cannot show up +as a filesystem one. + +The references are optional. Each is used only if its Python package (or +program) is installed, and is reported as `missing` otherwise; `--list` says +what is installed and how to get the rest. Strata itself needs only the +standard library. + +## Running it + + python3 tests/crosscheck.py synthetic corpus + python3 tests/crosscheck.py --list readers and cases + python3 tests/crosscheck.py --case 'hfsplus*' some cases + python3 tests/crosscheck.py --images A.E01 ... real images + python3 tests/crosscheck.py --strict also fail on stale baseline entries + +The exit status is 1 when a difference is not in the baseline. CI runs it on +every change to the engine (`.github/workflows/crosscheck.yml`). + +## The baseline + +`tests/crosscheck_baseline.json` lists the differences that are understood, +each with a case, a facet (`container`, `volumes`, `fs`), a kind (`only_strata`, +`only_ref`, `differs`, `error`), a key pattern and the reason it is accepted. +An entry without a reason is refused. Many entries are about the synthetic +images rather than the readers: a reference reader that refuses a fixture +(because the builder leaves out something real writers always write) is +recorded as an `error`, so the gap stays on the list instead of being quietly +skipped. + +Add an entry only after finding out why the two disagree. If Strata is wrong, +fix Strata and leave the baseline alone. + +## Real images + +`--images` runs the same comparison over images you supply. File and folder +names are evidence, so for these they are replaced in everything printed and +saved by short stable tokens (`--names` turns that off); nothing about a real +image is written to the baseline, which refuses entries for anything but +synthetic cases; and results are saved to `crosscheck-out/`, which git +ignores. Review what you share from a real image before sharing it. + +## Not yet covered + +Encryption (BitLocker, LUKS), artefact parsers (event logs, registry, mail, +shortcuts, prefetch), AFF4, APFS and exFAT have no reference wired in, and +the ext4 fixture is not yet compared (see its baseline entry). diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f2c1d18..ccbd3cf 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -276,9 +276,10 @@ Agreeing with another tool is agreement, not verification. ## Open questions -- **Independent validation.** None of the parsers has been validated against - reference tooling on shared evidence; a repeatable comparison against an - established tool is the candidate before any new filesystem. +- **Independent validation.** `tests/crosscheck.py` now compares containers, + GPT and four filesystems against established readers on the synthetic + corpus (see `docs/CROSSCHECK.md`). Still open: running it over real images, + and references for encryption, artefacts, APFS and exFAT. - **Fuzz corpus growth.** The saved corpus is small; a standing fuzz job with coverage feedback would find the next #15 before a user does. diff --git a/tests/crosscheck.py b/tests/crosscheck.py new file mode 100644 index 0000000..840fa2d --- /dev/null +++ b/tests/crosscheck.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Compare what Strata reads with what established readers read. + + python3 tests/crosscheck.py the synthetic corpus, against + every reference that is installed + python3 tests/crosscheck.py --list what is installed, and the cases + python3 tests/crosscheck.py --case 'hfs*' only cases matching a pattern + python3 tests/crosscheck.py --images PATH... real images (names withheld) + +A run reads each image with Strata and with each reference reader that +handles its format, layer by layer (the disk container, the partition table, +each filesystem), always giving both the same input, and reports every +difference. A difference is not a verdict: either reader may be the one that +is wrong. The ones that are understood are listed, with their reasons, in +tests/crosscheck_baseline.json; a run fails on any that are not, and (with +--strict) on baseline entries that no longer match anything. + +Real images. Names of files and folders are evidence. For --images they are +withheld from everything printed and saved unless --names is given, nothing +about such an image is ever written to the baseline, and results are saved +only under --out (default: crosscheck-out/, which is ignored by git). + +Needs nothing but the standard library to run; each reference reader is used +only if its package (or program) is installed, and is reported as missing +otherwise. See the output of --list, and docs/CROSSCHECK.md.""" + +import argparse +import fnmatch +import json +import os +import sys +import tempfile +import time +import traceback + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.dirname(HERE)) + +import crosscheck_corpus as corpus # noqa: E402 +import crosscheck_lib as lib # noqa: E402 +import crosscheck_refs as refs # noqa: E402 +import crosscheck_strata as ours # noqa: E402 + +BASELINE = os.path.join(HERE, "crosscheck_baseline.json") + +# coverage statuses +AGREE, DIFFER, NO_REF, MISSING = "agree", "differ", "no reference", "missing" + + +class Run(object): + """Everything one invocation found.""" + + def __init__(self, opts): + self.opts = opts + self.diffs = [] + self.coverage = [] # (case, layer, reference, status, note) + + def cover(self, case, layer, reference, status, note=""): + self.coverage.append((case, layer, reference, status, note)) + + def add(self, found): + self.diffs.extend(found) + + +def guarded(run, case, facet, label, fn): + """fn() -> result, or None after recording the exception as a + difference: an error on either side is a finding, not a crash.""" + try: + return fn() + except refs.Rejected as exc: + run.add([lib.difference(facet, case, lib.ERROR, label + ":refused", + str(exc))]) + except Exception as exc: # noqa: BLE001 + if run.opts.trace: + traceback.print_exc() + run.add([lib.difference( + facet, case, lib.ERROR, label + ":" + type(exc).__name__, + str(exc)[:240])]) + return None + + +def check_container(run, case, path, img): + key = ours.image_format(img) + plan = lib.chunk_plan(img.size, run.opts.chunks, run.opts.seed) + mine = guarded(run, case, "container", "strata", + lambda: ours.container(img, plan)) + candidates = refs.references_for("container", key) + if not candidates: + run.cover(case, "container", "-", NO_REF, "format %s" % key) + return + for ref in candidates: + ok, why = ref.available() + if not ok: + run.cover(case, "container", ref.name, MISSING, why) + continue + theirs = guarded(run, case, "container", ref.name, + lambda: ref.run(path, plan, lib.CHUNK, key)) + if mine is None or theirs is None: + run.cover(case, "container", ref.name, DIFFER, "one side failed") + continue + found = lib.compare_disks(case, mine, theirs) + run.add(found) + run.cover(case, "container", ref.name, DIFFER if found else AGREE) + + +def check_volumes(run, case, img): + layout, mine = ours.partitions(img) + scheme = layout.get("scheme") + if scheme != "GPT": + return layout, mine + for ref in refs.references_for("volumes", "gpt"): + ok, why = ref.available() + if not ok: + run.cover(case, "volumes", ref.name, MISSING, why) + continue + theirs = guarded(run, case, "volumes", ref.name, + lambda: ref.run(lib.SliceIO(img))) + if theirs is None: + run.cover(case, "volumes", ref.name, DIFFER, "reference failed") + continue + found = lib.compare_lists(case, "volumes", mine, theirs, + lambda p: "partition %s" % p["index"]) + run.add(found) + run.cover(case, "volumes", ref.name, DIFFER if found else AGREE) + return layout, mine + + +def check_filesystems(run, case, img): + layout = ours.volume.scan(img) + for p in layout["partitions"]: + if p.get("allocated") is False or not p.get("detected"): + continue + label = "%s@%d" % (p["detected"], p["offset"]) + try: + fs = ours.open_partition(img, p["offset"], p["size"]) + except Exception as exc: # noqa: BLE001 + run.cover(case, "fs " + label, "-", NO_REF, + "Strata cannot open it: %s" % type(exc).__name__) + continue + kind = ours.fs_kind(fs) + candidates = refs.references_for("fs", kind) + if not candidates: + run.cover(case, "fs " + label, "-", NO_REF, "filesystem %s" % kind) + continue + mine = guarded(run, case, "fs", "strata", + lambda: ours.tree(fs, run.opts.hash_bytes)) + for ref in candidates: + ok, why = ref.available() + if not ok: + run.cover(case, "fs " + label, ref.name, MISSING, why) + continue + theirs = guarded( + run, case, "fs", ref.name, + lambda: ref.run(lib.SliceIO(img, p["offset"], p["size"]), + run.opts.hash_bytes)) + if mine is None or theirs is None: + run.cover(case, "fs " + label, ref.name, DIFFER, + "one side failed") + continue + found = lib.compare_trees(case, mine, theirs) + run.add(found) + run.cover(case, "fs " + label, ref.name, + DIFFER if found else AGREE, + "%d entries / %d" % (len(mine), len(theirs))) + + +def run_case(run, case_id, path): + try: + img = ours.open_image(path) + except Exception as exc: # noqa: BLE001 + run.add([lib.difference("container", case_id, lib.ERROR, + "strata:open", "%s: %s" % ( + type(exc).__name__, str(exc)[:200]))]) + return + try: + check_container(run, case_id, path, img) + check_volumes(run, case_id, img) + check_filesystems(run, case_id, img) + finally: + img.close() + + +def synthetic_cases(opts): + wanted = opts.case or ["*"] + return [c for c in corpus.synthetic() + if any(fnmatch.fnmatchcase(c.id, "synthetic:" + w) + or fnmatch.fnmatchcase(c.id, w) for w in wanted)] + + +def real_case_id(path): + return "image:" + lib.sha(os.path.abspath(path).encode())[:8] + + +def print_list(): + print("Reference readers:") + for r in refs.REFERENCES: + ok, why = r.available() + print(" %-10s %-12s %-22s %s" % ( + r.facet, r.name, ",".join(r.keys), "ok" if ok else why)) + print("\nSynthetic cases:") + for c in corpus.synthetic(): + print(" " + c.id) + + +def report(run, baseline, redact): + explained, unexplained, stale = lib.classify(run.diffs, baseline) + print("\nCoverage") + width = max([len(c[0]) for c in run.coverage] + [10]) + for case, layer, ref, status, note in run.coverage: + print(" %-*s %-22s %-12s %-12s %s" % ( + width, case, layer, ref, status, note)) + print("\n%d difference(s): %d explained by the baseline, %d not" + % (len(run.diffs), len(explained), len(unexplained))) + for line in lib.summarise(unexplained, redact): + print(line) + if stale: + print("\n%d baseline entr%s match nothing now:" % ( + len(stale), "y" if len(stale) == 1 else "ies")) + for e in stale: + print(" %s %s %s %s" % (e["case"], e["facet"], e["kind"], + e["key"])) + return explained, unexplained, stale + + +def save(run, out_dir, explained, unexplained, stale, redact): + os.makedirs(out_dir, exist_ok=True) + + def shown(d): + d = dict(d) + if d["facet"] == "fs": + d["key"] = redact.path(d["key"]) + d["detail"] = redact.text(d["detail"]) if ( + redact.enabled and d["detail"]) else d["detail"] + return d + name = os.path.join(out_dir, "crosscheck-%d.json" % int(time.time())) + with open(name, "w", encoding="utf-8") as fh: + json.dump({ + "coverage": [list(c) for c in run.coverage], + "unexplained": [shown(d) for d in unexplained], + "explained": [shown(d) for d in explained], + "stale_baseline": stale, + "names_withheld": redact.enabled}, fh, indent=2) + print("\nSaved %s" % name) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--list", action="store_true") + ap.add_argument("--case", action="append", metavar="PATTERN") + ap.add_argument("--images", nargs="+", metavar="PATH") + ap.add_argument("--names", action="store_true", + help="show file and folder names of real images") + ap.add_argument("--baseline", default=BASELINE) + ap.add_argument("--strict", action="store_true", + help="also fail on baseline entries that match nothing") + ap.add_argument("--chunks", type=int, default=None, + help="read at most this many 1 MiB chunks of each disk") + ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--hash-bytes", type=int, default=1 << 20, + help="content digested per file (the first N bytes)") + ap.add_argument("--out", default=os.path.join( + os.path.dirname(HERE), "crosscheck-out")) + ap.add_argument("--no-save", action="store_true") + ap.add_argument("--trace", action="store_true") + opts = ap.parse_args(argv) + + if opts.list: + print_list() + return 0 + + run = Run(opts) + real = bool(opts.images) + redact = lib.Redactor(real and not opts.names) + with tempfile.TemporaryDirectory(prefix="crosscheck-") as tmp: + if real: + for p in opts.images: + if not os.path.isfile(p): + print("not a file: %s" % ("(withheld)" if redact.enabled + else p), file=sys.stderr) + return 2 + cid = real_case_id(p) + print("running %s" % cid) + run_case(run, cid, p) + else: + for case in synthetic_cases(opts): + print("running %s" % case.id) + folder = os.path.join(tmp, case.id.split(":")[1]) + os.makedirs(folder) + path = os.path.join(folder, "image" + case.suffix) + with open(path, "wb") as fh: + fh.write(case.build()) + for name, data in case.companions().items(): + with open(os.path.join(folder, name), "wb") as fh: + fh.write(data) + run_case(run, case.id, path) + + baseline = [] if real else lib.load_baseline(opts.baseline) + explained, unexplained, stale = report(run, baseline, redact) + if not opts.no_save: + save(run, opts.out, explained, unexplained, stale, redact) + failed = bool(unexplained) or (opts.strict and bool(stale)) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/crosscheck_baseline.json b/tests/crosscheck_baseline.json new file mode 100644 index 0000000..be02ede --- /dev/null +++ b/tests/crosscheck_baseline.json @@ -0,0 +1,86 @@ +[ + { + "case": "synthetic:ntfs*", + "facet": "fs", + "kind": "only_strata", + "key": "/$*", + "reason": "Strata lists NTFS's own metadata files ($MFT, $Bitmap, $Boot, ...) in the root, as FTK Imager and X-Ways do, so they can be examined; libfsntfs leaves them out of the root's listing. A choice of presentation, not a content difference." + }, + { + "case": "synthetic:fat1?", + "facet": "fs", + "kind": "differs", + "key": "/CYCLE.BIN", + "reason": "The fixture deliberately declares 5000 bytes on a corrupt cluster chain that loops (17 -> 18 -> 17). Strata follows the chain for the declared size and notes the damage; libfsfat refuses to read it. Both are defensible on corrupt input." + }, + { + "case": "synthetic:ext4", + "facet": "fs", + "kind": "error", + "key": "libfsext:refused", + "reason": "libfsext refuses an entry of the synthetic ext4 image whose in-inode extended attribute area (inline data) it considers out of bounds. Not yet established whether the builder or either reader is wrong; kept here so the ext4 tree is not compared until it is looked at." + }, + { + "case": "synthetic:hfsplus.links", + "facet": "fs", + "kind": "only_strata", + "key": "/*HFS+ Private Data*", + "reason": "The private metadata folder's name begins with four NUL characters. Strata keeps them as they are on disk; libfshfs replaces each with the symbol for NUL (U+2400), so the same folder has two spellings. Content identical." + }, + { + "case": "synthetic:hfsplus.links", + "facet": "fs", + "kind": "only_ref", + "key": "/*HFS+ Private Data*", + "reason": "The same folder as the entry above, as libfshfs names it." + }, + { + "case": "synthetic:hfsplus.compressed", + "facet": "fs", + "kind": "differs", + "key": "/*", + "reason": "libfshfs does not apply HFS+ transparent compression (decmpfs) and reports the files as empty; Strata decompresses them. Strata's output is the one covered by its own decmpfs tests against known-good vectors." + }, + { + "case": "synthetic:vmdk.*", + "facet": "container", + "kind": "error", + "key": "libvmdk:OSError", + "reason": "The builder writes 8-sector grains so the fixtures stay tiny; libvmdk accepts only grains of more than 8 sectors (real writers use 128). Strata reads the image; the reference refuses it." + }, + { + "case": "synthetic:vhd.dynamic", + "facet": "container", + "kind": "differs", + "key": "size", + "reason": "qemu-img works the disk size out from the CHS geometry in the footer, and a fixture of 40 sectors has zero cylinders, so it reports an empty disk. Strata and libvhdi both use the size the footer states. Needs a larger fixture to compare on." + }, + { + "case": "synthetic:vhd.dynamic", + "facet": "container", + "kind": "differs", + "key": "content", + "reason": "The same: qemu-img converts an empty disk, so there is nothing to compare." + }, + { + "case": "synthetic:qcow2.*", + "facet": "container", + "kind": "error", + "key": "qemu-img:refused", + "reason": "The builder writes no reference count table, which qemu requires and Strata and libqcow do not. A fixture shortcoming, not a difference in what the image holds." + }, + { + "case": "synthetic:vdi", + "facet": "container", + "kind": "error", + "key": "qemu-img:refused", + "reason": "The builder uses 4 KiB blocks to stay small; qemu accepts only the 1 MiB block size real writers use. Strata reads the image." + }, + { + "case": "synthetic:dmg", + "facet": "container", + "kind": "error", + "key": "qemu-img:refused", + "reason": "The fixture's blocks are LZFSE-compressed and qemu-img's LZFSE module is not installed, so it cannot read them. Strata decodes LZFSE against known-good vectors." + } +] diff --git a/tests/crosscheck_corpus.py b/tests/crosscheck_corpus.py new file mode 100644 index 0000000..86b735b --- /dev/null +++ b/tests/crosscheck_corpus.py @@ -0,0 +1,159 @@ +"""The synthetic images the cross-check runs over: the same builders the unit +tests use, so there is one description of each format in the repository. A +case is an image plus the name it should be given on disk (formats are +recognised partly by extension). An image that is not a valid example of its +format to a stricter reader is still included: the reader refusing it is a +finding about the builder, and the baseline says so, so it stays on the list +of things to improve rather than being quietly left out.""" + +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +class Case(object): + + def __init__(self, name, suffix, build, companions=None): + self.id = "synthetic:" + name + self.suffix = suffix + self.build = build + self.companions = companions or (lambda: {}) + + +def _first(x): + return x[0] if isinstance(x, (tuple, list)) else x + + +def _hfs(entries=None, **kw): + import imagebuild_hfsplus as hb + return hb.build(entries or [ + hb.Dir("Top", [hb.Dir("Inner", [hb.File("deep.txt", b"deep text")]), + hb.File("a.txt", b"first file")]), + hb.File("root.txt", b"x" * 5000)], **kw) + + +def _hfs_links(): + import imagebuild_hfsplus as hb + node = hb.File("iNode100", b"shared contents", cnid=100, + xattrs={"user.tag": b"on the node"}) + return hb.build( + [hb.File("plain.txt", b"plain"), + hb.Dir("sub", [hb.Link("second", 100)]), hb.Link("first", 100)], + private_files=[node]) + + +def _hfs_compressed(): + import imagebuild_decmpfs as dc + import imagebuild_hfsplus as hb + big = bytes((i * 7 + i // 251) & 0xFF for i in range(150000)) + small = b"held in the attribute " * 8 + xa = "com.apple.decmpfs" + + def packed(kind, data): + if kind in (3, 7): + codec = dc.zlib_block if kind == 3 else dc.lzvn_literals + return dc.inline_value(kind, len(data), codec(data)), b"" + blocks = dc.split(data) + fork = (dc.zlib_fork([dc.zlib_block(b) for b in blocks]) + if kind == 4 else + dc.lzvn_fork([dc.lzvn_literals(b) for b in blocks])) + return dc.header(kind, len(data)), fork + + entries = [] + for name, kind, data in (("z-inline", 3, small), ("z-fork", 4, big), + ("l-inline", 7, small), ("l-fork", 8, big)): + value, fork = packed(kind, data) + entries.append(hb.File(name, rsrc=fork, xattrs={xa: value}, + owner_flags=0x20)) + return hb.build(entries) + + +def _hfs_fragmented(): + import imagebuild_hfsplus as hb + return hb.build( + [hb.Dir("Top", [hb.File("f%d.txt" % i, b"data%d" % i) + for i in range(250)])], + fragment=("catalog",), reverse_leaves=True) + + +def _gpt_hfs(): + import imagebuild_gpt as g + hfs = _hfs() + sectors = -(-len(hfs) // 512) + return g.build([(40, sectors, g.HFS, "first slot", hfs)], + 40 + sectors + 100 + 34 + 1)[0] + + +def _gpt_ntfs(): + import imagebuild_ntfs as n + return n.wrap_gpt(n.build_ntfs()) + + +def _vhd_dynamic(): + import imagebuild_vhd + return _first(imagebuild_vhd.sparse( + 5 * 4096, {0: (bytes(range(256)) * 16, None), + 2: (b"\xA5" * 4096, None)})) + + +def _qcow2(version, pack): + """A compressed cluster has to inflate to a whole cluster, as every real + writer makes it; and the zero flag exists only from version 3.""" + def build(): + import imagebuild_qcow2 + clusters = {0: bytes(range(256)) * 16, + 2: ("z", b"\xA5" * 4096), + 4: ("z", (b"strata " * 500).ljust(4096, b"\0"))} + if version >= 3: + clusters[3] = "zero" + return imagebuild_qcow2.build(5 * 4096, clusters, version=version, + pack=pack) + return build + + +def _vdi(): + import imagebuild_vdi + return _first(imagebuild_vdi.build( + 5 * 4096, {0: bytes(range(256)) * 16, 2: b"\xA5" * 4096}, zero=(3,))) + + +def _dmg(): + import imagebuild_dmg + rng = random.Random(1) + disk = bytes(rng.choice(b"abcdefgh \n") for _ in range(512 * 48)) + return imagebuild_dmg.build([("disk image", 0, disk)]) + + +def synthetic(): + import imagebuild_ewf + import imagebuild_ext4 + import imagebuild_fat + import imagebuild_apfs + import imagebuild_ntfs + import imagebuild_vmdk + return [ + Case("ntfs", ".img", imagebuild_ntfs.build_ntfs), + Case("ntfs.gpt", ".img", _gpt_ntfs), + Case("fat12", ".img", lambda: imagebuild_fat.build_fat(12)), + Case("fat16", ".img", lambda: imagebuild_fat.build_fat(16)), + Case("ext4", ".img", imagebuild_ext4.build_ext4), + Case("ext2", ".img", imagebuild_ext4.build_ext2_legacy), + Case("apfs", ".img", imagebuild_apfs.build_apfs), + Case("hfsplus", ".img", _hfs), + Case("hfsplus.links", ".img", _hfs_links), + Case("hfsplus.compressed", ".img", _hfs_compressed), + Case("hfsplus.fragmented", ".img", _hfs_fragmented), + Case("hfsplus.gpt", ".img", _gpt_hfs), + Case("ewf", ".E01", lambda: _first(imagebuild_ewf.build_e01())), + Case("vhd.dynamic", ".vhd", _vhd_dynamic), + Case("qcow2.v2", ".qcow2", _qcow2(2, False)), + Case("qcow2.v3.compressed", ".qcow2", _qcow2(3, True)), + Case("vdi", ".vdi", _vdi), + Case("vmdk.sparse", ".vmdk", imagebuild_vmdk.build_sparse), + Case("vmdk.stream", ".vmdk", + lambda: _first(imagebuild_vmdk.build_stream_optimized())), + Case("dmg", ".dmg", _dmg), + ] diff --git a/tests/crosscheck_lib.py b/tests/crosscheck_lib.py new file mode 100644 index 0000000..039d9ca --- /dev/null +++ b/tests/crosscheck_lib.py @@ -0,0 +1,298 @@ +"""Cross-checking Strata's readers against established ones. + +Nothing here imports a third-party package. A reference reader is only ever +reached through the adapters in crosscheck_refs.py, which are skipped when the +package behind them is not installed; this module holds what is common to all +of them: + + * how an image is cut into the pieces both sides are asked to read, so a + disagreement can be pinned to a byte range rather than to a whole disk; + * the shape of what each side reports (an "observation": plain dicts and + lists, so it can be saved and compared later); + * the comparison itself, which names every difference and says which side + it is in; + * the baseline of differences that are understood and accepted, each with + its reason, so a new difference fails a run and a known one does not; + * the report, which can leave out names (a real image's file and folder + names are evidence, and must not end up in anything that is published). + +A difference is never resolved by deciding which reader is right. The two +disagree; a person finds out why.""" + +import fnmatch +import hashlib +import json +import os +import random + +CHUNK = 1 << 20 +HASH_LEN = 16 # hex digits kept of each digest + +# What a difference is in. The report groups on these. +ONLY_STRATA, ONLY_REF, DIFFERS, ERROR = ( + "only_strata", "only_ref", "differs", "error") +KINDS = (ONLY_STRATA, ONLY_REF, DIFFERS, ERROR) + + +def sha(data): + return hashlib.sha256(data).hexdigest()[:HASH_LEN] + + +class SliceIO(object): + """A file object over part of anything with read_at(offset, length) and a + size: how a reference reader is given Strata's view of a partition (so a + filesystem is compared on the same bytes on both sides) or a synthetic + image held in memory.""" + + def __init__(self, source, offset=0, size=None): + self._source = source + self._offset = offset + self._size = (source.size - offset) if size is None else size + self._pos = 0 + + def read(self, n=-1): + if n is None or n < 0: + n = self._size - self._pos + n = max(0, min(n, self._size - self._pos)) + if not n: + return b"" + data = self._source.read_at(self._offset + self._pos, n) + self._pos += len(data) + return data + + def seek(self, offset, whence=0): + if whence == 0: + self._pos = offset + elif whence == 1: + self._pos += offset + else: + self._pos = self._size + offset + self._pos = max(0, self._pos) + return self._pos + + def tell(self): + return self._pos + + def get_size(self): + return self._size + + def close(self): + pass + + def seekable(self): + return True + + def readable(self): + return True + + +class BytesSource(object): + """read_at over bytes.""" + bytes_per_sector = 512 + + def __init__(self, data): + self.data, self.size = data, len(data) + + def read_at(self, offset, length): + if offset < 0 or offset >= self.size: + return b"" + return self.data[offset:offset + length] + + +# -- cutting an image into pieces --------------------------------------------- + +def chunk_plan(size, budget=None, seed=1, chunk=CHUNK, edge=8): + """The chunk numbers to read: all of them when the image is small enough + (or `budget` is None), otherwise the first and last `edge` and a seeded + random choice between, so both sides are asked for the same ranges and a + run can be repeated exactly.""" + total = -(-size // chunk) if size else 0 + everything = list(range(total)) + if budget is None or total <= budget: + return everything + pick = set(everything[:edge]) | set(everything[-edge:]) + rng = random.Random(seed) + middle = everything[edge:-edge] if total > 2 * edge else [] + pick.update(rng.sample(middle, min(len(middle), + max(0, budget - len(pick))))) + return sorted(pick) + + +def digest_chunks(read_at, size, plan, chunk=CHUNK): + """{chunk number: digest of what read_at returns for it}, over `plan`.""" + out = {} + for n in plan: + want = min(chunk, size - n * chunk) + data = read_at(n * chunk, want) if want > 0 else b"" + out[n] = sha(data) if len(data) == want else \ + "short:%d" % len(data) + return out + + +# -- observations -------------------------------------------------------------- + +def disk_observation(read_at, size, plan, chunk=CHUNK): + return {"size": size, "chunk": chunk, + "digests": digest_chunks(read_at, size, plan, chunk)} + + +def entry_observation(path, kind, size=None, digest=None, streams=None): + """One file or folder. `digest` is of the content's first bytes only (as + many as the run was told to read); `streams` is {name: (size, digest)} + for the extra streams a filesystem has (NTFS alternate data streams, an + HFS+ resource fork).""" + out = {"path": path, "kind": kind} + if kind == "file": + out["size"] = size + out["digest"] = digest + if streams: + out["streams"] = {k: list(v) for k, v in sorted(streams.items())} + return out + + +# -- differences ----------------------------------------------------------------- + +def difference(facet, case, kind, key, detail=""): + return {"facet": facet, "case": case, "kind": kind, "key": key, + "detail": detail} + + +def compare_disks(case, ours, ref, facet="container"): + out = [] + if ours["size"] != ref["size"]: + out.append(difference(facet, case, DIFFERS, "size", + "Strata %d, reference %d" + % (ours["size"], ref["size"]))) + bad = sorted(n for n in set(ours["digests"]) & set(ref["digests"]) + if ours["digests"][n] != ref["digests"][n]) + if bad: + chunk = ours["chunk"] + first = bad[0] * chunk + out.append(difference( + facet, case, DIFFERS, "content", + "%d of %d chunks differ; the first at byte %d" + % (len(bad), len(set(ours["digests"]) & set(ref["digests"])), + first))) + for n in sorted(set(ours["digests"]) ^ set(ref["digests"])): + out.append(difference(facet, case, ERROR, "chunk %d" % n, + "read on one side only")) + return out + + +def compare_lists(case, facet, ours, ref, key): + """Two lists of dicts compared by `key(item)`: what is on one side only, + and which fields differ where both have it.""" + a = {key(x): x for x in ours} + b = {key(x): x for x in ref} + out = [] + for k in sorted(set(a) - set(b)): + out.append(difference(facet, case, ONLY_STRATA, k)) + for k in sorted(set(b) - set(a)): + out.append(difference(facet, case, ONLY_REF, k)) + for k in sorted(set(a) & set(b)): + fields = sorted(f for f in set(a[k]) | set(b[k]) + if f != "path" and a[k].get(f) != b[k].get(f)) + if fields: + out.append(difference( + facet, case, DIFFERS, k, "; ".join( + "%s: Strata %r, reference %r" + % (f, a[k].get(f), b[k].get(f)) for f in fields))) + return out + + +def compare_trees(case, ours, ref): + return compare_lists(case, "fs", ours, ref, lambda e: e["path"]) + + +# -- the baseline ------------------------------------------------------------------ + +class BaselineError(ValueError): + pass + + +def load_baseline(path): + """The accepted differences, checked. Each says which case, facet and kind + it covers, a pattern for the key, and why it is accepted; an entry + without a reason is refused, and so is one about a real image, so the file + can never name anyone's evidence.""" + if not os.path.exists(path): + return [] + with open(path, encoding="utf-8") as fh: + entries = json.load(fh) + for i, e in enumerate(entries): + for need in ("case", "facet", "kind", "key", "reason"): + if not e.get(need): + raise BaselineError("baseline entry %d has no %s" % (i, need)) + if not e["case"].startswith("synthetic:"): + raise BaselineError( + "baseline entry %d is about %r; only synthetic cases may be " + "listed" % (i, e["case"])) + if e["kind"] not in KINDS: + raise BaselineError("baseline entry %d: unknown kind %r" + % (i, e["kind"])) + return entries + + +def covers(entry, diff): + return (fnmatch.fnmatchcase(diff["case"], entry["case"]) + and entry["facet"] == diff["facet"] + and entry["kind"] == diff["kind"] + and fnmatch.fnmatchcase(diff["key"], entry["key"])) + + +def classify(diffs, baseline): + """(explained, unexplained, stale): differences a baseline entry covers, + those none does, and baseline entries nothing matched.""" + used = set() + explained, unexplained = [], [] + for d in diffs: + hit = next((i for i, e in enumerate(baseline) if covers(e, d)), None) + if hit is None: + unexplained.append(d) + else: + used.add(hit) + explained.append(dict(d, reason=baseline[hit]["reason"])) + stale = [e for i, e in enumerate(baseline) if i not in used] + return explained, unexplained, stale + + +# -- reporting ---------------------------------------------------------------------- + +class Redactor(object): + """Replaces names with stable tokens, so a report of a real image can say + how many files differ, and where in the tree, without saying what they + are called.""" + + def __init__(self, enabled): + self.enabled = enabled + + def path(self, p): + if not self.enabled or not p.startswith("/"): + return p + parts = [x for x in p.split("/") if x] + return "/" + "/".join("#" + sha(x.encode("utf-8", "replace"))[:6] + for x in parts) + + def text(self, s): + return s if not self.enabled else "(withheld)" + + +def summarise(diffs, redact): + """Lines for the console: counts per facet and kind, then each + difference.""" + lines = [] + for facet in sorted({d["facet"] for d in diffs}): + mine = [d for d in diffs if d["facet"] == facet] + counts = ", ".join("%d %s" % (sum(1 for d in mine if d["kind"] == k), + k.replace("_", " ")) + for k in KINDS + if any(d["kind"] == k for d in mine)) + lines.append(" %s: %s" % (facet, counts)) + for d in diffs: + key = redact.path(d["key"]) if d["facet"] == "fs" else d["key"] + detail = d.get("detail") or "" + if d["facet"] == "fs" and redact.enabled: + detail = redact.text(detail) if detail else "" + lines.append(" [%s] %s %s%s" % ( + d["kind"], d["case"], key, (" -- " + detail) if detail else "")) + return lines diff --git a/tests/crosscheck_refs.py b/tests/crosscheck_refs.py new file mode 100644 index 0000000..6a11e24 --- /dev/null +++ b/tests/crosscheck_refs.py @@ -0,0 +1,326 @@ +"""The reference readers: established tools that read the same formats as +Strata, reached through a thin adapter each so that what they report has the +same shape as what crosscheck_strata reports. + +An adapter is registered with its facet, the formats it can check, and what it +needs (Python modules, or programs on the PATH). One whose needs are missing is +listed as unavailable and skipped; nothing here is imported until it is used, +so Strata's own tests need none of them. + +Most are the Python bindings of the libyal libraries (libewf, libvhdi, +libqcow, libvmdk, libvsgpt, libfsntfs, libfshfs, libfsext, libfsfat, ...), +which are written and maintained independently of Strata and are what many +other forensic tools build on. `qemu-img` is the reference for the virtual +disk formats it reads (QCOW2, VDI, VHD, VMDK, DMG).""" + +import importlib +import os +import shutil +import subprocess +import tempfile + +import crosscheck_lib as lib + + +class Rejected(Exception): + """The reference reader refused the input or could not finish reading it. + Not a disagreement about content, and not agreement either; it is + reported, and has to be explained in the baseline.""" + + +class Reference(object): + + def __init__(self, facet, keys, name, modules, tools, run, hint): + self.facet, self.keys, self.name = facet, tuple(keys), name + self.modules, self.tools, self.run, self.hint = modules, tools, run, hint + + def available(self): + """(True, "") or (False, what is missing).""" + for m in self.modules: + try: + importlib.import_module(m) + except Exception: # noqa: BLE001 + return False, "needs the Python module %s%s" % ( + m, (" (" + self.hint + ")") if self.hint else "") + for t in self.tools: + if not shutil.which(t): + return False, "needs %s on the PATH%s" % ( + t, (" (" + self.hint + ")") if self.hint else "") + return True, "" + + +REFERENCES = [] + + +def reference(facet, keys, name, modules=(), tools=(), hint=""): + def register(fn): + REFERENCES.append(Reference(facet, keys, name, modules, tools, fn, + hint)) + return fn + return register + + +def references_for(facet, key): + return [r for r in REFERENCES if r.facet == facet and key in r.keys] + + +# -- whole-disk formats --------------------------------------------------------- + +def _from_handle(handle, plan, chunk): + size = handle.media_size + + def read_at(offset, length): + return handle.read_buffer_at_offset(length, offset) + return lib.disk_observation(read_at, size, plan, chunk) + + +@reference("container", ("raw",), "the file itself") +def _raw(path, plan, chunk, key=None): + size = os.path.getsize(path) + with open(path, "rb") as fh: + def read_at(offset, length): + fh.seek(offset) + return fh.read(length) + return lib.disk_observation(read_at, size, plan, chunk) + + +@reference("container", ("ewf",), "libewf", modules=("pyewf",), + hint="pip install libewf-python") +def _libewf(path, plan, chunk, key=None): + import pyewf + handle = pyewf.handle() + handle.open(pyewf.glob(path)) + try: + return _from_handle(handle, plan, chunk) + finally: + handle.close() + + +@reference("container", ("vhd", "vhdx"), "libvhdi", modules=("pyvhdi",), + hint="pip install libvhdi-python") +def _libvhdi(path, plan, chunk, key=None): + import pyvhdi + disk, parent = pyvhdi.file(), None + disk.open(path) + try: + name = disk.parent_filename + if name: + beside = os.path.join(os.path.dirname(path), + os.path.basename(name.replace("\\", "/"))) + if not os.path.exists(beside): + raise Rejected("the parent disk %r is not beside it" % name) + parent = pyvhdi.file() + parent.open(beside) + disk.set_parent(parent) + return _from_handle(disk, plan, chunk) + finally: + disk.close() + if parent: + parent.close() + + +@reference("container", ("qcow2",), "libqcow", modules=("pyqcow",), + hint="pip install libqcow-python") +def _libqcow(path, plan, chunk, key=None): + import pyqcow + handle = pyqcow.file() + handle.open(path) + try: + return _from_handle(handle, plan, chunk) + finally: + handle.close() + + +@reference("container", ("vmdk",), "libvmdk", modules=("pyvmdk",), + hint="pip install libvmdk-python") +def _libvmdk(path, plan, chunk, key=None): + import pyvmdk + handle = pyvmdk.handle() + handle.open(path) + try: + handle.open_extent_data_files() + return _from_handle(handle, plan, chunk) + finally: + handle.close() + + +QEMU_FORMATS = {"qcow2": "qcow2", "vdi": "vdi", "vhd": "vpc", "vmdk": "vmdk", + "dmg": "dmg"} + + +@reference("container", tuple(QEMU_FORMATS), "qemu-img", tools=("qemu-img",), + hint="apt-get install qemu-utils") +def _qemu(path, plan, chunk, key=None): + out = tempfile.mkdtemp(prefix="crosscheck-qemu-") + try: + raw = os.path.join(out, "disk.raw") + cmd = ["qemu-img", "convert", "-f", QEMU_FORMATS[key], "-O", "raw", + path, raw] + done = subprocess.run(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=600) + if done.returncode: + raise Rejected(done.stderr.decode("utf-8", "replace").strip()[:200]) + return _raw(raw, plan, chunk) + finally: + shutil.rmtree(out, ignore_errors=True) + + +# -- partition tables ------------------------------------------------------------- + +@reference("volumes", ("gpt",), "libvsgpt", modules=("pyvsgpt",), + hint="pip install libvsgpt-python") +def _libvsgpt(fileobj): + import pyvsgpt + volume = pyvsgpt.volume() + volume.open_file_object(fileobj) + try: + return [{"index": p.entry_index, "offset": p.volume_offset, + "size": p.size, + "type": str(p.type_identifier).strip("{}").lower(), + "identifier": str(p.identifier).strip("{}").lower()} + for p in volume.partitions] + finally: + volume.close() + + +# -- filesystems ---------------------------------------------------------------- + +MAX_DEPTH = 64 +MAX_ENTRIES = 500000 +S_IFMT, S_IFDIR, S_IFLNK = 0o170000, 0o040000, 0o120000 + + +def _digest(read, size, hash_bytes): + if not size: + return lib.sha(b"") + try: + return lib.sha(read(min(size, hash_bytes))) + except Exception as exc: # noqa: BLE001 + return "error:" + type(exc).__name__ + + +def _walk(root, hash_bytes, describe): + """Every file and folder below `root` (which has no name of its own), + depth first. `describe(entry)` gives (name, kind, size, read, streams, + children): `read(n)` returns the first n bytes, `streams` is + {name: (size, digest)} and `children` is a list for a folder, else None.""" + out = [] + stack = [("", c, 1) for c in reversed(describe(root)[5] or [])] + while stack and len(out) < MAX_ENTRIES: + base, entry, depth = stack.pop() + name, kind, size, read, streams, kids = describe(entry) + path = base + "/" + name + out.append(lib.entry_observation( + path, kind, size, + _digest(read, size, hash_bytes) if kind != "dir" else None, + streams)) + if kids is not None and depth < MAX_DEPTH: + stack.extend((path, k, depth + 1) for k in reversed(kids)) + out.sort(key=lambda e: e["path"]) + return out + + +def _reader(entry): + def read(n): + entry.seek_offset(0) + return entry.read_buffer(n) + return read + + +def _posix_kind(entry): + mode = entry.file_mode & S_IFMT + return ("dir" if mode == S_IFDIR else "link" if mode == S_IFLNK + else "file") + + +@reference("fs", ("ntfs",), "libfsntfs", modules=("pyfsntfs",), + hint="pip install libfsntfs-python") +def _libfsntfs(fileobj, hash_bytes): + import pyfsntfs + volume = pyfsntfs.volume() + try: + volume.open_file_object(fileobj) + except OSError as exc: + raise Rejected(str(exc)[:240]) + try: + def describe(e): + is_dir = e.has_directory_entries_index() + streams = {} + for i in range(e.number_of_alternate_data_streams): + s = e.get_alternate_data_stream(i) + streams[s.name] = (s.size, _digest( + _reader(s), s.size, hash_bytes)) + return (e.name, "dir" if is_dir else "file", e.size, + _reader(e), streams, + list(e.sub_file_entries) if is_dir else None) + root = volume.get_root_directory() + return _walk(root, hash_bytes, describe) + finally: + volume.close() + + +@reference("fs", ("hfsplus",), "libfshfs", modules=("pyfshfs",), + hint="pip install libfshfs-python") +def _libfshfs(fileobj, hash_bytes): + import pyfshfs + volume = pyfshfs.volume() + try: + volume.open_file_object(fileobj) + except OSError as exc: + raise Rejected(str(exc)[:240]) + try: + def describe(e): + kind = _posix_kind(e) + streams = {} + if kind == "file" and e.has_resource_fork(): + fork = e.get_resource_fork() + streams["rsrc"] = (fork.size, _digest( + _reader(fork), fork.size, hash_bytes)) + return (e.name, kind, e.size, _reader(e), streams, + list(e.sub_file_entries) if kind == "dir" else None) + return _walk(volume.root_directory, hash_bytes, describe) + finally: + volume.close() + + +@reference("fs", ("ext",), "libfsext", modules=("pyfsext",), + hint="pip install libfsext-python") +def _libfsext(fileobj, hash_bytes): + import pyfsext + volume = pyfsext.volume() + try: + volume.open_file_object(fileobj) + except OSError as exc: + raise Rejected(str(exc)[:240]) + try: + def describe(e): + kind = _posix_kind(e) + return (e.name, kind, e.size, _reader(e), {}, + list(e.sub_file_entries) if kind == "dir" else None) + try: + return _walk(volume.root_directory, hash_bytes, describe) + except OSError as exc: + raise Rejected(str(exc)[:240]) + finally: + volume.close() + + +@reference("fs", ("fat",), "libfsfat", modules=("pyfsfat",), + hint="pip install libfsfat-python") +def _libfsfat(fileobj, hash_bytes): + import pyfsfat + volume = pyfsfat.volume() + try: + volume.open_file_object(fileobj) + except OSError as exc: + raise Rejected(str(exc)[:240]) + try: + def describe(e): + flags = e.file_attribute_flags # none for the root + is_dir = flags is None or bool(flags & 0x10) + return (e.name, "dir" if is_dir else "file", e.size, + _reader(e), {}, + list(e.sub_file_entries) if is_dir else None) + return _walk(volume.root_directory, hash_bytes, describe) + finally: + volume.close() diff --git a/tests/crosscheck_strata.py b/tests/crosscheck_strata.py new file mode 100644 index 0000000..9a32d5b --- /dev/null +++ b/tests/crosscheck_strata.py @@ -0,0 +1,108 @@ +"""What Strata reports, in the form crosscheck_lib compares. + +Every function goes through the same engine entry points the application uses +(ewf.open_image, volume.scan, fs.open_fs, and the walker that search, hashing +and indexing share), not a private shortcut, so what is compared is what an +examiner would be shown.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import crosscheck_lib as lib # noqa: E402 +from engine import ewf, filesearch, volume # noqa: E402 +from engine.fs.ntfs import open_fs # noqa: E402 + +# The class name of the image object Strata made -> the format it read. +FORMATS = { + "EwfImage": "ewf", "RawImage": "raw", "VhdImage": "vhd", + "VhdxImage": "vhdx", "VmdkImage": "vmdk", "Qcow2Image": "qcow2", + "VdiImage": "vdi", "DmgImage": "dmg", "Aff4Image": "aff4", + "Ad1Image": "ad1", "RaidImage": "raid", "LogicalImage": "logical", +} + +# fs.name -> the key references are registered under +FS_KINDS = (("NTFS", "ntfs"), ("EXFAT", "exfat"), ("EXT", "ext"), + ("HFS", "hfsplus"), ("APFS", "apfs"), ("FAT", "fat")) + + +def image_format(img): + return FORMATS.get(type(img).__name__, type(img).__name__.lower()) + + +def fs_kind(fs): + name = (getattr(fs, "name", "") or "").upper() + for prefix, kind in FS_KINDS: + if name.startswith(prefix): + return kind + return None + + +def open_image(path): + return ewf.open_image(path) + + +def container(img, plan, chunk=lib.CHUNK): + return lib.disk_observation(img.read_at, img.size, plan, chunk) + + +def partitions(img): + """The GPT partitions Strata found, one dict each, with what a + reference partition reader also reports.""" + layout = volume.scan(img) + out = [] + for p in layout["partitions"]: + if p.get("allocated") is False or p.get("scheme") != "GPT": + continue + out.append({"index": p["index"], "offset": p["offset"], + "size": p["size"], "type": p["type_id"], + "identifier": p.get("guid")}) + return layout, out + + +def open_partition(img, offset, size): + return open_fs(ewf.OffsetReader(img, offset, size)) + + +def _read(fs, entry, limit, stream=""): + try: + data = fs.read_file(entry, limit, stream=stream) if stream \ + else fs.read_file(entry, limit) + except Exception as exc: # noqa: BLE001 + return "error:" + type(exc).__name__ + return lib.sha(data or b"") + + +def tree(fs, hash_bytes, budget=500000): + """Every live file and folder, as the shared walker finds them.""" + root = getattr(fs, "root_node", 0) + entries = filesearch.collect(fs, root, budget=budget) + skipped = [] # a deleted folder's contents are too + out = [] + for e in entries: + path = e.get("path") or "" + if e.get("deleted") or any(path.startswith(s + "/") for s in skipped): + if e.get("is_dir"): + skipped.append(path) + continue + if e.get("is_dir"): + out.append(lib.entry_observation(path, "dir")) + continue + size = e.get("size") or 0 + streams = {} + if hasattr(fs, "streams"): + try: + for s in fs.streams(e): + if not s.get("default"): + streams[s["name"]] = ( + s["size"], _read(fs, e, hash_bytes, s["name"])) + except Exception: # noqa: BLE001 + pass + if e.get("resource_size"): + streams["rsrc"] = (e["resource_size"], + _read(fs, e, hash_bytes, "rsrc")) + out.append(lib.entry_observation( + path, "file", size, _read(fs, e, min(size, hash_bytes)) + if size else lib.sha(b""), streams)) + return out diff --git a/tests/imagebuild_hfsplus.py b/tests/imagebuild_hfsplus.py index ad09ae8..732ce8c 100644 --- a/tests/imagebuild_hfsplus.py +++ b/tests/imagebuild_hfsplus.py @@ -224,8 +224,8 @@ def _fork_extents(size, extents, blocks): return struct.pack(">QII", size, 0, blocks) + ext.ljust(64, b"\x00") -def _perms(flags, special): - return struct.pack(">IIBBHI", 501, 20, 0, flags, 0o100644, special) +def _perms(flags, special, mode=0o100644): + return struct.pack(">IIBBHI", 501, 20, 0, flags, mode, special) def build(entries, block_size=4096, node_size=4096, name="TESTVOL", @@ -311,14 +311,14 @@ def take(nbytes): if isinstance(c, Dir): val = len(c.children) data = struct.pack(">HHIIIIIII", 1, 0, val, c.cnid, t, t, t, t, 0) - data += _perms(0, 0) + bytes(16) + bytes(16) + struct.pack(">II", 0, 0) + data += _perms(0, 0, 0o040755) + bytes(16) + bytes(16) + struct.pack(">II", 0, 0) cat.append((cat_sort(parent, c.name), cat_key(parent, c.name), data)) thread = struct.pack(">HHI", 3, 0, parent) + uname(c.name) cat.append((cat_sort(c.cnid, ""), cat_key(c.cnid, ""), thread)) # root folder record and thread rdata = struct.pack(">HHIIIIIII", 1, 0, len(root.children), 2, rec_time, rec_time, rec_time, rec_time, 0) - rdata += _perms(0, 0) + bytes(16) + bytes(16) + struct.pack(">II", 0, 0) + rdata += _perms(0, 0, 0o040755) + bytes(16) + bytes(16) + struct.pack(">II", 0, 0) cat.append((cat_sort(1, root.name), cat_key(1, root.name), rdata)) cat.append((cat_sort(2, ""), cat_key(2, ""), struct.pack(">HHI", 3, 0, 1) + uname(root.name))) diff --git a/tests/imagebuild_ntfs.py b/tests/imagebuild_ntfs.py index dc99905..ff55679 100644 --- a/tests/imagebuild_ntfs.py +++ b/tests/imagebuild_ntfs.py @@ -376,7 +376,7 @@ def sysfile(n, name, data_attrs=(), is_dir=False, extra=()): 0x80, [(BITMAP_LCN, 1)], (TOTAL_CLUSTERS + 7) // 8)]) sysfile(7, "$Boot", [nonresident_attr(0x80, [(0, 1)], CLUSTER)]) sysfile(8, "$BadClus", [resident_attr(0x80, b"")]) - sysfile(9, "$Secure", []) + sysfile(9, "$Secure", [resident_attr(0x80, b"", name="$SDS")]) sysfile(10, "$UpCase", [resident_attr(0x80, b"")]) sysfile(11, "$Extend", [], is_dir=True) for n in range(12, 16): diff --git a/tests/test_crosscheck.py b/tests/test_crosscheck.py new file mode 100644 index 0000000..3b31d5c --- /dev/null +++ b/tests/test_crosscheck.py @@ -0,0 +1,231 @@ +"""The cross-check harness itself (tests/crosscheck.py): that it cuts images +into the same pieces on both sides, finds differences, keeps the baseline +honest, and withholds names. None of this needs a reference reader.""" + +import json +import os +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.dirname(HERE)) + +import crosscheck as cc # noqa: E402 +import crosscheck_lib as lib # noqa: E402 +import crosscheck_refs as refs # noqa: E402 +import crosscheck_strata as ours # noqa: E402 + + +def entry(path, kind="file", size=3, digest="aa", streams=None): + return lib.entry_observation(path, kind, size, digest, streams) + + +class ChunkPlan(unittest.TestCase): + + def test_small_image_is_read_whole(self): + self.assertEqual(lib.chunk_plan(3 * lib.CHUNK + 1), [0, 1, 2, 3]) + self.assertEqual(lib.chunk_plan(0), []) + + def test_budget_keeps_both_ends_and_is_repeatable(self): + size = 100 * lib.CHUNK + a = lib.chunk_plan(size, 30, seed=7) + self.assertEqual(a, lib.chunk_plan(size, 30, seed=7)) + self.assertNotEqual(a, lib.chunk_plan(size, 30, seed=8)) + self.assertEqual(len(a), 30) + self.assertTrue(set(range(8)) <= set(a)) + self.assertTrue(set(range(92, 100)) <= set(a)) + + def test_slice_reads_the_same_bytes(self): + data = bytes(range(256)) * 8 + s = lib.SliceIO(lib.BytesSource(data), 100, 1000) + self.assertEqual(s.read(10), data[100:110]) + s.seek(-5, 2) + self.assertEqual(s.read(50), data[1095:1100]) + self.assertEqual(s.read(5), b"") + self.assertEqual(s.get_size(), 1000) + + +class Comparison(unittest.TestCase): + + def test_equal_trees_have_no_differences(self): + t = [entry("/a"), entry("/d", "dir")] + self.assertEqual(lib.compare_trees("c", t, list(t)), []) + + def test_each_kind_of_difference_is_named(self): + ours_ = [entry("/a"), entry("/b", digest="x"), entry("/s")] + theirs = [entry("/a"), entry("/b", digest="y"), entry("/r")] + found = {(d["kind"], d["key"]) + for d in lib.compare_trees("c", ours_, theirs)} + self.assertEqual(found, {(lib.ONLY_STRATA, "/s"), + (lib.ONLY_REF, "/r"), + (lib.DIFFERS, "/b")}) + + def test_streams_are_compared(self): + a = [entry("/a", streams={"x": (1, "d")})] + b = [entry("/a", streams={"x": (1, "e")})] + self.assertEqual(len(lib.compare_trees("c", a, b)), 1) + + def test_disk_content_and_size(self): + plan = [0] + good = lib.disk_observation(lambda o, n: b"a" * n, 10, plan) + bad = lib.disk_observation(lambda o, n: b"b" * n, 10, plan) + self.assertEqual(lib.compare_disks("c", good, good), []) + self.assertEqual([d["key"] for d in lib.compare_disks("c", good, bad)], + ["content"]) + short = lib.disk_observation(lambda o, n: b"a" * n, 11, plan) + self.assertIn("size", [d["key"] + for d in lib.compare_disks("c", good, short)]) + + def test_short_read_is_not_a_match(self): + obs = lib.disk_observation(lambda o, n: b"a" * (n - 1), 10, [0]) + self.assertTrue(obs["digests"][0].startswith("short:")) + + +class Baseline(unittest.TestCase): + + def write(self, entries): + fd, path = tempfile.mkstemp(suffix=".json") + with os.fdopen(fd, "w") as fh: + json.dump(entries, fh) + self.addCleanup(os.unlink, path) + return path + + def good(self, **kw): + e = {"case": "synthetic:x", "facet": "fs", "kind": "differs", + "key": "/a*", "reason": "because"} + e.update(kw) + return e + + def test_shipped_baseline_is_valid(self): + for e in lib.load_baseline(cc.BASELINE): + self.assertTrue(e["case"].startswith("synthetic:")) + + def test_entry_needs_a_reason(self): + with self.assertRaises(lib.BaselineError): + lib.load_baseline(self.write([self.good(reason="")])) + + def test_real_images_cannot_be_listed(self): + with self.assertRaises(lib.BaselineError): + lib.load_baseline(self.write([self.good(case="image:12345678")])) + + def test_unknown_kind_refused(self): + with self.assertRaises(lib.BaselineError): + lib.load_baseline(self.write([self.good(kind="odd")])) + + def test_classify(self): + base = [self.good(), self.good(key="/never")] + diffs = [lib.difference("fs", "synthetic:x", lib.DIFFERS, "/abc"), + lib.difference("fs", "synthetic:x", lib.DIFFERS, "/zzz"), + lib.difference("fs", "synthetic:x", lib.ONLY_REF, "/abc")] + explained, unexplained, stale = lib.classify(diffs, base) + self.assertEqual([d["key"] for d in explained], ["/abc"]) + self.assertEqual(len(unexplained), 2) + self.assertEqual([e["key"] for e in stale], ["/never"]) + + +class Redaction(unittest.TestCase): + + def test_names_are_replaced_stably(self): + r = lib.Redactor(True) + a = r.path("/Secret folder/plan.doc") + self.assertNotIn("Secret", a) + self.assertNotIn("plan", a) + self.assertEqual(a, r.path("/Secret folder/plan.doc")) + self.assertEqual(a.count("/"), 2) + self.assertEqual(r.path("/Secret folder").split("/")[1], + a.split("/")[1]) + + def test_summary_withholds_names_and_detail(self): + d = [lib.difference("fs", "image:1", lib.DIFFERS, "/Secret/x.doc", + "size: Strata 1, reference 2")] + text = "\n".join(lib.summarise(d, lib.Redactor(True))) + self.assertNotIn("Secret", text) + self.assertNotIn("x.doc", text) + shown = "\n".join(lib.summarise(d, lib.Redactor(False))) + self.assertIn("Secret", shown) + + +class Runner(unittest.TestCase): + """The runner end to end, with only the always-present reference (the + file itself) and Strata.""" + + def run_case(self, name): + case = next(c for c in cc.corpus.synthetic() + if c.id == "synthetic:" + name) + opts = cc.argparse.Namespace(chunks=None, seed=1, hash_bytes=1 << 20, + trace=False) + run = cc.Run(opts) + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "image" + case.suffix) + with open(path, "wb") as fh: + fh.write(case.build()) + cc.run_case(run, case.id, path) + return run + + def test_raw_image_agrees_with_the_file(self): + run = self.run_case("fat12") + self.assertIn(("synthetic:fat12", "container", "the file itself", + cc.AGREE, ""), run.coverage) + + def test_every_case_opens_and_reads_back_as_itself(self): + for case in cc.corpus.synthetic(): + with self.subTest(case=case.id): + run = self.run_case(case.id.split(":")[1]) + bad = [d for d in run.diffs + if d["facet"] == "container" + and d["key"].startswith("strata")] + self.assertEqual(bad, []) + + def test_a_reference_that_disagrees_is_reported(self): + saved = list(refs.REFERENCES) + + def liar(path, plan, chunk, key=None): + with open(path, "rb") as fh: + size = os.path.getsize(path) + return lib.disk_observation(lambda o, n: b"\xFF" * n, size, + plan, chunk) + refs.REFERENCES.append(refs.Reference( + "container", ("raw",), "liar", (), (), liar, "")) + try: + run = self.run_case("ext2") + finally: + refs.REFERENCES[:] = saved + self.assertTrue([d for d in run.diffs + if d["facet"] == "container" + and d["kind"] == lib.DIFFERS]) + + def test_a_refusal_is_a_difference_not_a_crash(self): + saved = list(refs.REFERENCES) + + def refuse(fileobj, hash_bytes): + raise refs.Rejected("will not") + refs.REFERENCES.append(refs.Reference( + "fs", ("fat",), "refuser", (), (), refuse, "")) + try: + run = self.run_case("fat16") + finally: + refs.REFERENCES[:] = saved + self.assertIn("refuser:refused", [d["key"] for d in run.diffs]) + + def test_tree_matches_a_known_listing(self): + case = next(c for c in cc.corpus.synthetic() + if c.id == "synthetic:hfsplus") + from engine import ewf + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "i.img") + with open(path, "wb") as fh: + fh.write(case.build()) + img = ewf.open_image(path) + try: + fs = ours.open_partition(img, 0, img.size) + paths = sorted(e["path"] for e in ours.tree(fs, 1 << 20)) + finally: + img.close() + self.assertEqual(paths, ["/Top", "/Top/Inner", "/Top/Inner/deep.txt", + "/Top/a.txt", "/root.txt"]) + + +if __name__ == "__main__": + unittest.main()