diff --git a/.github/ct.yaml b/.github/ct.yaml new file mode 100644 index 00000000..59bc3300 --- /dev/null +++ b/.github/ct.yaml @@ -0,0 +1,7 @@ +chart-dirs: + - deploy/helm +# The chart version comes from the git tag at publish time, not from a bump +# in each commit, so ct must not require a version increment here. +check-version-increment: false +validate-maintainers: false +target-branch: main diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4f43908..c919b1d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: components: clippy, rustfmt @@ -23,42 +23,48 @@ jobs: # examples and the `bytecode_gating` integration test) is excluded from the # default build, so lint it in a dedicated pass. - run: cargo clippy -p exav-core --all-targets --features unstable-internals -- -D warnings - # The full native test matrix (default + http + checksums + no-decrypt + - # unstable-internals feature passes). Defined once in the Makefile so this - # and a local `make test` can't drift. + - run: cargo clippy -p exav-render --all-targets --features dwg -- -D warnings + - run: cargo clippy -p exav-render --all-targets --no-default-features --features ifc,stl -- -D warnings + # The full native test matrix, every feature pass. Defined once in the + # Makefile so this and a local `make test` can't drift. - run: make test-native - # The promised floor is `rust-version` in Cargo.toml (1.91, imposed by - # yara-x-parser). `make msrv` reads it there and installs that toolchain on - # demand, so a bump flows through with no edit here — and a contributor on + # The promised floor is `rust-version` in Cargo.toml (1.92, imposed by the + # hayro image decoders). `make msrv` reads it there and installs that toolchain on + # demand, so a bump flows through with no edit here, and a contributor on # stable can't land a newer feature without this going red. msrv: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: make msrv # The yara-x A/B differential harness. Deliberately NOT part of `make test` - # (see the Makefile) — a differential test can go red because the other engine + # (see the Makefile): a differential test can go red because the other engine # changed, so it is not a merge gate. It runs here anyway, unlike the clamav - # harness, only because it is self-contained: no signature database, no second - # engine to install. Its own job so the optional `yara-x` dependency — 83 extra - # crates, 21 of them cranelift/wasmtime — never lands in the `test` job, and so - # it costs no wall clock. + # harness, only because it is self-contained: no signature database, just + # yara-x's `yr` binary. The tests skip when `yr` is missing, so it is installed + # and checked first; without it this job would pass having compared nothing. yara-diff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + - name: Install yr (yara-x CLI) + run: | + YR=v1.21.0 + curl -sSL "https://github.com/VirusTotal/yara-x/releases/download/$YR/yara-x-$YR-x86_64-unknown-linux-gnu.tar.gz" | tar xz -C /tmp + echo "/tmp" >> "$GITHUB_PATH" + - run: yr --version - run: make test-yara-diff wasm: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-wasip1, wasm32-unknown-unknown @@ -66,7 +72,7 @@ jobs: - name: Install binaryen (wasm-opt) run: sudo apt-get update && sudo apt-get install -y binaryen # The scanning core compiled to WASI, and the archive-extractor WASM - # bindings (excluded from the workspace, built on their own target) — both + # bindings (excluded from the workspace, built on their own target), both # the default (all-formats + decrypt) build and a minimal single-format, # no-decrypt build, so the feature-gated paths can't silently rot. - run: cargo build -p exav-core --features wasi-bin --target wasm32-wasip1 @@ -97,11 +103,28 @@ jobs: with: node-version: 22 - run: make test-js + # @exav/viewer: its wasm modules and its archive plugin's peer + # (exav-unpack-wasm's pkg/, which test-js built with test-only features) + # are rebuilt from source, with the wasm-bindgen-cli its Cargo.toml pins, + # and its demo is driven in Chromium, its sandboxed frame in Chromium, + # Firefox and WebKit. See scripts/test-viewer.sh. + - uses: taiki-e/install-action@v2 + with: + tool: wasm-bindgen@0.2.129 + - run: make test-viewer + # What the browser saw, for the failures that do not happen on a + # developer's machine. + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: viewer-e2e-results + path: crates/exav-viewer/test-results + if-no-files-found: ignore docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: actions/setup-node@v4 with: node-version: 22 @@ -116,7 +139,7 @@ jobs: audit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - name: cargo-audit run: | @@ -137,16 +160,57 @@ jobs: run: | cargo deny --manifest-path crates/exav-unpack-wasm/Cargo.toml \ --config deny.toml check advisories bans sources licenses + # The viewer's wasm modules, also its own workspace. Every feature: the + # DWG module is not a default. + - name: cargo-deny (viewer) + run: | + cargo deny --manifest-path crates/exav-viewer/Cargo.toml --all-features \ + --config deny.toml check advisories bans sources licenses fuzz-smoke: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@nightly - uses: Swatinem/rust-cache@v2 + with: + workspaces: | + . -> target + fuzz -> target + # What the targets found interesting last time, so a 60 s run starts + # where the last one stopped. A new key each run, the newest restored. + - uses: actions/cache@v4 + with: + path: fuzz/corpus + key: fuzz-corpus-${{ github.run_id }} + restore-keys: fuzz-corpus- + - name: The fuzz crate's own tests (the mutator and the input framing) + run: cargo test --manifest-path fuzz/Cargo.toml -p exav-fuzz-support - name: cargo-fuzz smoke run run: | cargo install cargo-fuzz --locked - for t in filetype sigs cvd pe unpack analyze bytecode full_pipeline ndb_compile rar3_ppmd; do - cargo +nightly fuzz run "$t" -- -max_total_time=60 -rss_limit_mb=2048 -timeout=10 + # Seeded from the test fixtures where they match a target's input: + # random bytes rarely get past a header. Outside the checkout, as + # the masked fixtures are unmasked. + seeds="$RUNNER_TEMP/fuzz-seeds" + scripts/fuzz-seeds.sh "$seeds" + for t in filetype sigs cvd pe unpack analyze bytecode full_pipeline seekable yara pcre clamd icap volumes authenticode database ndb_compile rar3_ppmd pe_emulator x86_decode parser_recursion imagehash drawing cad_dxf cad_dwg ifc; do + mkdir -p "fuzz/corpus/$t" + extra="" + [ -d "$seeds/$t" ] && extra="$seeds/$t" + # Strings for the targets whose input is text or a protocol. + dict="" + case "$t" in + yara|pcre|icap|clamd) dict="-dict=fuzz/dicts/$t.dict" ;; + full_pipeline|ndb_compile|sigs|cvd) dict="-dict=fuzz/dicts/signatures.dict" ;; + esac + # iced-x86, the oracle x86_decode compares against, builds its + # decoder tables in a `lazy_static` it never frees: the newest + # nightly's LeakSanitizer reports them at exit (552 bytes, whatever + # the input), which is not a leak of exav's. The other targets keep + # the check. + asan="" + [ "$t" = x86_decode ] && asan="detect_leaks=0" + ASAN_OPTIONS="$asan" cargo +nightly fuzz run "$t" "fuzz/corpus/$t" $extra \ + -- $dict -max_total_time=60 -rss_limit_mb=2048 -timeout=10 done diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 46cac9d2..d86142dc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,8 @@ # binaries natively on amd64 (via `cross`, NOT under QEMU) and `publish` # assembles the multi-arch image around them with `COPY` — compiling the # Rust+ring tree emulated for arm64 is what used to make publishing slow. +# `publish` then re-checks the pushed tag, because the image must carry Docker +# (not OCI) mediatypes for podman to honour its HEALTHCHECK; see that step. # Pull with: # docker run -d -p 3310:3310 -v exav-db:/var/lib/exav ghcr.io/sylvinus/exav name: docker @@ -23,7 +25,7 @@ jobs: build-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: docker/setup-buildx-action@v3 - name: Build image (amd64, load into docker) uses: docker/build-push-action@v6 @@ -46,6 +48,12 @@ jobs: exit 1 fi docker logs exav + # The image's HEALTHCHECK is on the container, and its command works. + # Not `docker inspect .State.Health`: the first probe is 30s away. + HC=$(docker inspect --format '{{json .Config.Healthcheck.Test}}' exav) + echo "healthcheck: $HC" + [ "$HC" = '["CMD","/exav","--ping"]' ] + docker exec exav /exav --ping docker rm -f exav # Static musl binaries for the published image, cross-compiled natively on @@ -64,7 +72,7 @@ jobs: arch: arm64 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - name: Install cross run: cargo install cross --locked @@ -97,7 +105,7 @@ jobs: contents: read packages: write # push to ghcr.io steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 # No QEMU anywhere in this job: each platform build only COPYs its # prebuilt binary (plus mkdir/chown, pinned to the build machine in the # Dockerfile), so there is nothing to emulate. @@ -125,9 +133,116 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64 - push: true # Take bins/exav-$TARGETARCH from the `binaries` job instead of # compiling in-image (Dockerfile documents both branches). build-args: BIN_SOURCE=prebuilt tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # Publish Docker manifest-list/schema2 mediatypes rather than OCI + # ones, so that podman honours the image's HEALTHCHECK. + # + # `Healthcheck` is a Docker config extension with no equivalent in + # the OCI image spec. Nothing strips it: flipping this exporter + # option leaves the config blob byte-identical (same sha256), and + # only the declared mediatype changes. But podman reads the field + # back solely from a schema2 config — containers/common fills + # `HealthCheck` in the `DockerV2Schema2MediaType` branch of + # libimage/inspect.go and in no other branch — so under OCI + # mediatypes it reports "has no defined healthcheck" however healthy + # the daemon actually is. That leaves a rootless podman deployment + # with no readiness signal at all: the image is distroless, so a + # compose-level `test:` has no shell to run in either. Docker reads + # the field from both configs, so this is a no-op for `docker run`. + # + # `provenance: false` is load-bearing, not tidying: attestations can + # only be referenced from an OCI index, so buildx's default + # provenance attestation would force OCI mediatypes straight back. + # Attaching SLSA provenance as a separate referrer instead (e.g. + # actions/attest-build-provenance with push-to-registry) leaves the + # image's own mediatype alone. + # + # `push` moves into `outputs` so there is a single image exporter + # making a single mediatype decision. + provenance: false + outputs: type=image,oci-mediatypes=false,push=true + + # The `build-test` job above smoke-tests a locally `load`ed image, which + # cannot catch a *push-time* mediatype change — the exporter is what + # differs. These two steps therefore run against the pushed tag. + - name: Assert the published manifest is a Docker manifest list + env: + # Via env: rather than inlined into `run:`, so a tag reaches bash as + # data and not as code. + TAGS: ${{ steps.meta.outputs.tags }} + EXPECTED: application/vnd.docker.distribution.manifest.list.v2+json + run: | + set -euo pipefail + # The tags are already public when this runs: a failure here is an + # alarm to fix and re-publish, not a gate. + bad=0 + while read -r TAG; do + [ -n "$TAG" ] || continue + GOT=$(docker buildx imagetools inspect "$TAG" --raw | jq -r '.mediaType') + echo "$TAG -> $GOT" + if [ "$GOT" != "$EXPECTED" ]; then + echo "::error::$TAG is already published as $GOT, expected $EXPECTED -- podman will ignore the image HEALTHCHECK" + bad=1 + fi + done <<<"$TAGS" + exit "$bad" + + # The assertion above pins the wire format; this one tests the consumer + # that the format exists for. podman is preinstalled on ubuntu-latest. + - name: Verify podman sees the HEALTHCHECK on the published image + env: + TAGS: ${{ steps.meta.outputs.tags }} + REGISTRY_USER: ${{ github.actor }} + REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TAG=$(printf '%s\n' "$TAGS" | head -1) + podman --version + # Log in rather than relying on an anonymous pull: a GHCR package + # starts out private, so the very first publish of a fork would + # otherwise fail here for the wrong reason. + podman login ghcr.io -u "$REGISTRY_USER" --password-stdin <<<"$REGISTRY_PASSWORD" + podman pull -q "$TAG" + + # This coming back empty *is* the regression being guarded against: + # podman populates it only from a Docker schema2 config. + IMAGE_HC=$(podman image inspect \ + --format '{{if .HealthCheck}}{{.HealthCheck.Test}}{{end}}' "$TAG") + echo "image healthcheck: ${IMAGE_HC:-}" + if [ -z "$IMAGE_HC" ]; then + echo "::error::podman reports no healthcheck on $TAG" + exit 1 + fi + + # Same env as the `build-test` smoke test: no mirror configured, so + # opt into the built-in EICAR-only baseline and skip the wait for a + # sidecar that is not coming. + podman run -d --name exav-podman \ + -e EXAV_STARTUP_WAIT_SECS=0 -e EXAV_ALLOW_NO_DB=1 "$TAG" + CONTAINER_HC=$(podman inspect \ + --format '{{if .Config.Healthcheck}}{{.Config.Healthcheck.Test}}{{end}}' exav-podman) + echo "container healthcheck: ${CONTAINER_HC:-}" + if [ -z "$CONTAINER_HC" ]; then + echo "::error::healthcheck did not reach the container config" + podman rm -f exav-podman + exit 1 + fi + + # `/exav --ping` answers only once the database is loaded, so poll + # instead of asserting on the first probe. + for _ in $(seq 60); do + if podman healthcheck run exav-podman; then + echo "podman reports: $(podman inspect --format '{{.State.Health.Status}}' exav-podman)" + podman rm -f exav-podman + exit 0 + fi + sleep 1 + done + echo "::error::podman healthcheck never passed" + podman logs exav-podman + podman rm -f exav-podman + exit 1 diff --git a/.github/workflows/fuzz-nightly.yml b/.github/workflows/fuzz-nightly.yml new file mode 100644 index 00000000..08d96070 --- /dev/null +++ b/.github/workflows/fuzz-nightly.yml @@ -0,0 +1,58 @@ +name: fuzz-nightly + +# A longer run of every fuzz target than the 60 seconds a push gets, in fork +# mode so that one run yields every distinct crash and not the first +# (scripts/fuzz-campaign.sh). What it finds is uploaded, never committed: +# triage is docs/FUZZING.md's. +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + total: + description: "Seconds of fuzzing, split across the targets" + default: "10800" + +permissions: + contents: read + +jobs: + campaign: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@nightly + - uses: Swatinem/rust-cache@v2 + with: + workspaces: | + . -> target + fuzz -> target + # The corpora of earlier nights: each one starts where the last stopped. + - uses: actions/cache@v4 + with: + path: tmp/data + key: fuzz-nightly-${{ github.run_id }} + restore-keys: fuzz-nightly- + - name: cargo-fuzz campaign + run: | + cargo install cargo-fuzz --locked + # A hosted runner has the memory the campaign script's defaults + # hold back for a small VM. + CARGO_BUILD_JOBS=2 CODEGEN_UNITS=16 TOTAL="${{ inputs.total || 10800 }}" scripts/fuzz-campaign.sh + - uses: actions/upload-artifact@v4 + if: always() + with: + name: fuzz-artifacts + path: fuzz/artifacts + if-no-files-found: ignore + # Fork mode keeps going after a crash, so the job's status says nothing: + # a crash, a hang or an out-of-memory is a file here (a `slow-unit` is + # only slow). + - name: Fail if the campaign found a crash, a timeout or an OOM + run: | + found="$(find fuzz/artifacts -type f \( -name 'crash-*' -o -name 'timeout-*' -o -name 'oom-*' \) 2>/dev/null)" + if [ -n "$found" ]; then + echo "$found" + exit 1 + fi diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml new file mode 100644 index 00000000..0d6daf43 --- /dev/null +++ b/.github/workflows/helm.yml @@ -0,0 +1,68 @@ +# Lint and (on version tags) publish the Helm chart. +# +# `lint` runs on every push/PR: it runs `ct lint`, then `make test-helm`, the +# same render checks a contributor runs locally. `publish` runs only on `v*` +# tags: it checks the tag against the workspace version in Cargo.toml, then +# pushes the chart to ghcr.io as an OCI artifact. Install with: +# helm install exav oci://ghcr.io/sylvinus/exav-chart --version \ +# --set 'signatures.sources={https://mirror.example/db/}' +name: helm + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # ct needs history to diff the target branch + # `make test-helm` runs the repository's own script: it gets no token. + persist-credentials: false + - uses: azure/setup-helm@v5 + - uses: helm/chart-testing-action@v2.8.0 + - name: ct lint + run: ct lint --all --config .github/ct.yaml + - run: make test-helm + + publish: + needs: lint + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # push to ghcr.io + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: azure/setup-helm@v5 + - name: Verify the tag matches the workspace version + run: | + set -euo pipefail + tag_version="${GITHUB_REF_NAME#v}" + cargo_version=$(sed -n '/^\[workspace\.package\]/,/^\[/{s/^version = "\(.*\)"$/\1/p}' Cargo.toml | head -n1) + if [ "$tag_version" != "$cargo_version" ]; then + echo "tag version ($tag_version) does not match Cargo.toml workspace version ($cargo_version)" >&2 + exit 1 + fi + echo "CHART_VERSION=$tag_version" >> "$GITHUB_ENV" + - name: Log in to ghcr.io + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_ACTOR: ${{ github.actor }} + run: echo "$GH_TOKEN" | helm registry login ghcr.io --username "$GH_ACTOR" --password-stdin + - name: Package and push the chart + run: | + set -euo pipefail + v="$CHART_VERSION" + owner="${GITHUB_REPOSITORY_OWNER,,}" + helm package deploy/helm/exav-chart --version "$v" --app-version "$v" -d dist + helm push "dist/exav-chart-$v.tgz" "oci://ghcr.io/$owner" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc19543c..827b13ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,9 @@ # Tag-driven binary releases. Push a tag like `v0.1.0` and this cross-compiles -# the `exav` CLI for each target, archives it, and attaches it to the GitHub -# Release for that tag. Static musl Linux builds run on any distro (no glibc). +# the `exav` CLI and the standalone `exav-unpack`, `exav-grep`, `exav-pe-emu` +# and `exav-imagehash` for each target, archives each with LICENSE and NOTICE, and +# attaches them to the +# GitHub Release for that tag. Static musl Linux builds run on any distro (no +# glibc). # The `wasm` job adds the two WebAssembly artifacts: the unpacker npm tarball # (`@exav/unpack-wasm`, standard preset — exactly what `npm publish` ships via # `prepublishOnly`) and the engine as a WASI module (`exav-core --features @@ -46,7 +49,7 @@ jobs: os: windows-latest runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable # Builds the `exav` binary, cross-compiling musl targets # via `cross` as needed, archives it as `exav-.tar.gz`/`.zip`, and @@ -58,6 +61,42 @@ jobs: archive: exav-$tag-$target # YARA rule support is on by default; keep it for full parity. # For a smaller binary, add: features: "" / no-default-features. + # The licenses of everything linked go with every binary (NOTICE). + include: LICENSE,NOTICE + token: ${{ secrets.GITHUB_TOKEN }} + checksum: sha256 + # The standalone tools, each in an archive of its own: most people + # want one of them, not the scanner. + - uses: taiki-e/upload-rust-binary-action@v1 + with: + bin: exav-unpack + target: ${{ matrix.target }} + archive: exav-unpack-$tag-$target + include: LICENSE,NOTICE + token: ${{ secrets.GITHUB_TOKEN }} + checksum: sha256 + - uses: taiki-e/upload-rust-binary-action@v1 + with: + bin: exav-grep + target: ${{ matrix.target }} + archive: exav-grep-$tag-$target + include: LICENSE,NOTICE + token: ${{ secrets.GITHUB_TOKEN }} + checksum: sha256 + - uses: taiki-e/upload-rust-binary-action@v1 + with: + bin: exav-pe-emu + target: ${{ matrix.target }} + archive: exav-pe-emu-$tag-$target + include: LICENSE,NOTICE + token: ${{ secrets.GITHUB_TOKEN }} + checksum: sha256 + - uses: taiki-e/upload-rust-binary-action@v1 + with: + bin: exav-imagehash + target: ${{ matrix.target }} + archive: exav-imagehash-$tag-$target + include: LICENSE,NOTICE token: ${{ secrets.GITHUB_TOKEN }} checksum: sha256 @@ -66,7 +105,7 @@ jobs: needs: create-release runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-unknown-unknown,wasm32-wasip1 diff --git a/.github/workflows/www.yml b/.github/workflows/www.yml index ddf3d572..bdf952f5 100644 --- a/.github/workflows/www.yml +++ b/.github/workflows/www.yml @@ -1,8 +1,5 @@ -# Sample workflow for building and deploying an Astro site to GitHub Pages -# -# To get started with Astro see: https://docs.astro.build/en/getting-started/ -# -name: Deploy Astro site to Pages +# Builds the website and the viewer demo, and deploys them to GitHub Pages. +name: Deploy site to Pages on: # Runs on pushes targeting the default branch @@ -33,7 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Setup Node uses: actions/setup-node@v7 with: @@ -52,6 +49,36 @@ jobs: --site "${{ steps.pages.outputs.origin }}" \ --base "${{ steps.pages.outputs.base_path }}" working-directory: ${{ env.BUILD_PATH }} + # The @exav/viewer demo, served at /viewer/demo/: the package and its wasm + # modules built from this commit, the archive engine's pkg/ too. + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + with: + workspaces: | + crates/exav-viewer + crates/exav-unpack-wasm + - name: Install binaryen (wasm-opt) + run: sudo apt-get update && sudo apt-get install -y binaryen + - uses: taiki-e/install-action@v2 + with: + tool: wasm-bindgen@0.2.129,wasm-pack + - name: Build the viewer demo + run: | + (cd crates/exav-unpack-wasm && wasm-pack build --release --target web) + cd crates/exav-viewer + npm ci + npm run build:wasm + npm run build + # The sidebar's files: github.com/sylvinus/exav-samples, HEAD. + npm run demo:showcase + # DEMO_ORIGIN goes into the sandboxed frame's policy, for Safari. + DEMO_BASE="${{ steps.pages.outputs.base_path }}/viewer/demo/" DEMO_ORIGIN="${{ steps.pages.outputs.origin }}" npm run demo:build + # dist/viewer/ holds the File viewer docs; the demo must not land on them. + test ! -e "$GITHUB_WORKSPACE/${{ env.BUILD_PATH }}/dist/viewer/demo" + cp -R demo/dist "$GITHUB_WORKSPACE/${{ env.BUILD_PATH }}/dist/viewer/demo" + test -f "$GITHUB_WORKSPACE/${{ env.BUILD_PATH }}/dist/viewer/demo/index.html" - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/.gitignore b/.gitignore index ee137e36..cd36514a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,9 @@ Cargo.lock.orig # real-malware test samples: kept locally (gitignored) for extra decoder # coverage, NEVER committed — the tests that use them skip when absent, and CI # relies on the synthetic/EICAR fixtures. Provenance (sha256) is in each dir's -# README.md. The synthetic `eicar.lnk` and the READMEs stay committed. +# README.md. Each is kept as `.zip`, AES-256 with the password +# `infected` (crates/exav-unpack/tests/fixtures/README.md). The synthetic +# `eicar.lnk` and the READMEs stay committed. crates/exav-unpack/tests/fixtures/**/real-malware-* /yara-x/ # internal strategy notes — never commit diff --git a/.oss-scanner/Dockerfile b/.oss-scanner/Dockerfile new file mode 100644 index 00000000..420c5858 --- /dev/null +++ b/.oss-scanner/Dockerfile @@ -0,0 +1,44 @@ +# Build environment for Anthropic's OSS Scanner (https://red.anthropic.com/oss-scanner/). +# The checkout is at /src, the build context is the repository, and nothing +# after the build has network access: every dependency is fetched here. +# +# Build it the way the scanner does, from the repository root: +# docker build -f .oss-scanner/Dockerfile -t exav-oss-scanner . +FROM rust:1.92-bookworm + +# Frame pointers and line tables for a debugger, without the full debug info +# that makes the build several times larger. +ENV DEBIAN_FRONTEND=noninteractive \ + CARGO_TERM_COLOR=never \ + CARGO_PROFILE_DEV_DEBUG=line-tables-only \ + RUSTFLAGS="-C force-frame-pointers=yes" + +# gdb and valgrind are for the agent that reproduces a finding; clang is what a +# C dependency's build script may look for. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates build-essential clang pkg-config git python3 gdb valgrind \ + && rm -rf /var/lib/apt/lists/* + +COPY . /src +WORKDIR /src + +# Every dependency of every crate, test-only ones included, so that nothing +# is fetched after this step. `--locked` uses exactly Cargo.lock. +RUN cargo fetch --locked + +# Every crate of the workspace, in the dev profile (overflow checks on, as in +# the release profile). `exav` is built with the optional network-facing half +# (`SCANURL`, `--sig-sources`), which pulls the TLS stack. +RUN cargo build --workspace --locked \ + && cargo build -p exav --locked --features http + +# The tools, on PATH. They read a file given on the command line; `exav` needs +# a signature database to scan, which `threat_model.md` says how to make. +RUN for t in exav exav-unpack exav-grep exav-imagehash; do \ + ln -s "/src/target/debug/$t" "/usr/local/bin/$t"; \ + done + +# A cheap check that the image works: the unit tests of the extractor. A +# failure is reported and does not fail the image. +RUN (cargo test -p exav-unpack --lib --locked 2>&1 | tail -n 5) \ + || echo "WARNING: the exav-unpack unit tests reported failures" diff --git a/.oss-scanner/Dockerfile.dockerignore b/.oss-scanner/Dockerfile.dockerignore new file mode 100644 index 00000000..7d75e32e --- /dev/null +++ b/.oss-scanner/Dockerfile.dockerignore @@ -0,0 +1,10 @@ +# Used instead of the repository's .dockerignore for this Dockerfile: the +# scanner should see the docs, the fuzz targets and the test corpus, which the +# image of the product leaves out. Only what is built or fetched is left out. +**/target/ +**/node_modules/ +.git/ +.env* +*.exavdb +exav-db/ +tmp/ diff --git a/.oss-scanner/threat_model.md b/.oss-scanner/threat_model.md new file mode 100644 index 00000000..83d03a1c --- /dev/null +++ b/.oss-scanner/threat_model.md @@ -0,0 +1,111 @@ +# Threat model + +exav is a set of memory-safe tools for files the user did not write: a malware +scanner (`exav`, with a clamd-compatible daemon and an ICAP server), a bounded +extractor (`exav-unpack`), a signature-aware grep (`exav-grep`), an image +hasher (`exav-imagehash`), the renderers behind a browser viewer +(`exav-render`, `exav-viewer`) and WebAssembly bindings (`exav-unpack-wasm`). +`SECURITY.md` at the root is the maintainers' own statement of the same model. + +## What this project does and where untrusted input enters + +- **A scanned file is fully hostile.** Every byte of it, at any depth of + nesting: `crates/exav-core` (file typing, PE/ELF/Mach-O parsing, the + bytecode interpreter, the YARA engine, `.ndb`/`.ldb` matching), + `crates/exav-unpack` (about 80 archive, disk image, filesystem, document and + packer decoders), `crates/exav-x86` and `crates/exav-pe-emu` (an x86 + emulator that runs packer stubs), and `crates/exav-render` (PDF, image, DWG, + DXF, IFC, STL decoding for the viewer). +- **Network input**: the daemon's clamd protocol (`crates/exav/src/daemon.rs`), + the ICAP listener (`crates/exav/src/icap/`, which parses straight off the + socket), and with `--features http` the `SCANURL` command and the signature + updater (`crates/exav-update`). +- **Command-line and daemon arguments** reach `exav-imagehash`, `exav-grep` + and the `exav` options; they come from the operator, not from a file. +- **The 32-bit WebAssembly build** (`crates/exav-unpack-wasm`, + `crates/exav-viewer`) has a 32-bit `usize`, no overflow checks and + `panic=abort`. A sum of two header numbers that cannot wrap on a 64-bit host + can wrap there and defeat a bounds check. + +## Components that matter most / least + +Most: the decoders in `crates/exav-unpack/src/formats/`, the PE emulator +(`crates/exav-pe-emu`, `crates/exav-x86`), the bytecode interpreter +(`crates/exav-core/src/bytecode/`), the signature matching engine +(`crates/exav-core/src/engine/`), the daemon and ICAP code, and the code that +decides a result is `OK` rather than `PARTIAL`. + +Least: `www/` (the documentation site), `scripts/`, `packaging/`, `corpus/`, +`tmp/`, the test suites and fixtures, and `crates/exav-viewer`'s JavaScript +beyond the places it hands bytes to the Rust or wasm decoders. + +## How to exercise it + +- Everything is built in `/src/target/debug` and on `PATH`: `exav`, + `exav-unpack`, `exav-grep`, `exav-imagehash`. +- `exav-unpack FILE` extracts any supported container in memory with the + default budgets; this is the shortest path to a decoder. `exav-unpack -l` + lists. It never writes to a path named by the archive. +- `exav` refuses to scan with no signature database. A minimal one for a + test is a `.ndb` file in a directory: `echo 'Test.Sig:0:*:4558415600' > + /tmp/db/t.ndb`, then `exav -d /tmp/db FILE`. A `.cbc` bytecode program or + a YARA rule file in the same directory exercises those engines. +- A decoder panic shows as `decoder panicked` in the output; this is how the + tests recognise one. +- `fuzz/fuzz_targets/` has a `cargo-fuzz` target for each parser and for the + whole pipeline (`full_pipeline`, `unpack`). The image has no nightly + toolchain, so write a proof of concept as an input file for the CLI, or as a + unit test in the crate, rather than as a fuzz target. +- For the 32-bit class, `crates/exav-unpack/tests/suites/extreme.rs` sets + header fields to extreme values; `scripts/test-wasm.sh` runs the unit tests + on `wasm32-wasip1` (it needs `wasmtime`, which the image does not have), so + reason about 32-bit wraps from the code. + +## How you rate severity + +- **Critical**: code execution or memory corruption in the exav process from + scanned bytes (this needs `unsafe`, which is confined to + `crates/exav/src/daemon.rs`, `crates/exav/src/main.rs` and third-party + crates); a file-system write outside what the operator configured (the + extractor decodes in memory and spills only to unnamed temp files, so a + write to a path taken from an archive is a finding); a remote client of the + daemon or ICAP port running code or reading files it should not. +- **High**: a file reported `OK` (clean) that was not fully scanned, or whose + malicious content a signature should have seen: a member skipped, a decoder + that returns a prefix and no error, a limit that is hit and not reported. This + is the worst outcome for a scanner and ranks above a crash. Also a + stack overflow, an allocation that aborts the process, or a loop that does + not end within the configured limits, from one small file in the one-shot + CLI; in the prefork daemon these cost one worker, so rate them medium. +- **Medium**: memory or CPU far beyond the documented budgets (decompressed + bytes, per-member size, compression ratio, file count, recursion depth, + emulation steps) from a file much smaller than the work it causes + (amplification); an integer wrap on the 32-bit wasm build that defeats a + bounds check; a panic that the container boundary does not contain. +- **Low**: a panic that `catch_unwind` contains and reports for that one + container (`decoder panicked`), with no wrong verdict; a wrong or missing + name or size in a listing. + +## Anything to leave alone + +- **A signature database is a trusted input.** A `.cbc`/`.ldb` author running + a long bytecode loop, or a YARA rule that matches slowly, is not a finding; + the instruction cap is a failsafe against exav's own bugs. A parser bug + reachable with a crafted database (a panic or abort on load) is in scope + at low severity only. +- The `testing-faults` feature lets a scanned file ask a decoder to panic or + abort so tests can check the reporting; it is never enabled in a shipped + build. +- A panic that the walk reports as `decoder panicked` with the container then + marked `PARTIAL` is the designed behaviour, not a bug. +- Findings in third-party crates (hayro, zune, image, ruzstd, xz4rust and the + like) are welcome only when exav's use of them is what makes them reachable + or unbounded; report the dependency's own bugs upstream. +- Detection quality (a malware family not detected, a false positive) is not a + security finding. +- `exav-viewer` rendering differences from the original application are not + findings; script execution in the sandboxed frames is, but only when it + crosses the frame boundary. +- Prefer a minimal fix with a test that fails without it: every fixed finding + in this project has one (see `docs/FUZZING.md`). Deduplicate by root cause + in one function, not by file format. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5acd3b79..dcfda3d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,923 @@ Notable changes per release. Dates are release dates; the format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) loosely, and versions follow [semantic versioning](https://semver.org/). +## [Unreleased] + +## [0.0.2] - 2026-10-11 + +### Changed + +- A file larger than `--max-object-bytes` (256 MiB by default) gets the full + scan: wildcard and logical signatures, PCRE, YARA, bytecode, normalised text, + carving, embedded payloads and heuristics, read through an 8 MiB block cache + instead of held in memory. It got a literal-and-hash pass only. So does its + archive's own bytes, and an archive member that large once it is spilled to + disk; with `--spill-dir off` such a member is not scanned, and the file is + `LIMITS-EXCEEDED`. What still needs the object in memory (PE structure: EP and section + offsets, section hashes, imphash, icons, Authenticode, UPX and packer + unpacking; YARA's `pe`, `elf` and `dotnet` modules; containers read whole, + such as RAR, 7z or OLE) makes the file `LIMITS-EXCEEDED` naming the size + limit, unless something is found. +- `--max-object-bytes` no longer decides which signatures run, only what may + be held in memory at once. +- A DMG is walked without decompressing its whole disk, so an image whose disk + is larger than `--max-object-bytes` gives up its files; it was refused as + `LIMITS-EXCEEDED`. Each file is still held whole, under the same limit. A + file the image holds but that cannot be read out of it makes the scan + `UNSCANNABLE` unless something is found; it was skipped. LZ4 and Unix + `compress` (`.Z`) are decoded as they are read, as gzip is, instead of + whole. +- `--all-matches` and `ALLMATCHSCAN` list every detection at any size; past + `--max-object-bytes` they fell back to a single match. Past + `--max-input-bytes` they list what the first bytes hold and report the rest + unscanned, as a single-match scan does; `--all-matches` fell back to a single + match there, without saying the search stopped short, and `ALLMATCHSCAN` + ignored the limit. +- Every environment variable without a flag is named `EXAV_DEBUG_*`: + `EXAV_BC_WARN`, `EXAV_BC_TRACE`, `EXAV_BC_FN` and `EXAV_FORCED` are + `EXAV_DEBUG_BC_WARN`, `EXAV_DEBUG_BC_TRACE`, `EXAV_DEBUG_BC_FN` and + `EXAV_DEBUG_BC_FORCED`; `EXAV_YR_BIN`, `EXAV_YARA_CORPUS` and + `EXAV_YARA_BENCH_RULES` (tests and examples) are `EXAV_DEBUG_YR_BIN`, + `EXAV_DEBUG_YARA_CORPUS` and `EXAV_DEBUG_YARA_BENCH_RULES`. + `EXAV_SPLIT_MATCH`, `EXAV_VERIFY_BUDGET`, `EXAV_SIM_BUDGET` and + `EXAV_YARA_NO_GATE` are gone, along with the backtracking signature verifier + `EXAV_SPLIT_MATCH=0` selected. +- The `exav-unpack` command has `unzip`'s command line, as a subset: every + option it takes means what it means to Info-ZIP `unzip` 6.00 (`-l`, `-t`, + `-p`, `-c`, `-Z1`, `-d`, `-x`, `-o`, `-n`, `-P`, `-j`, `-C`, `-q`, `-D`), an + `unzip` option it does not take is an error, and its exit statuses are + `unzip`'s. `exav-unpack ARCHIVE` extracts; the `list` and `extract` + subcommands are gone (`-l`; `-d DIR`). It extracts every format the library + reads, writes each member as it is decoded rather than holding the archive + and its members in memory, restores ZIP and tar times, permission bits and + symbolic links, never writes through a link, and asks before replacing a + file. A ZIP split by `zip -s` (`.z01`, ..., `.zip`) and a RAR volume set + (`.part1.rar`, ... or `.rar`, `.r00`, ...) are read from any part, and + `--volume` names parts that are elsewhere. +- Releases carry `exav-unpack`, `exav-grep`, `exav-pe-emu` and `exav-imagehash` + archives beside `exav`'s, and every archive carries `LICENSE` and `NOTICE`. +- `--partial-as password-protected=found` reports any encrypted member as + `Heuristics.Encrypted.*`, as ClamAV's `--alert-encrypted` does: one exav + decrypted with a password, and one whose encryption flag is set over plain + content (APK packers set it on every member), included. A detection in the + content still wins. Before, a scan without `--all-matches` reported neither, + and a falsely flagged member was not even `.cdb`-matchable as encrypted. + `PASSWORD-PROTECTED` is still only content exav could not decrypt. For a ZIP + member the alert reads the local header as ClamAV does (bit 0 set, bit 13 + clear), and a `.cdb` encryption field matches either the local header or the + central directory record; both read only the central directory before. +- A YARA pattern keeps at most a million matches, as yara-x does; one with more + makes the scan `LIMITS-EXCEEDED` unless a rule matches. +- exav-core: `ScanOptions::spill` gives a scan somewhere to write what it makes + of an object too large to hold (see the `spill` module); `exav` passes its + `--spill-dir` files and budgets. `analyze_all_seekable` is the all-match scan + over a seekable input. `unpack::source` exposes random access over an object + not held in memory. +- Every object goes through one pipeline, the input and whatever is unpacked, + decoded or carved out of it, at any size and depth: + - An archive's members are scanned before its own bytes at every depth, so a + detection in a member stored in a nested archive names the member; the + nested archive's own bytes were matched first. + - The buffers a bytecode unpacker makes are scanned as objects, carving and + heuristics included, under `--max-unpack-depth` and `--max-matcher-bytes`; + they had a separate path capped at 4 levels and 256 MiB. + - The `.fp`/`.sfp` allowlist applies to every object, decoded payloads and + carved images included. + - YARA's filename externals are set for a top-level archive and under + `--all-matches`, as for any other top-level file. + - The signature engine is the only matcher: the separate literal automaton + for large inputs is gone. The `.exavdb` format is at version 4; rebuild + with `--build-db`. +- exav-unpack: `walk(fmt, source, budget, visit)` is the one walk. A member + comes as `Member::Bytes` or, decoded as it is read, `Member::Stream`; + `Member::into_bytes` reads either whole, and `MemberMeta::size` is the size + the archive declares. `walk`, `detect`, `extract` and the heuristics take a + `ByteSource` (a slice, a `Vec`, or a `source::BlockCache` over a reader). + Removed: `extract_each`, `Sink`, `stream_members`, `StreamVisit`, + `is_streamable`, `BudgetReader`, `Archive`, `MemberInfo`, the `*_source` + variants and `source::Slice`. exav-core: `patterns::PatternSet` and + `LoadError::Patterns` are removed. +- A streamed member is held to the compression-ratio guard, as a member + decoded whole is. +- `ScanOptions::deep_analysis_max` also holds `Limits::max_buffer_bytes` for a + scan, so no object is held in memory past it. +- exav-unpack-wasm: `list()` decodes nothing it can avoid, and reports + `uncompressedSize` as the archive declares it, -1 where it declares none + (it was the decoded size there); + `extract(i)` walks the archive up to member `i`. +- YARA detections are named `YARA.` outside `--clamav-compat`; the + `.UNOFFICIAL` suffix is compat-only, as for every other signature. +- A signature reload lets scans in progress finish: old workers and the old ICAP + child exit once idle instead of being killed. One still busy after + `--max-scan-secs` plus 5 seconds (an open session, a client that never + finishes) is stopped. +- A worker stopped mid-job (shutdown, reload deadline, supervisor gone) answers + the job with `ERROR` instead of closing the connection without a reply. +- `off` and `0` on `--max-object-bytes` and `--max-matcher-bytes` mean no limit, + as their help said; they used to set a limit of zero. +- Every input goes through one scan: a file, stdin, a FIFO given as a path, + `INSTREAM`, `EXINSTREAM`, ICAP and `http(s)://`. In particular: + - Past `--max-input-bytes`, the first bytes get the whole scan. A file got + only literal signatures and hashes there, and stdin and the daemon's stream + verbs no scan at all. + - A stream larger than the spill budgets is scanned as far as it was held and + reported `LIMITS-EXCEEDED`. It was `UNSCANNABLE`, unscanned. + - Both follow `--partial-as` as a file does; under `found` they are + `Heuristics.Limits.Exceeded.MaxFileSize` everywhere. + - A container over `--max-object-bytes` gets the full scan over its own + bytes on every entry point, not only a file path. + - Stdin honours `--all-matches`, `--profile` and `--max-process-bytes`, and + counts in the summary. +- exav-core: `scan_stream` is removed (the one scan is `scan_seekable`, and + `scan_path` hands a file to it); `warm_up` builds the lazily-initialised + structures before a fork. +- A stream that ends before its terminator is answered `ERROR` on `INSTREAM` + and `EXINSTREAM`, whatever `--partial-as` says. `EXINSTREAM` used to answer + `PARTIAL` with a `TRUNCATED` category, and `INSTREAM` `UNSCANNABLE`. +- A compressed stream damaged part way (gzip, ZIP, PDF, embedded PowerPoint + storages) is `UNSCANNABLE` when the part before the damage holds no + detection, wherever it sits; a checksum mismatch after a full decode is not. + Nested in another container it used to be `OK`, and a gzip with only a bad + CRC was `UNSCANNABLE` at the top level. A deflated ZIP member declaring 0 + bytes compressed and uncompressed (Civil 3D writes them) is empty, not + damaged. +- The daemon's signals match clamd's: `SIGUSR2` reloads the signatures and + `SIGHUP` reopens the `--log` file (for logrotate), under the worker pool, the + thread model and ICAP alone. `SIGUSR2` used to kill the process in every + mode, as `SIGHUP` did under the thread model and ICAP alone; under the pool + `SIGHUP` reloaded the signatures. +- `--workers threads` and an ICAP-only listener reload on `SIGUSR2`, `RELOAD` and + a change in the signature directory, as the pool does. +- The systemd unit is `exav.service` (it was `exav-clamd.service`). It reads + `/var/lib/clamav`, reloads with `systemctl reload exav` (`SIGUSR2`), and says + how to stop `clamav-daemon.socket` alongside `clamav-daemon`. +- `--max-extracted-bytes` is removed. What a scan holds is 1 GiB (400M under + `--clamav-compat`), and `--max-process-bytes` lowers it to what fits the + memory a scan gets. The flag, or `EXAV_MAX_EXTRACTED_BYTES`, stops the run with a + message naming the change. It also set the largest object held in memory, so + `--max-extracted-bytes 0` loaded any file whole. +- `--max-process-bytes` also lowers `--max-object-bytes` to a quarter of the + memory a scan gets, and the total held to what leaves room for an image + decoded from the largest object and its grey copy plus a tenth, so a file + past them is `LIMITS-EXCEEDED`; it was `out of memory ERROR`. The default 2G + changes neither the default limits nor `--clamav-compat`'s. +- Matching an object no longer copies it whole. PCRE subsignatures ran over a + Latin-1 string of the object, up to twice its size, and case-insensitive + bodies over a lowercased copy: the first now match the bytes directly, the + second lowercase what they read past 16 MiB. A crafted image no longer takes + more memory than an object may: its decoded pixels are held to + `--max-object-bytes` (past it, and short of the 512 MiB ClamAV decodes, the + scan is `LIMITS-EXCEEDED`), and it is turned grey + without a full-size RGB copy. A decoder panic leaves the image unhashed + rather than stopping its scan. +- `0` and `off`, flag by flag: `--max-members 0`/`off` and + `--max-jobs-per-worker 0`/`off` mean no limit; `--max-unpack-depth` and the + `--dlp-` counts refuse `0` (and depth refuses `off`); `--update-interval-secs + off` fetches at startup only; `--spill-threshold-bytes`, `--icap-idle-secs` + and `--icap-max-header-bytes` (at least `1K`) have no `off`; + `--icap-preview-bytes off` and `--icap-options-ttl-secs off` leave their + header out of the `OPTIONS` answer. +- `--quiet` keeps error lines, and error lines are written to `--log`; the + summary counts `Partial files` apart from `Total errors`. +- A scan that fails behind the ICAP listener is a block, with + `X-Exav-Status: ERROR` and `Heuristics.Exav.ScanError`. +- A bytecode program that runs out of instructions makes the scan + `LIMITS-EXCEEDED` unless something else is found. So does one stopped by an + opcode or host API exav does not model, or by the interpreter's call-depth or + frame-size cap; its result was discarded as though it had found nothing. A + program's own fault (an out-of-bounds access, a division by zero) is still + discarded, as ClamAV discards it. +- Bytecode follows ClamAV 1.4.3 in four more places: + - A program that aborts has its detection discarded; it was reported. + - `malloc` returns NULL from 128 MiB less 8 bytes; it allocated up to + 256 MiB. + - `__clambc_pedata` reads as zeros on a file that is not a PE; reading it + discarded the program's result. On a PE that exav has no header data for + (too large to load, or not parsed), it makes the scan `LIMITS-EXCEEDED` + unless something else is found. + - What a program wrote before its run failed is scanned; it was dropped + with the program's detection. A program that asks `disasm_x86` about bytes + the decoder does not know now stops there, so nothing it writes after that + point is scanned. +- A bytecode program that writes more than 256 MiB into one extracted file + makes the scan `LIMITS-EXCEEDED` unless something else is found; the rest + of its output was dropped silently. +- A bytecode `write` fails with -1 once one extracted file would pass + `--max-object-bytes`, or the run's output would pass what is left of + `--max-matcher-bytes`, as ClamAV's does at its file and scan size limits. + The program carries on and the scan is `LIMITS-EXCEEDED` unless something + else is found; the write used to succeed. +- The `docker-compose.yml` example publishes the clamd port on localhost only. +- `make lint` runs the clippy passes CI runs, and `make test-www` checks that + every internal link and anchor of the documentation site resolves. +- exav.org is organised by product: malware scanning, archive extraction + (with new pages for the `exav-unpack` command and `@exav/unpack-wasm`), the + file viewer, subprojects, and about the project, each listed in the header + of every page and with its own sidebar. The old page URLs redirect to the + new ones. The site is dark only. +- A signature path that loads nothing is an error: one that does not exist, or a + directory holding only a prebuilt `.exavdb` (load that with `-d`). +- `clamscan`'s `--alert-encrypted`, `--alert-encrypted-archive`, + `--alert-encrypted-doc` and `--alert-exceeds-max` are refused with the + `--partial-as` value that replaces them. +- The `exav` crate's `checksums` feature is removed; no flag used it. +- exav-unpack: the bzip2 decoder is vendored from `bzip2-rs` (with the fix + below) instead of depending on it; the internal `bzip2-decoder` feature + groups it. ALZ and EGG decode their bzip2 and LZMA members in a build without + the `bzip2` and `lzip` features. +- A build without the `yara` feature warns when rule files are loaded; they + never match. +- A long run of one byte value costs the signature engine about what its first + few hundred bytes do. The search skips the run's interior, and a signature + anchored inside it is checked once for the whole interior and one position at + a time only near its edges. Where several matches lie inside one run, a + first-match scan may report a different one of them. +- Scanning is faster on ordinary files too: + - The signature engine finds its signatures through one index of their + literal anchors, looked up at each position of the object, instead of + running automatons (four for a PE) over it. An anchor longer than five + bytes is indexed by the seven of its bytes the signatures share least, + looked up at every other position. + Case-insensitive anchors are looked up on the bytes + lowercased as they are read; the object is copied lowercased only when a + case-insensitive signature has to be checked. + - A signature whose offset window is at most 4096 bytes wide, counted from + the start, the end, the entry point or a section, is checked where it can + start, not searched for through the object. + - A PE section's MD5 is computed only when a section-hash signature names + its size. + - The bytecode interpreter lays out a function's values once per run rather + than at every call, and reads and writes them whole. + - Bytecode triggers are matched in the same sweep as every other + signature, not in a second sweep of their own. + - A literal a signature searches for after a gap is looked up in the + blocks of the object its check reaches, each searched once per scan, not + through the whole object the first time it is met. + - Whole-file digests are computed once for the allowlist and the hash + signatures together, and only those some signature of the object's size + can match: most objects need no SHA-1 or SHA-256 at all, and SHA-256 uses + the processor's instructions where it has them. + - The group cost check is skipped until a group passes its cap. +- An object read through the block cache, such as a large file or one fetched + over HTTP, is read far fewer times over: format detection's search, carving + and the digests share one read of it, the signature sweep is a second, + carving confirms an archive by its first bytes, and YARA's prefilter looks + up its atoms in one read and its literal and base64 patterns share another. Over HTTP, a request that + continues the previous one fetches twice as much, up to 8 MiB. +- A prebuilt database loads faster: the signature index, anchor groups, hash + tables and the indexes derived from the signatures are stored as the build + made them and read back with a copy, the file is read once rather than once + to check its digest and again to decode it, and the bodies and logical + signatures decode on threads of their own while the index is read. The file + is smaller, and building it takes less memory. Its trailer is a CRC-32 of the + contents instead of a SHA-256: it is there to catch a torn or damaged file, + which is all the digest was checked for. +- `--build-shard-bytes` is removed: the signature index has no automaton to + split, and builds in far less memory. The flag, or `EXAV_BUILD_SHARD_BYTES`, + stops the run with a message naming the change. exav-core: + `loader::load_with_options_mem` and `Builder::set_max_build_mem` are removed. +- Bytecode triggers are evaluated with the object's container, as every other + logical signature is, so a trigger with a `Container:` condition can fire. + exav-core (`unstable-internals`): `BytecodeRuntime::from_sources` adds the + triggers to the `EngineBuilder` it is given, and `BytecodeRuntime::standalone` + builds a runtime with an engine of its own, for tools. +- A first-match scan meets first the signatures their offsets pin, then the + others in the order the object's bytes hit them, rather than one file-type + automaton after another, so when several match it may report a different + one of them. Carving keeps the first + candidates of each kind by position when more than its cap are found. +- The RustCrypto crates move to their current generation (`digest` 0.11, + `cipher` 0.5: `sha2`, `sha1`, `md-5`, `aes`, `cbc`, `des`, `hmac`, `pbkdf2`). +- Dependencies: `bincode` (unmaintained, RUSTSEC-2025-0141) and `serde` are + gone from `exav-unpack`, the encrypted-DMG header being read field by field; + `tlsh2` 1.x; `ext4-view` 1.0 (panics and infinite loops on corrupt + filesystems fixed upstream); `lzma-rust2` 0.21 (malformed-input fixes + upstream); `ureq` 3 for the `http` features, which drops `url` and the ICU + crates from that build and, as ureq 3 does by default, follows the + `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` environment. + +### Added + +- AutoCAD DXF drawings, ASCII and binary, R12 to 2018 (feature `dxf`, in + `all-formats`): the object an OLE2FRAME embeds, which the file holds as + hexadecimal, is extracted and scanned (the compound file, then its streams), + as is any other binary chunk that is a whole file of a known kind. An ASCII + DXF keeps its text type; nothing changes under `--clamav-compat`. + `exav_unpack::dxf` exposes the pairs, records and code pages of the format. +- AutoCAD DWG drawings, R13 to 2018 (feature `dwg`, in + `all-formats`): the preview images are extracted and scanned, the bitmap + as a BMP file, and so is the object each OLE2FRAME embeds + (`ole2frame-.ole`, the compound file); a drawing whose objects + cannot be read is reported as not fully examined, and so is a drawing of + a release before R13 (R12, AC1009, and older). Nothing changes under + `--clamav-compat`. `exav_unpack::dwg` + reads the format's bit codes, the sections the file header locates (from + 2004: the encrypted file header, the page and section maps, checksummed + and compressed pages, with what they expand to bounded; 2007's own + Reed-Solomon coded pages, compression and CRCs, a damaged header or map + read from the copies the file keeps), the classes, the + object map and each object's common data and strings, from the ODA + specification. The DXF reader now reads MTEXT columns from the extended data (before 2018) + and the embedded object (2018). +- `exav-imagehash`, a crate and a command of its own: perceptual image hashes + with every step a parameter, and presets equal to `sigtool --fuzzy-img` + (ClamAV's `fuzzy_img` hash, which the scanner now computes through it) and + to Python `imagehash.phash`. No `unsafe` in it, its image decoders or its + DCT (rustdct's, vendored); its checksum and cast dependencies do use some. +- `exav-render`, a new crate of memory-safe decoders that turn a file into + something to draw: raster images into pixels (exav-imagehash now hashes + what it decodes; over 9,751 photos the hashes are unchanged), JPEG 2000, + JBIG2 and fax images (`jp2`, `jbig2`; hayro-jpeg2000 and hayro-jbig2, + `forbid(unsafe_code)`, without their SIMD code) including those of PDFs + (`pdf_image`), DWG (R13 to 2018) and DXF drawings into GPU-ready buffers + (feature `dwg`), and IFC (2X3, 4 and 4X3) and STL models into meshes + (features `ifc`, `stl`). +- `@exav/viewer`, a new npm package: a file viewer for the browser, for PDF, + images (TIFF, BMP and the formats browsers do not draw, through exav-render + in WebAssembly), DWG and DXF, Word, Excel, PowerPoint and CSV, IFC and STL + models, video, audio, and the files inside an archive. Each format's engine + loads on its first file; every asset is served by the host, so it runs under + a strict Content-Security-Policy and offline. A framework-free core, a React + UI, a Vite plugin, a command that copies the engines' assets, and a + sandboxed mode that opens each file in an `", + '', + '', + "", + "", + "", + 'text', + "", + ]) { + expect(allowedHtml(html), html).toBe(false); + } + }); +}); + +describe("a worker's blob", () => { + it("is a classic script that sets the script's URL and imports only it", () => { + const calls: string[] = []; + const self: Record = { importScripts: (u: string) => calls.push(u) }; + vm.runInNewContext(workerBoot("https://frame.test/cad.worker.js"), { self, importScripts: self.importScripts }); + expect(self.EXAV_SCRIPT).toBe("https://frame.test/cad.worker.js"); + expect(calls).toEqual(["https://frame.test/cad.worker.js"]); + }); + + it("quotes the URL as data, whatever it holds", () => { + const url = 'https://frame.test/a.js"); importScripts("https://x.test/evil.js'; // eslint-disable-line + const calls: string[] = []; + const self: Record = {}; + vm.runInNewContext(workerBoot(url), { self, importScripts: (u: string) => calls.push(u) }); + expect(calls).toEqual([url]); + }); + + it("goes through a policy that accepts that URL alone, where Trusted Types exist", () => { + const made: { name: string; accepts: (u: string) => unknown }[] = []; + const trustedTypes = { + createPolicy: (name: string, rules: { createScriptURL: (u: string) => string }) => { + made.push({ name, accepts: (u) => rules.createScriptURL(u) }); + return { createScriptURL: (u: string) => ({ trusted: rules.createScriptURL(u) }) }; + }, + }; + const calls: unknown[] = []; + vm.runInNewContext(workerBoot("https://frame.test/w.js"), { self: { trustedTypes }, importScripts: (u: unknown) => calls.push(u) }); + expect(made.map((p) => p.name)).toEqual(["exav-worker"]); + expect(calls).toEqual([{ trusted: "https://frame.test/w.js" }]); + // A TypeError of the script's own realm. + expect(() => made[0]!.accepts("https://frame.test/other.js")).toThrow("not this worker's script"); + }); +}); diff --git a/crates/exav-viewer/ts/frame/app/workers.ts b/crates/exav-viewer/ts/frame/app/workers.ts new file mode 100644 index 00000000..795dbe54 --- /dev/null +++ b/crates/exav-viewer/ts/frame/app/workers.ts @@ -0,0 +1,135 @@ +/** + * Workers and Trusted Types in the frame. + * + * The frame's origin is opaque: `new Worker(url)` on its own scripts throws, + * and a module worker from a `blob:` URL does not start. A classic worker + * from a `blob:` URL does. So every worker an engine asks for (ours, pdf.js's, + * the Office engine's) is started from a blob made here, + * which holds one `importScripts` of the script the engine named. That script + * must be one of the frame's own: the frame's build puts a classic bundle + * where each module worker was. + * + * Under `require-trusted-types-for 'script'` a worker's URL must come from a + * policy: `exav-worker` accepts only the blobs made here. The `default` + * policy takes what the engines write into `innerHTML` (the Office engine + * clears elements and draws an arrow) when it is + * inert SVG, and refuses the rest. + */ + +type Policy = { createScriptURL(url: string): unknown }; +type Factory = { createPolicy(name: string, rules: Record string | null>): Policy }; + +const tt = (globalThis as { trustedTypes?: Factory }).trustedTypes; + +/** Shapes only: no ``, ``, ``, ``, `