feat(hwdec): VideoToolbox backend — HEVC + AV1 still decode on macOS/iOS

.github/workflows/ci.yml

@@ -153,7 +153,7 @@ jobs:
       - uses: dtolnay/rust-toolchain@master
         with:
           toolchain: "1.92.0"
-          targets: aarch64-apple-darwin,x86_64-unknown-linux-musl
+          targets: aarch64-apple-darwin,aarch64-apple-ios,x86_64-unknown-linux-musl
       - uses: Swatinem/rust-cache@v2
       - name: Invalid combos must fail with the compile_error text
         run: |
@@ -186,6 +186,10 @@ jobs:
           # backend where one exists and compiles the no-backend stub (with a
           # build-script warning) where none does (musl).
           cargo check -p rawshift-image --no-default-features --features hw --target aarch64-apple-darwin
+          # iOS is a tier-1 build target (docs/SUPPORT.md); the VideoToolbox
+          # backend cfg-gates the macOS-only decoder-specification keys, so
+          # this is what proves that gating compiles.
+          cargo check -p rawshift-image --no-default-features --features hw --target aarch64-apple-ios
           cargo check -p rawshift-image --no-default-features --features hw --target x86_64-unknown-linux-musl
 
   # ── `full` on tier-1 targets (issue #34) ────────────────────────────────────

CHANGELOG.md

@@ -164,6 +164,20 @@ All entries below are **breaking**, grouped by area.
   `HwStillDecoder`, `decoder()`/`backend()`/`available_codecs()`), verified
   feature flags (`hw` portable; `videotoolbox`/`vaapi`/`mediacodec`
   `compile_error!` on foreign targets), and all platform FFI confined to it.
+- *(hwdec)* VideoToolbox hardware decode backend (macOS/iOS): HEVC
+  Main/Main10 and AV1 Profile 0 still pictures to NV12/P010 through
+  `VTDecompressionSession`, over the linked system frameworks. HEVC takes its
+  `hvcC` parameter sets with the length-prefixed payload passed through
+  unchanged; AV1 takes its `av1C` config atom. Availability is probed per
+  codec with `VTIsHardwareDecodeSupported`, so `available_codecs()` omits
+  `Av1` on hardware without an AV1 decode block (before M3 / A17 Pro) while
+  HEVC keeps working. Sessions are hardware-pinned on macOS and retried once
+  without that requirement, because Apple's HEVC block refuses pictures below
+  roughly 64x64 and HEIF thumbnails are routinely 32x32. HEIC and AVIF pixel
+  decode in `rawshift-image` work end-to-end on Apple hardware through this
+  backend. Bindings are the `objc2-*` framework crates
+  (`default-features = false`); the rationale for generated bindings here and
+  a hand-written `sys.rs` for VAAPI is recorded in the crate README.
 - *(hwdec)* VAAPI hardware decode backend (linux-gnu): HEVC Main/Main10 and
   AV1 Profile 0 still pictures to NV12/P010, libva dlopen'd at runtime —
   absence of libva or a render node degrades to `decoder() == None`, never a

DEVELOPMENT.md

@@ -69,6 +69,41 @@ the new version.
 > `master` before the PR is cut. To run CI on Release PRs, switch the workflow
 > to a GitHub App token / PAT via the action's `token:` input.
 
+### Pre-release hardware verification
+
+**CI does not test hardware decode, and cannot.** Hosted runners have no
+dependable hardware decode block, and every hardware test in this repo skips
+gracefully when no decoder is present — so a CI job would report green without
+having decoded anything. CI covers the compile boundaries only (see the
+`compile-boundaries` job for the feature × target matrix).
+
+That makes hardware decode the one part of `rawshift` whose regressions are
+invisible on `master`. The backstop is a manual gate:
+
+> **Before merging a Release PR, a maintainer must run `just test-hw` on at
+> least one machine per hardware backend that changed since the last release,
+> and record the result in the Release PR.** A change to `rawshift-hwdec` that
+> has not been run against real hardware is not releasable.
+
+```sh
+just test-hw
+```
+
+Read the output rather than trusting the exit code: tests that skip say so on
+stderr, and a run where everything skipped verifies nothing. A good run names
+the backend and the codecs it exercised, e.g. `VideoToolbox probe: available
+codecs: [Hevc, Av1]`.
+
+| Backend | Platform | What a run needs |
+| --- | --- | --- |
+| VideoToolbox | macOS (Apple silicon; AV1 needs M3 / A17 Pro or later) | `brew install ffmpeg libheif libavif` |
+| VAAPI | Linux (gnu) with an Intel/AMD GPU, or NVIDIA via `nvidia-vaapi-driver` | `ffmpeg`, a `/dev/dri/renderD*` node |
+| MediaCodec | Android device or emulator, API 29+ | see the backend's own harness |
+
+Codecs the machine cannot decode skip rather than fail — an Apple machine
+older than M3 has no AV1 block, and that is expected, not a regression. What
+must not happen is a *listed* codec failing to decode.
+
 ## One-time setup (bootstrap)
 
 crates.io trusted publishing **cannot create a brand-new crate** — a Trusted

TEST_FIXTURES.md

@@ -189,7 +189,13 @@ cargo test --features=arw --test ifd_decoder_tests
 cargo test --features=dng --test dng_check
 cargo test --features=heic --test heic_aux
 
-# Hardware decode (compiled with `hw`; skips gracefully without a GPU)
+# Hardware decode (compiled with `hw`; skips gracefully without a decoder).
+# Prefer the `just` target: it also runs the backend's own device tests and
+# prints what was actually exercised, which matters because these tests skip
+# silently when no hardware decoder is present.
+just test-hw
+
+# ...or the end-to-end half on its own:
 cargo test --features=full --test heic_hw_decode --test avif_hw_decode
 
 # With specific features

crates/rawshift-hwdec/Cargo.toml

@@ -30,6 +30,56 @@ thiserror   = { workspace = true }
 [target.'cfg(target_os = "linux")'.dependencies]
 libloading = "0.8"
 
+# VideoToolbox/CoreMedia/CoreVideo are always-present Apple system frameworks,
+# so unlike libva they are linked normally rather than dlopen'd. The `objc2-*`
+# framework crates are the maintained, generated bindings for them (MSRV 1.71,
+# Zlib OR Apache-2.0 OR MIT) and carry the `#[link(kind = "framework")]`
+# attributes, so build.rs needs no link directives. They also give us
+# `CFRetained<T>` RAII instead of hand-rolled CFRetain/CFRelease pairs — the
+# reason this backend uses generated bindings where the VAAPI one hand-writes
+# `sys.rs` (see the module docs in src/videotoolbox/mod.rs).
+#
+# `default-features = false` throughout: the default feature sets pull in the
+# Objective-C runtime, Metal, OpenGL and CoreAudio, none of which a still-frame
+# decode touches. Only the granular per-header features below are enabled.
+[target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies]
+objc2-video-toolbox = { version = "0.3.2", default-features = false, features = [
+    "std",
+    "bitflags",
+    "VTBase",
+    "VTErrors",
+    "VTDecompressionSession",
+    "VTDecompressionProperties",
+    "objc2-core-media",
+    "objc2-core-video",
+] }
+objc2-core-media = { version = "0.3.2", default-features = false, features = [
+    "std",
+    "bitflags",
+    "CMBase",
+    "CMTime",
+    "CMBlockBuffer",
+    "CMFormatDescription",
+    "CMSampleBuffer",
+] }
+objc2-core-video = { version = "0.3.2", default-features = false, features = [
+    "std",
+    "bitflags",
+    "CVBase",
+    "CVReturn",
+    "CVBuffer",
+    "CVImageBuffer",
+    "CVPixelBuffer",
+] }
+objc2-core-foundation = { version = "0.3.2", default-features = false, features = [
+    "std",
+    "CFBase",
+    "CFString",
+    "CFData",
+    "CFNumber",
+    "CFDictionary",
+] }
+
 [features]
 # Verified backend feature flags — see docs/SUPPORT.md for the permanent
 # target/API matrix. Each explicit backend flag hard-fails the compile
@@ -39,11 +89,12 @@ libloading = "0.8"
 # (windows-msvc, linux-musl, wasm) emits a build-script warning and compiles
 # the no-backend stub.
 #
-# Backends implemented: VAAPI (linux-gnu; dlopen'd libva, HEVC Main/Main10 +
-# AV1 Profile 0 still pictures). VideoToolbox / MediaCodec land as separate
-# issues; builds without a selected backend compile the no-backend stub —
-# `decoder()` returns `None`, `backend()` returns `None`, and
-# `available_codecs()` is empty.
+# Backends implemented: VideoToolbox (macOS/iOS; linked system frameworks,
+# hardware-only via VTIsHardwareDecodeSupported) and VAAPI (linux-gnu;
+# dlopen'd libva), both for HEVC Main/Main10 + AV1 Profile 0 still pictures.
+# MediaCodec lands as a separate issue; builds without a selected backend
+# compile the no-backend stub — `decoder()` returns `None`, `backend()`
+# returns `None`, and `available_codecs()` is empty.
 videotoolbox = []
 vaapi        = []
 mediacodec   = []

crates/rawshift-hwdec/README.md

@@ -8,11 +8,59 @@ MediaCodec (Android).
 
 This is the **only** crate in the workspace where platform FFI may live
 (`#![deny(unsafe_op_in_unsafe_fn)]`, safe public items, documented invariants
-on every unsafe block). The **VAAPI backend is implemented**; VideoToolbox
-and MediaCodec land as separate issues. On builds/targets with no backend the
-crate compiles a no-backend stub: `decoder()` returns `None`, `backend()`
-returns `None`, `available_codecs()` is empty, and dependants surface
-`HwDecoderUnavailable`.
+on every unsafe block). The **VideoToolbox and VAAPI backends are
+implemented**; MediaCodec lands as a separate issue. On builds/targets with no
+backend the crate compiles a no-backend stub: `decoder()` returns `None`,
+`backend()` returns `None`, `available_codecs()` is empty, and dependants
+surface `HwDecoderUnavailable`.
+
+## VideoToolbox backend (macOS / iOS)
+
+VideoToolbox, CoreMedia and CoreVideo are always-present system frameworks, so
+there is nothing to dlopen and no device to find: availability is answered by
+`VTIsHardwareDecodeSupported` per codec, cached once per process.
+
+Still-picture scope:
+
+| Codec | Profiles | Output |
+| --- | --- | --- |
+| HEVC (HEIC) | Main, Main 10 — `hvcC` parameter sets, length-prefixed payload passed through unchanged (no Annex-B conversion) | NV12 (8-bit), P010 (10-bit) |
+| AV1 (AVIF) | Profile 0 (Main) — `av1C` config atom, raw OBU temporal unit; **runtime-probed**, since AV1 hardware decode arrived with M3 / A17 Pro | NV12 (8-bit), P010 (10-bit) |
+
+On Apple silicon older than M3 and on Intel, `available_codecs()` honestly
+omits `Av1` while HEVC keeps working.
+
+Two behaviours are worth knowing about:
+
+- **Hardware first, with a narrow fallback.** Sessions are created on macOS
+  with `RequireHardwareAcceleratedVideoDecoder`, so the decode does not
+  silently drop to VideoToolbox's software decoder. Apple's hardware HEVC
+  block, however, refuses pictures below roughly 64x64 — and HEIF thumbnails
+  are routinely 32x32 — so a failed hardware-pinned create is retried once
+  without the requirement. Availability is still gated on the hardware probe,
+  so this only widens the accepted picture *sizes*, never the codec list.
+- **An explicit destination pixel format is requested.** Left to itself, a
+  10-bit decode natively emits `'p420'`, which appears in no public CoreVideo
+  header and whose plane contents do not match the documented `x420` samples
+  for the same bitstream. All four documented 4:2:0 surfaces are offered
+  instead, so the decoder stays on interpretable layouts while still choosing
+  the one that matches the stream — which is what preserves the video/full
+  range distinction.
+
+### Why generated bindings here and a hand-written `sys.rs` for VAAPI
+
+This backend uses the [`objc2`](https://github.com/madsmtm/objc2) framework
+crates (`objc2-video-toolbox`, `-core-media`, `-core-video`,
+`-core-foundation`). The VAAPI backend hand-writes its FFI, and the difference
+is deliberate: libva must be **dlopen'd** so a machine without it degrades to
+"no decoder" rather than failing to start, which rules out ordinary generated
+bindings. The Apple frameworks are guaranteed present and link normally, so
+the maintained bindings win — `CFRetained<T>` gives Create/Get-rule reference
+counting as RAII instead of hand-paired `CFRetain`/`CFRelease`, and the
+`#[link(kind = "framework")]` attributes live upstream. They are taken with
+`default-features = false` and only the per-header features used, so no
+Objective-C runtime, Metal, OpenGL or CoreAudio is compiled in; the only
+transitive addition is `bitflags`.
 
 ## VAAPI backend (linux-gnu)
 
@@ -44,7 +92,16 @@ justification in [`docs/SUPPORT.md`](../../docs/SUPPORT.md).
 
 | Feature | Meaning |
 | --- | --- |
-| `hw` | Portable: select the native backend for the compile target (VAAPI on linux-gnu; build-script warning + stub on targets with no hardware decode API). |
+| `hw` | Portable: select the native backend for the compile target (VideoToolbox on macOS/iOS, VAAPI on linux-gnu; build-script warning + stub on targets with no hardware decode API). |
 | `videotoolbox` | Pin VideoToolbox; `compile_error!` on non-Apple targets. |
 | `vaapi` | Pin VAAPI; `compile_error!` off linux-gnu. |
 | `mediacodec` | Pin MediaCodec; `compile_error!` off Android. |
+
+## Testing
+
+The device tests (`tests/videotoolbox_device.rs`, `tests/vaapi_device.rs`) run
+real decodes and skip gracefully when the machine has no decoder for the
+codec, so CI stays green without ever exercising hardware. That means CI does
+**not** cover decode correctness: run `just test-hw` locally, which is the
+documented pre-release gate (see "Pre-release hardware verification" in
+`DEVELOPMENT.md`).

crates/rawshift-hwdec/src/vaapi/av1.rs → crates/rawshift-hwdec/src/bitstream/av1.rs

@@ -1,5 +1,5 @@
-//! AV1 still-picture parsing for the VAAPI backend — **safe Rust only**
-//! (no `unsafe`; FFI stays in `sys.rs`/`mod.rs`).
+//! AV1 still-picture parsing shared by hardware backends — **safe Rust only**
+//! (no `unsafe`; platform FFI stays in backend modules).
 //!
 //! ## Scope
 //!
@@ -16,7 +16,8 @@
 //! `VASliceParameterBufferAV1` per tile.
 
 use super::bits::{BitReader, PResult, ParseError, clip3};
-use super::sys;
+#[cfg(hwdec_backend = "vaapi")]
+use crate::vaapi::sys;
 
 // ── OBU framing (§5.3) ──────────────────────────────────────────────────────
 
@@ -1144,6 +1145,7 @@ pub fn parse_still_picture(config_obus: &[u8], payload: &[u8]) -> PResult<StillP
 
 /// Fill `VADecPictureParameterBufferAV1` for `pic` decoded into `surface`
 /// (and, when film grain is applied, displayed into `display_surface`).
+#[cfg(hwdec_backend = "vaapi")]
 pub fn build_pic_param(
     pic: &StillPicture,
     surface: sys::VASurfaceID,
@@ -1411,6 +1413,7 @@ pub fn build_pic_param(
 
 /// Fill `VASliceParameterBufferAV1` for one tile placed at offset 0 of its
 /// own data buffer.
+#[cfg(hwdec_backend = "vaapi")]
 pub fn build_tile_param(tile: &Tile) -> sys::VASliceParameterBufferAV1 {
     sys::VASliceParameterBufferAV1 {
         slice_data_size: tile.data.len() as u32,
@@ -1516,6 +1519,9 @@ mod tests {
         );
     }
 
+    // Exercises the VAAPI parameter-buffer builders, which only exist in a
+    // VAAPI build (the parsers above are shared by every backend).
+    #[cfg(hwdec_backend = "vaapi")]
     #[test]
     fn pic_param_maps_seq_and_frame_fields() {
         let pic = parse_still_picture(&[], AV1_64X64_LIBAOM).unwrap();

crates/rawshift-hwdec/src/vaapi/bits.rs → crates/rawshift-hwdec/src/bitstream/bits.rs

@@ -1,4 +1,4 @@
-//! Safe bitstream readers shared by the HEVC and AV1 header parsers.
+//! Safe bitstream readers shared by the HEVC and AV1 still-picture parsers.
 //!
 //! **Safe Rust only** — this module (like `hevc.rs` / `av1.rs`) contains no
 //! `unsafe`; all FFI stays in `sys.rs` and the call sites in `mod.rs`.