diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 348149c..51357f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,7 +153,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: "1.92.0" - targets: aarch64-apple-darwin,x86_64-unknown-linux-musl + targets: aarch64-apple-darwin,aarch64-apple-ios,x86_64-unknown-linux-musl - uses: Swatinem/rust-cache@v2 - name: Invalid combos must fail with the compile_error text run: | @@ -186,6 +186,10 @@ jobs: # backend where one exists and compiles the no-backend stub (with a # build-script warning) where none does (musl). cargo check -p rawshift-image --no-default-features --features hw --target aarch64-apple-darwin + # iOS is a tier-1 build target (docs/SUPPORT.md); the VideoToolbox + # backend cfg-gates the macOS-only decoder-specification keys, so + # this is what proves that gating compiles. + cargo check -p rawshift-image --no-default-features --features hw --target aarch64-apple-ios cargo check -p rawshift-image --no-default-features --features hw --target x86_64-unknown-linux-musl # ── `full` on tier-1 targets (issue #34) ──────────────────────────────────── diff --git a/CHANGELOG.md b/CHANGELOG.md index 8728a8b..48808fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,6 +164,20 @@ All entries below are **breaking**, grouped by area. `HwStillDecoder`, `decoder()`/`backend()`/`available_codecs()`), verified feature flags (`hw` portable; `videotoolbox`/`vaapi`/`mediacodec` `compile_error!` on foreign targets), and all platform FFI confined to it. +- *(hwdec)* VideoToolbox hardware decode backend (macOS/iOS): HEVC + Main/Main10 and AV1 Profile 0 still pictures to NV12/P010 through + `VTDecompressionSession`, over the linked system frameworks. HEVC takes its + `hvcC` parameter sets with the length-prefixed payload passed through + unchanged; AV1 takes its `av1C` config atom. Availability is probed per + codec with `VTIsHardwareDecodeSupported`, so `available_codecs()` omits + `Av1` on hardware without an AV1 decode block (before M3 / A17 Pro) while + HEVC keeps working. Sessions are hardware-pinned on macOS and retried once + without that requirement, because Apple's HEVC block refuses pictures below + roughly 64x64 and HEIF thumbnails are routinely 32x32. HEIC and AVIF pixel + decode in `rawshift-image` work end-to-end on Apple hardware through this + backend. Bindings are the `objc2-*` framework crates + (`default-features = false`); the rationale for generated bindings here and + a hand-written `sys.rs` for VAAPI is recorded in the crate README. - *(hwdec)* VAAPI hardware decode backend (linux-gnu): HEVC Main/Main10 and AV1 Profile 0 still pictures to NV12/P010, libva dlopen'd at runtime — absence of libva or a render node degrades to `decoder() == None`, never a diff --git a/Cargo.lock b/Cargo.lock index f33e310..88c66fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1030,6 +1030,47 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.11.0", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.11.0", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-video-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05bf9a3c14831a7d9641b0d81d87dd913ee238a012b2fde27db5a84b56f5df3e" +dependencies = [ + "bitflags 2.11.0", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-video", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -1202,6 +1243,10 @@ version = "0.1.1" dependencies = [ "gamut-color", "libloading", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-video", + "objc2-video-toolbox", "thiserror", ] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3a48f74..c237ba9 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -69,6 +69,41 @@ the new version. > `master` before the PR is cut. To run CI on Release PRs, switch the workflow > to a GitHub App token / PAT via the action's `token:` input. +### Pre-release hardware verification + +**CI does not test hardware decode, and cannot.** Hosted runners have no +dependable hardware decode block, and every hardware test in this repo skips +gracefully when no decoder is present — so a CI job would report green without +having decoded anything. CI covers the compile boundaries only (see the +`compile-boundaries` job for the feature × target matrix). + +That makes hardware decode the one part of `rawshift` whose regressions are +invisible on `master`. The backstop is a manual gate: + +> **Before merging a Release PR, a maintainer must run `just test-hw` on at +> least one machine per hardware backend that changed since the last release, +> and record the result in the Release PR.** A change to `rawshift-hwdec` that +> has not been run against real hardware is not releasable. + +```sh +just test-hw +``` + +Read the output rather than trusting the exit code: tests that skip say so on +stderr, and a run where everything skipped verifies nothing. A good run names +the backend and the codecs it exercised, e.g. `VideoToolbox probe: available +codecs: [Hevc, Av1]`. + +| Backend | Platform | What a run needs | +| --- | --- | --- | +| VideoToolbox | macOS (Apple silicon; AV1 needs M3 / A17 Pro or later) | `brew install ffmpeg libheif libavif` | +| VAAPI | Linux (gnu) with an Intel/AMD GPU, or NVIDIA via `nvidia-vaapi-driver` | `ffmpeg`, a `/dev/dri/renderD*` node | +| MediaCodec | Android device or emulator, API 29+ | see the backend's own harness | + +Codecs the machine cannot decode skip rather than fail — an Apple machine +older than M3 has no AV1 block, and that is expected, not a regression. What +must not happen is a *listed* codec failing to decode. + ## One-time setup (bootstrap) crates.io trusted publishing **cannot create a brand-new crate** — a Trusted diff --git a/TEST_FIXTURES.md b/TEST_FIXTURES.md index 1673997..c4d8af1 100644 --- a/TEST_FIXTURES.md +++ b/TEST_FIXTURES.md @@ -189,7 +189,13 @@ cargo test --features=arw --test ifd_decoder_tests cargo test --features=dng --test dng_check cargo test --features=heic --test heic_aux -# Hardware decode (compiled with `hw`; skips gracefully without a GPU) +# Hardware decode (compiled with `hw`; skips gracefully without a decoder). +# Prefer the `just` target: it also runs the backend's own device tests and +# prints what was actually exercised, which matters because these tests skip +# silently when no hardware decoder is present. +just test-hw + +# ...or the end-to-end half on its own: cargo test --features=full --test heic_hw_decode --test avif_hw_decode # With specific features diff --git a/crates/rawshift-hwdec/Cargo.toml b/crates/rawshift-hwdec/Cargo.toml index c24b58e..3a45631 100644 --- a/crates/rawshift-hwdec/Cargo.toml +++ b/crates/rawshift-hwdec/Cargo.toml @@ -30,6 +30,56 @@ thiserror = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] libloading = "0.8" +# VideoToolbox/CoreMedia/CoreVideo are always-present Apple system frameworks, +# so unlike libva they are linked normally rather than dlopen'd. The `objc2-*` +# framework crates are the maintained, generated bindings for them (MSRV 1.71, +# Zlib OR Apache-2.0 OR MIT) and carry the `#[link(kind = "framework")]` +# attributes, so build.rs needs no link directives. They also give us +# `CFRetained` RAII instead of hand-rolled CFRetain/CFRelease pairs — the +# reason this backend uses generated bindings where the VAAPI one hand-writes +# `sys.rs` (see the module docs in src/videotoolbox/mod.rs). +# +# `default-features = false` throughout: the default feature sets pull in the +# Objective-C runtime, Metal, OpenGL and CoreAudio, none of which a still-frame +# decode touches. Only the granular per-header features below are enabled. +[target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies] +objc2-video-toolbox = { version = "0.3.2", default-features = false, features = [ + "std", + "bitflags", + "VTBase", + "VTErrors", + "VTDecompressionSession", + "VTDecompressionProperties", + "objc2-core-media", + "objc2-core-video", +] } +objc2-core-media = { version = "0.3.2", default-features = false, features = [ + "std", + "bitflags", + "CMBase", + "CMTime", + "CMBlockBuffer", + "CMFormatDescription", + "CMSampleBuffer", +] } +objc2-core-video = { version = "0.3.2", default-features = false, features = [ + "std", + "bitflags", + "CVBase", + "CVReturn", + "CVBuffer", + "CVImageBuffer", + "CVPixelBuffer", +] } +objc2-core-foundation = { version = "0.3.2", default-features = false, features = [ + "std", + "CFBase", + "CFString", + "CFData", + "CFNumber", + "CFDictionary", +] } + [features] # Verified backend feature flags — see docs/SUPPORT.md for the permanent # target/API matrix. Each explicit backend flag hard-fails the compile @@ -39,11 +89,12 @@ libloading = "0.8" # (windows-msvc, linux-musl, wasm) emits a build-script warning and compiles # the no-backend stub. # -# Backends implemented: VAAPI (linux-gnu; dlopen'd libva, HEVC Main/Main10 + -# AV1 Profile 0 still pictures). VideoToolbox / MediaCodec land as separate -# issues; builds without a selected backend compile the no-backend stub — -# `decoder()` returns `None`, `backend()` returns `None`, and -# `available_codecs()` is empty. +# Backends implemented: VideoToolbox (macOS/iOS; linked system frameworks, +# hardware-only via VTIsHardwareDecodeSupported) and VAAPI (linux-gnu; +# dlopen'd libva), both for HEVC Main/Main10 + AV1 Profile 0 still pictures. +# MediaCodec lands as a separate issue; builds without a selected backend +# compile the no-backend stub — `decoder()` returns `None`, `backend()` +# returns `None`, and `available_codecs()` is empty. videotoolbox = [] vaapi = [] mediacodec = [] diff --git a/crates/rawshift-hwdec/README.md b/crates/rawshift-hwdec/README.md index 2098724..8008b8b 100644 --- a/crates/rawshift-hwdec/README.md +++ b/crates/rawshift-hwdec/README.md @@ -8,11 +8,59 @@ MediaCodec (Android). This is the **only** crate in the workspace where platform FFI may live (`#![deny(unsafe_op_in_unsafe_fn)]`, safe public items, documented invariants -on every unsafe block). The **VAAPI backend is implemented**; VideoToolbox -and MediaCodec land as separate issues. On builds/targets with no backend the -crate compiles a no-backend stub: `decoder()` returns `None`, `backend()` -returns `None`, `available_codecs()` is empty, and dependants surface -`HwDecoderUnavailable`. +on every unsafe block). The **VideoToolbox and VAAPI backends are +implemented**; MediaCodec lands as a separate issue. On builds/targets with no +backend the crate compiles a no-backend stub: `decoder()` returns `None`, +`backend()` returns `None`, `available_codecs()` is empty, and dependants +surface `HwDecoderUnavailable`. + +## VideoToolbox backend (macOS / iOS) + +VideoToolbox, CoreMedia and CoreVideo are always-present system frameworks, so +there is nothing to dlopen and no device to find: availability is answered by +`VTIsHardwareDecodeSupported` per codec, cached once per process. + +Still-picture scope: + +| Codec | Profiles | Output | +| --- | --- | --- | +| HEVC (HEIC) | Main, Main 10 — `hvcC` parameter sets, length-prefixed payload passed through unchanged (no Annex-B conversion) | NV12 (8-bit), P010 (10-bit) | +| AV1 (AVIF) | Profile 0 (Main) — `av1C` config atom, raw OBU temporal unit; **runtime-probed**, since AV1 hardware decode arrived with M3 / A17 Pro | NV12 (8-bit), P010 (10-bit) | + +On Apple silicon older than M3 and on Intel, `available_codecs()` honestly +omits `Av1` while HEVC keeps working. + +Two behaviours are worth knowing about: + +- **Hardware first, with a narrow fallback.** Sessions are created on macOS + with `RequireHardwareAcceleratedVideoDecoder`, so the decode does not + silently drop to VideoToolbox's software decoder. Apple's hardware HEVC + block, however, refuses pictures below roughly 64x64 — and HEIF thumbnails + are routinely 32x32 — so a failed hardware-pinned create is retried once + without the requirement. Availability is still gated on the hardware probe, + so this only widens the accepted picture *sizes*, never the codec list. +- **An explicit destination pixel format is requested.** Left to itself, a + 10-bit decode natively emits `'p420'`, which appears in no public CoreVideo + header and whose plane contents do not match the documented `x420` samples + for the same bitstream. All four documented 4:2:0 surfaces are offered + instead, so the decoder stays on interpretable layouts while still choosing + the one that matches the stream — which is what preserves the video/full + range distinction. + +### Why generated bindings here and a hand-written `sys.rs` for VAAPI + +This backend uses the [`objc2`](https://github.com/madsmtm/objc2) framework +crates (`objc2-video-toolbox`, `-core-media`, `-core-video`, +`-core-foundation`). The VAAPI backend hand-writes its FFI, and the difference +is deliberate: libva must be **dlopen'd** so a machine without it degrades to +"no decoder" rather than failing to start, which rules out ordinary generated +bindings. The Apple frameworks are guaranteed present and link normally, so +the maintained bindings win — `CFRetained` gives Create/Get-rule reference +counting as RAII instead of hand-paired `CFRetain`/`CFRelease`, and the +`#[link(kind = "framework")]` attributes live upstream. They are taken with +`default-features = false` and only the per-header features used, so no +Objective-C runtime, Metal, OpenGL or CoreAudio is compiled in; the only +transitive addition is `bitflags`. ## VAAPI backend (linux-gnu) @@ -44,7 +92,16 @@ justification in [`docs/SUPPORT.md`](../../docs/SUPPORT.md). | Feature | Meaning | | --- | --- | -| `hw` | Portable: select the native backend for the compile target (VAAPI on linux-gnu; build-script warning + stub on targets with no hardware decode API). | +| `hw` | Portable: select the native backend for the compile target (VideoToolbox on macOS/iOS, VAAPI on linux-gnu; build-script warning + stub on targets with no hardware decode API). | | `videotoolbox` | Pin VideoToolbox; `compile_error!` on non-Apple targets. | | `vaapi` | Pin VAAPI; `compile_error!` off linux-gnu. | | `mediacodec` | Pin MediaCodec; `compile_error!` off Android. | + +## Testing + +The device tests (`tests/videotoolbox_device.rs`, `tests/vaapi_device.rs`) run +real decodes and skip gracefully when the machine has no decoder for the +codec, so CI stays green without ever exercising hardware. That means CI does +**not** cover decode correctness: run `just test-hw` locally, which is the +documented pre-release gate (see "Pre-release hardware verification" in +`DEVELOPMENT.md`). diff --git a/crates/rawshift-hwdec/src/vaapi/av1.rs b/crates/rawshift-hwdec/src/bitstream/av1.rs similarity index 99% rename from crates/rawshift-hwdec/src/vaapi/av1.rs rename to crates/rawshift-hwdec/src/bitstream/av1.rs index 66b6d96..be3e52c 100644 --- a/crates/rawshift-hwdec/src/vaapi/av1.rs +++ b/crates/rawshift-hwdec/src/bitstream/av1.rs @@ -1,5 +1,5 @@ -//! AV1 still-picture parsing for the VAAPI backend — **safe Rust only** -//! (no `unsafe`; FFI stays in `sys.rs`/`mod.rs`). +//! AV1 still-picture parsing shared by hardware backends — **safe Rust only** +//! (no `unsafe`; platform FFI stays in backend modules). //! //! ## Scope //! @@ -16,7 +16,8 @@ //! `VASliceParameterBufferAV1` per tile. use super::bits::{BitReader, PResult, ParseError, clip3}; -use super::sys; +#[cfg(hwdec_backend = "vaapi")] +use crate::vaapi::sys; // ── OBU framing (§5.3) ────────────────────────────────────────────────────── @@ -1144,6 +1145,7 @@ pub fn parse_still_picture(config_obus: &[u8], payload: &[u8]) -> PResult sys::VASliceParameterBufferAV1 { sys::VASliceParameterBufferAV1 { slice_data_size: tile.data.len() as u32, @@ -1516,6 +1519,9 @@ mod tests { ); } + // Exercises the VAAPI parameter-buffer builders, which only exist in a + // VAAPI build (the parsers above are shared by every backend). + #[cfg(hwdec_backend = "vaapi")] #[test] fn pic_param_maps_seq_and_frame_fields() { let pic = parse_still_picture(&[], AV1_64X64_LIBAOM).unwrap(); diff --git a/crates/rawshift-hwdec/src/vaapi/bits.rs b/crates/rawshift-hwdec/src/bitstream/bits.rs similarity index 99% rename from crates/rawshift-hwdec/src/vaapi/bits.rs rename to crates/rawshift-hwdec/src/bitstream/bits.rs index 4e34c14..3f328d8 100644 --- a/crates/rawshift-hwdec/src/vaapi/bits.rs +++ b/crates/rawshift-hwdec/src/bitstream/bits.rs @@ -1,4 +1,4 @@ -//! Safe bitstream readers shared by the HEVC and AV1 header parsers. +//! Safe bitstream readers shared by the HEVC and AV1 still-picture parsers. //! //! **Safe Rust only** — this module (like `hevc.rs` / `av1.rs`) contains no //! `unsafe`; all FFI stays in `sys.rs` and the call sites in `mod.rs`. diff --git a/crates/rawshift-hwdec/src/vaapi/hevc.rs b/crates/rawshift-hwdec/src/bitstream/hevc.rs similarity index 98% rename from crates/rawshift-hwdec/src/vaapi/hevc.rs rename to crates/rawshift-hwdec/src/bitstream/hevc.rs index f9df68c..b52d30d 100644 --- a/crates/rawshift-hwdec/src/vaapi/hevc.rs +++ b/crates/rawshift-hwdec/src/bitstream/hevc.rs @@ -1,5 +1,5 @@ -//! HEVC still-picture header parsing for the VAAPI backend — **safe Rust -//! only** (no `unsafe`; FFI stays in `sys.rs`/`mod.rs`). +//! HEVC still-picture header parsing shared by hardware backends — **safe +//! Rust only** (no `unsafe`; platform FFI stays in backend modules). //! //! ## Scope //! @@ -23,7 +23,8 @@ //! `slice_data()`) and the emulation-prevention-byte count VAAPI wants. use super::bits::{BitReader, PResult, ParseError, Rbsp, rbsp_from_nal_payload}; -use super::sys; +#[cfg(hwdec_backend = "vaapi")] +use crate::vaapi::sys; // ── NAL classification ────────────────────────────────────────────────────── @@ -743,6 +744,7 @@ fn ceil_log2(x: u32) -> u32 { /// Uniform tile partition of `total` CTBs into `count` parts, as the /// `minus1` sizes VAAPI wants (H.265 §6.5.1 derivation). +#[cfg(hwdec_backend = "vaapi")] fn uniform_partition_minus1(total: u32, count: u32) -> Vec { (0..count) .map(|i| (((i + 1) * total) / count - (i * total) / count - 1) as u16) @@ -752,6 +754,7 @@ fn uniform_partition_minus1(total: u32, count: u32) -> Vec { /// Fill `VAPictureParameterBufferHEVC` for a still picture decoded into /// `surface`. `nut` is the slice NAL type; `st_rps_bits` comes from the /// first independent slice header. +#[cfg(hwdec_backend = "vaapi")] pub fn build_pic_param( sps: &Sps, pps: &Pps, @@ -894,6 +897,7 @@ pub fn build_pic_param( /// Fill `VASliceParameterBufferHEVC` for one coded slice NAL of /// `slice_data_size` bytes at offset 0 of its own data buffer. +#[cfg(hwdec_backend = "vaapi")] pub fn build_slice_param( sh: &SliceHeader, pps: &Pps, @@ -1019,6 +1023,9 @@ mod tests { assert!(!pps.slice_segment_header_extension_present); } + // Exercises the VAAPI parameter-buffer builders, which only exist in a + // VAAPI build (the parsers above are shared by every backend). + #[cfg(hwdec_backend = "vaapi")] #[test] fn pic_param_packs_bitfields() { let sps = parse_sps(SPS_64X64_X265).unwrap(); @@ -1044,6 +1051,9 @@ mod tests { ); } + // Exercises the VAAPI parameter-buffer builders, which only exist in a + // VAAPI build (the parsers above are shared by every backend). + #[cfg(hwdec_backend = "vaapi")] #[test] fn uniform_tile_partition_covers_exactly() { // 10 CTBs into 3 columns: 3+3+4 (spec derivation gives 3,3,4). @@ -1052,6 +1062,9 @@ mod tests { assert_eq!(parts.len(), 3); } + // Exercises the VAAPI parameter-buffer builders, which only exist in a + // VAAPI build (the parsers above are shared by every backend). + #[cfg(hwdec_backend = "vaapi")] #[test] fn slice_param_marks_last_slice_and_i_type() { let sh = SliceHeader { diff --git a/crates/rawshift-hwdec/src/bitstream/mod.rs b/crates/rawshift-hwdec/src/bitstream/mod.rs new file mode 100644 index 0000000..d36ca63 --- /dev/null +++ b/crates/rawshift-hwdec/src/bitstream/mod.rs @@ -0,0 +1,17 @@ +//! Safe parsing and framing shared by the platform hardware-decode backends. +//! +//! Each backend consumes a different subset of this module, because the +//! platform APIs sit at different levels. VAAPI is a slice-level API, so it +//! needs everything down to slice headers, tile groups and the +//! `build_*_param` builders. VideoToolbox takes the configuration record and +//! the coded sample almost as-is, so it only needs `hvcC`/`av1C` parsing, NAL +//! classification and the AV1 sequence header — it lets the framework do the +//! rest. The unused remainder is therefore expected rather than dead, and the +//! allow below keeps `-D warnings` builds honest for every backend selection +//! without scattering per-item `cfg`s that would couple this module to the +//! backend list. +#![allow(dead_code)] + +pub(crate) mod av1; +pub(crate) mod bits; +pub(crate) mod hevc; diff --git a/crates/rawshift-hwdec/src/lib.rs b/crates/rawshift-hwdec/src/lib.rs index 5d77541..bed51e1 100644 --- a/crates/rawshift-hwdec/src/lib.rs +++ b/crates/rawshift-hwdec/src/lib.rs @@ -10,11 +10,12 @@ //! Per `PRINCIPLES.md`, **all** platform FFI for hardware decode lives in this //! crate and nowhere else: `#![deny(unsafe_op_in_unsafe_fn)]`, every public //! item is safe, and every `unsafe` block documents its invariants inside the -//! platform backend module that owns it. The **VAAPI backend** (linux-gnu, -//! dlopen'd libva — see the `vaapi` module) is implemented; VideoToolbox and -//! MediaCodec land as separate issues. On targets/builds with no backend -//! every entry point reports "no decoder" — [`decoder`] returns `None`, -//! [`backend`] returns `None`, and [`available_codecs`] is empty. +//! platform backend module that owns it. The **VideoToolbox backend** +//! (macOS/iOS, linked system frameworks, hardware-only) and the **VAAPI +//! backend** (linux-gnu, dlopen'd libva) are implemented; MediaCodec lands as +//! a separate issue. On targets/builds with no backend every entry point +//! reports "no decoder" — [`decoder`] returns `None`, [`backend`] returns +//! `None`, and [`available_codecs`] is empty. //! //! On NVIDIA GPUs the VAAPI backend works through the maintained //! [`nvidia-vaapi-driver`](https://github.com/elFarto/nvidia-vaapi-driver) @@ -48,6 +49,19 @@ #![deny(unsafe_op_in_unsafe_fn)] +#[cfg(any( + hwdec_backend = "vaapi", + hwdec_backend = "videotoolbox", + hwdec_backend = "mediacodec", + test +))] +mod bitstream; + +// The VideoToolbox platform backend: compiled only when build.rs selected it +// (`videotoolbox` explicit flag, or `hw` on a macOS/iOS target). +#[cfg(hwdec_backend = "videotoolbox")] +mod videotoolbox; + // The VAAPI platform backend: compiled only when build.rs selected it // (`vaapi` explicit flag, or `hw` on a linux-gnu target). #[cfg(hwdec_backend = "vaapi")] @@ -475,19 +489,29 @@ pub enum HwDecodeError { /// Returns a decoder for `codec`, or `None` when no compiled-in backend can /// decode it at runtime. /// +/// With the VideoToolbox backend compiled in (`videotoolbox`, or `hw` on +/// macOS/iOS), this answers from `VTIsHardwareDecodeSupported` — so AV1 is +/// reported only on hardware that actually has an AV1 decode block (M3 / +/// A17 Pro and later), while HEVC keeps working on older machines. +/// /// With the VAAPI backend compiled in (`vaapi`, or `hw` on linux-gnu), this /// dlopens libva on first use and answers from the driver's actual /// profile/entrypoint list; missing libraries, render nodes, or driver /// support all degrade to `None` (never a link or startup failure). -/// VideoToolbox and MediaCodec land as separate issues; without a backend -/// this returns `None` everywhere. +/// +/// MediaCodec lands as a separate issue; without a backend this returns +/// `None` everywhere. #[must_use] pub fn decoder(codec: HwCodec) -> Option> { + #[cfg(hwdec_backend = "videotoolbox")] + { + videotoolbox::decoder(codec) + } #[cfg(hwdec_backend = "vaapi")] { vaapi::decoder(codec) } - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] { let _ = codec; None @@ -497,32 +521,42 @@ pub fn decoder(codec: HwCodec) -> Option> { /// The platform backend compiled into this build and usable at runtime, or /// `None`. /// -/// Reports `Some` only when the runtime probe succeeds (e.g. VAAPI's dlopen +/// Reports `Some` only when the runtime probe succeeds (e.g. VideoToolbox +/// reporting hardware decode for at least one codec, or VAAPI's dlopen /// finding a driver with at least one supported codec at the VLD entry /// point). #[must_use] pub fn backend() -> Option { + #[cfg(hwdec_backend = "videotoolbox")] + { + videotoolbox::backend() + } #[cfg(hwdec_backend = "vaapi")] { vaapi::backend() } - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] { None } } /// The codecs [`decoder`] can currently return a decoder for, per the -/// runtime probe (e.g. `vaQueryConfigProfiles` for VAAPI). +/// runtime probe (`VTIsHardwareDecodeSupported` for VideoToolbox, +/// `vaQueryConfigProfiles` for VAAPI). /// /// Empty when no backend is compiled in or usable. #[must_use] pub fn available_codecs() -> &'static [HwCodec] { + #[cfg(hwdec_backend = "videotoolbox")] + { + videotoolbox::available_codecs() + } #[cfg(hwdec_backend = "vaapi")] { vaapi::available_codecs() } - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] { &[] } @@ -534,14 +568,14 @@ mod tests { // ── stub behaviour (builds with no selected backend) ──────────────────── - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] #[test] fn stub_has_no_decoder_for_any_codec() { assert!(decoder(HwCodec::Hevc).is_none()); assert!(decoder(HwCodec::Av1).is_none()); } - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] #[test] fn stub_reports_no_backend_and_no_codecs() { assert_eq!(backend(), None); diff --git a/crates/rawshift-hwdec/src/vaapi/mod.rs b/crates/rawshift-hwdec/src/vaapi/mod.rs index faf0aed..52fff27 100644 --- a/crates/rawshift-hwdec/src/vaapi/mod.rs +++ b/crates/rawshift-hwdec/src/vaapi/mod.rs @@ -36,16 +36,14 @@ //! invariant. The bitstream parsers ([`bits`], [`hevc`], [`av1`]) are safe //! Rust. -mod av1; -mod bits; -mod hevc; -mod sys; +pub(crate) mod sys; use std::ffi::c_int; use std::fs::File; use std::os::fd::AsRawFd; use std::sync::OnceLock; +use crate::bitstream::{av1, bits, hevc}; use crate::{ CodecConfig, ColorRange, DecodedFrame, HwBackend, HwCodec, HwDecodeError, HwStillDecoder, PixelFormat, Plane, StillDecodeRequest, diff --git a/crates/rawshift-hwdec/src/videotoolbox/format.rs b/crates/rawshift-hwdec/src/videotoolbox/format.rs new file mode 100644 index 0000000..497c0dc --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/format.rs @@ -0,0 +1,425 @@ +//! Building the `CMVideoFormatDescription` that a decompression session is +//! created from, for both codecs. +//! +//! ## HEVC +//! +//! `CMVideoFormatDescriptionCreateFromHEVCParameterSets` takes the VPS/SPS/PPS +//! NAL units out of the `hvcC` record plus the record's NAL length prefix +//! width. Two things fall out of that choice: +//! +//! - VideoToolbox parses the SPS itself, so the coded dimensions never have to +//! come from [`StillDecodeRequest::width`]/`height` — which the HEIC adapter +//! passes as `0` precisely because `hvcC` carries no picture size. +//! - `nal_unit_header_length` *is* the "length-prefixed samples, no Annex-B +//! conversion" contract: the payload is handed to the decoder byte for byte. +//! +//! ## AV1 +//! +//! CoreMedia has no `…CreateFromAV1ParameterSets`, so the generic +//! `CMVideoFormatDescriptionCreate` is used with the whole `av1C` record body +//! carried in the standard `SampleDescriptionExtensionAtoms` dictionary. That +//! call *does* require non-zero dimensions, which come from the AV1 sequence +//! header (authoritative, and what VideoToolbox itself will parse) with the +//! advisory request fields as a last resort. + +use std::ffi::c_int; +use std::ptr::{self, NonNull}; + +use objc2_core_foundation::{CFData, CFDictionary, CFRetained, CFString, CFType}; +use objc2_core_media::{ + CMFormatDescription, CMVideoFormatDescriptionCreate, + CMVideoFormatDescriptionCreateFromHEVCParameterSets, + kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, kCMVideoCodecType_AV1, +}; + +use super::status::{NO_ERR, decode_err, map_status}; +use crate::bitstream::{av1, hevc}; +use crate::{HwCodec, HwDecodeError, StillDecodeRequest}; + +/// The NAL unit types `CMVideoFormatDescriptionCreateFromHEVCParameterSets` +/// accepts: VPS (32), SPS (33), PPS (34), prefix SEI (39), suffix SEI (40). +const PARAMETER_SET_NALS: [u8; 5] = [32, 33, 34, 39, 40]; + +/// Erase a typed dictionary's phantom generics for the C signatures, which all +/// take the fully opaque `CFDictionary`. +pub(super) fn as_opaque_dict(dict: &CFDictionary) -> &CFDictionary { + // SAFETY: `CFDictionary` is a zero-sized opaque wrapper whose type + // parameters are pure `PhantomData` — every instantiation has identical + // layout and wraps the same underlying `CFDictionaryRef`. The C entry + // points only read it as an opaque dictionary. + unsafe { &*(dict as *const CFDictionary).cast::() } +} + +/// Take ownership of a `Create`-rule format description returned through an +/// out-parameter. +fn adopt( + codec: HwCodec, + operation: &str, + status: i32, + raw: *const CMFormatDescription, +) -> Result, HwDecodeError> { + if status != NO_ERR { + return Err(map_status(codec, operation, status)); + } + let ptr = NonNull::new(raw.cast_mut()) + .ok_or_else(|| decode_err(codec, format!("{operation} returned no format description")))?; + // SAFETY: the call succeeded, so `ptr` is a format description the Create + // rule already gave us a +1 reference to; `CFRetained::from_raw` adopts + // that reference and releases it exactly once on drop. + Ok(unsafe { CFRetained::from_raw(ptr) }) +} + +/// Build the format description for an HEVC still from its `hvcC` record. +pub(super) fn hevc_format_description( + hvcc_bytes: &[u8], + request: &StillDecodeRequest<'_>, +) -> Result, HwDecodeError> { + let codec = HwCodec::Hevc; + let to_err = |e: crate::bitstream::bits::ParseError| decode_err(codec, e.0); + let hvcc = hevc::parse_hvcc(hvcc_bytes).map_err(to_err)?; + + // Parameter sets from the hvcC arrays, in record order. + let mut sets: Vec<&[u8]> = Vec::new(); + let (mut has_vps, mut has_sps, mut has_pps) = (false, false, false); + for nal in &hvcc.nal_units { + let nal_type = hevc::nal_type(nal).map_err(to_err)?; + if !PARAMETER_SET_NALS.contains(&nal_type) { + continue; + } + match nal_type { + 32 => has_vps = true, + 33 => has_sps = true, + 34 => has_pps = true, + _ => {} + } + sets.push(nal.as_slice()); + } + + // Some encoders leave one or more parameter sets out of the hvcC arrays + // and send them in band instead; the VAAPI backend accepts either source, + // so this one does too. + if !(has_vps && has_sps && has_pps) { + let payload_nals = + hevc::split_length_prefixed(request.payload, hvcc.nal_length_size).map_err(to_err)?; + for nal in payload_nals { + let nal_type = hevc::nal_type(nal).map_err(to_err)?; + let wanted = match nal_type { + 32 if !has_vps => { + has_vps = true; + true + } + 33 if !has_sps => { + has_sps = true; + true + } + 34 if !has_pps => { + has_pps = true; + true + } + _ => false, + }; + if wanted { + sets.push(nal); + } + } + } + + if !(has_vps && has_sps && has_pps) { + return Err(decode_err( + codec, + "hvcC and payload carry no complete VPS/SPS/PPS set \ + (VideoToolbox requires all three)", + )); + } + + let pointers: Vec> = sets + .iter() + .map(|nal| { + NonNull::new(nal.as_ptr().cast_mut()) + .ok_or_else(|| decode_err(codec, "empty parameter set NAL")) + }) + .collect::>()?; + let sizes: Vec = sets.iter().map(|nal| nal.len()).collect(); + + let mut raw: *const CMFormatDescription = ptr::null(); + // SAFETY: `pointers` and `sizes` are non-empty parallel arrays of exactly + // `pointers.len()` entries. Each pointer is the start of a live parameter + // set NAL borrowed from `hvcc` or `request.payload`, both of which outlive + // this call, and the matching `sizes` entry is that NAL's length. + // `nal_length_size` is 1, 2 or 4 (`parse_hvcc` rejects 3). `raw` is a live + // local that the framework writes exactly one +1 reference into. + let status = unsafe { + CMVideoFormatDescriptionCreateFromHEVCParameterSets( + None, + pointers.len(), + NonNull::from(&pointers[0]), + NonNull::from(&sizes[0]), + hvcc.nal_length_size as c_int, + None, + NonNull::from(&mut raw), + ) + }; + adopt( + codec, + "CMVideoFormatDescriptionCreateFromHEVCParameterSets", + status, + raw, + ) +} + +/// `OBU_SEQUENCE_HEADER` (AV1 §6.2.2). +const OBU_SEQUENCE_HEADER: u8 = 1; + +/// Encode a length as a LEB128 `obu_size` field (AV1 §4.10.5). +fn leb128(mut value: usize, out: &mut Vec) { + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + out.push(byte); + if value == 0 { + break; + } + } +} + +/// Re-frame a sequence-header payload as a canonical OBU with a size field. +/// +/// `split_obus` hands back payloads rather than byte ranges, so a sequence +/// header recovered from the sample is rebuilt here instead of being sliced +/// out of the original stream. The result is byte-identical in meaning: OBU +/// type 1, `obu_has_size_field = 1`, no extension. +fn sequence_header_obu(payload: &[u8]) -> Vec { + let mut obu = Vec::with_capacity(payload.len() + 3); + obu.push((OBU_SEQUENCE_HEADER << 3) | 0b10); // type, obu_has_size_field + leb128(payload.len(), &mut obu); + obu.extend_from_slice(payload); + obu +} + +/// The sequence header for an AV1 still, from `av1C.configOBUs` if it is +/// there and from the sample otherwise. +/// +/// AVIF permits the sequence header in either place, but **VideoToolbox +/// requires it in the `av1C` config atom**: given a record whose `configOBUs` +/// are empty it fails `VTDecompressionSessionCreate` outright with +/// `kVTVideoDecoderMalfunctionErr`, at every picture size. So when the sample +/// is carrying it, the record is rebuilt with it (see [`av1_format_description`]). +fn av1_sequence_header( + config_obus: &[u8], + payload: &[u8], +) -> Result<(av1::SequenceHeader, Option>), HwDecodeError> { + let codec = HwCodec::Av1; + let to_err = |e: crate::bitstream::bits::ParseError| decode_err(codec, e.0); + // `parse_sequence_header` takes an OBU *payload*, so the stream has to be + // split first — handing it the raw stream would parse the OBU header bytes + // as sequence-header fields and yield nonsense dimensions. + for (source, from_config) in [(config_obus, true), (payload, false)] { + if source.is_empty() { + continue; + } + let obus = av1::split_obus(source).map_err(to_err)?; + let Some(obu) = obus.iter().find(|obu| obu.obu_type == OBU_SEQUENCE_HEADER) else { + continue; + }; + let seq = av1::parse_sequence_header(obu.payload).map_err(to_err)?; + let rebuilt = (!from_config).then(|| sequence_header_obu(obu.payload)); + return Ok((seq, rebuilt)); + } + Err(decode_err( + codec, + "AV1 stream carries no sequence header in either the av1C record or \ + the sample; VideoToolbox cannot be configured without one", + )) +} + +/// Build the format description for an AV1 still from its `av1C` record. +pub(super) fn av1_format_description( + av1c_bytes: &[u8], + request: &StillDecodeRequest<'_>, +) -> Result, HwDecodeError> { + let codec = HwCodec::Av1; + let to_err = |e: crate::bitstream::bits::ParseError| decode_err(codec, e.0); + let av1c = av1::parse_av1c(av1c_bytes).map_err(to_err)?; + if av1c.seq_profile != 0 { + return Err(decode_err( + codec, + format!( + "AV1 profile {} is outside the still-picture scope \ + (VideoToolbox decodes Main / Profile 0)", + av1c.seq_profile + ), + )); + } + + let (seq, rebuilt) = av1_sequence_header(av1c.config_obus, request.payload)?; + + // The sequence header is authoritative for the coded size — and it is what + // VideoToolbox itself parses. The container's advisory `ispe` size is only + // a fallback, because it can legitimately disagree (rotation, cropping). + let (width, height) = match (seq.max_frame_width, seq.max_frame_height) { + (w, h) if w > 0 && h > 0 => (w, h), + _ if request.width > 0 && request.height > 0 => (request.width, request.height), + _ => { + return Err(decode_err( + codec, + "AV1 sequence header declares no picture size and the container \ + supplied none", + )); + } + }; + let width = i32::try_from(width) + .map_err(|_| decode_err(codec, "AV1 picture width does not fit in i32"))?; + let height = i32::try_from(height) + .map_err(|_| decode_err(codec, "AV1 picture height does not fit in i32"))?; + + // `SampleDescriptionExtensionAtoms = { "av1C": }` — the + // whole record including its four-byte header, not just `configOBUs`. When + // the sequence header lived in the sample rather than the record, it is + // appended here so VideoToolbox gets the config it insists on. + let record: Vec; + let atom_bytes = match rebuilt { + Some(seq_obu) => { + record = av1c_bytes + .get(..4) + .unwrap_or(av1c_bytes) + .iter() + .copied() + .chain(seq_obu) + .collect(); + record.as_slice() + } + None => av1c_bytes, + }; + let atom_key = CFString::from_static_str("av1C"); + let atom_value = CFData::from_bytes(atom_bytes); + let atoms: CFRetained> = + CFDictionary::from_slices(&[&*atom_key], &[atom_value.as_ref() as &CFType]); + // SAFETY: an immortal constant `CFStringRef` exported by CoreMedia. + let atoms_key = unsafe { kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms }; + let extensions: CFRetained> = + CFDictionary::from_slices(&[atoms_key], &[atoms.as_ref() as &CFType]); + + let mut raw: *const CMFormatDescription = ptr::null(); + // SAFETY: `extensions` is a live dictionary whose keys are `CFString` and + // whose values are CF objects, which is what the "extensions generics must + // be of the correct type" precondition asks for; it is deep-copied by the + // call, so it need not outlive it. `raw` is a live local the framework + // writes exactly one +1 reference into. + let status = unsafe { + CMVideoFormatDescriptionCreate( + None, + kCMVideoCodecType_AV1, + width, + height, + Some(as_opaque_dict(&extensions)), + NonNull::from(&mut raw), + ) + }; + adopt(codec, "CMVideoFormatDescriptionCreate", status, raw) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ChromaSubsampling, CodecConfig}; + + fn request<'a>(config: CodecConfig<'a>, payload: &'a [u8]) -> StillDecodeRequest<'a> { + StillDecodeRequest { + config, + payload, + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + } + } + + /// A 23-byte hvcC header plus one array per parameter set, built the way + /// `tests/vaapi_device.rs` builds them from an Annex-B stream. + fn hvcc_with(sets: &[(u8, &[u8])]) -> Vec { + let mut hvcc = vec![0u8; 23]; + hvcc[0] = 1; // configurationVersion + hvcc[21] = 0x03; // lengthSizeMinusOne = 3 -> 4-byte prefixes + hvcc[22] = sets.len() as u8; + for (nal_type, nal) in sets { + hvcc.push(0x80 | nal_type); + hvcc.extend_from_slice(&1u16.to_be_bytes()); + hvcc.extend_from_slice(&(nal.len() as u16).to_be_bytes()); + hvcc.extend_from_slice(nal); + } + hvcc + } + + /// A minimal NAL with the given type in its two-byte header. + fn nal(nal_type: u8) -> Vec { + vec![nal_type << 1, 0x01, 0xAA, 0xBB] + } + + #[test] + fn hevc_requires_a_complete_parameter_set() { + // VPS + SPS but no PPS, and nothing in band to make up for it. + let vps = nal(32); + let sps = nal(33); + let hvcc = hvcc_with(&[(32, &vps), (33, &sps)]); + let err = hevc_format_description(&hvcc, &request(CodecConfig::Hvcc(&hvcc), &[])) + .expect_err("an incomplete parameter set must be rejected"); + assert!( + err.to_string().contains("VPS/SPS/PPS"), + "message must say what is missing: {err}" + ); + } + + #[test] + fn av1_rejects_profiles_outside_main() { + // av1C: marker/version, seq_profile = 1 in the top three bits. + let av1c = [0x81u8, 0x20, 0x0c, 0x00]; + let err = av1_format_description(&av1c, &request(CodecConfig::Av1c(&av1c), &[])) + .expect_err("profile 1 must be rejected"); + assert!( + err.to_string().contains("profile 1"), + "message must name the profile: {err}" + ); + } + + #[test] + fn av1_without_any_sequence_header_is_rejected() { + let av1c = [0x81u8, 0x00, 0x0c, 0x00]; + let err = av1_format_description(&av1c, &request(CodecConfig::Av1c(&av1c), &[])) + .expect_err("no sequence header anywhere must be an error"); + assert!( + err.to_string().contains("no sequence header"), + "message must say what is missing: {err}" + ); + } + + #[test] + fn leb128_matches_the_spec_encoding() { + let mut out = Vec::new(); + leb128(0, &mut out); + assert_eq!(out, [0x00]); + out.clear(); + leb128(127, &mut out); + assert_eq!(out, [0x7f]); + out.clear(); + leb128(128, &mut out); + assert_eq!(out, [0x80, 0x01]); + out.clear(); + leb128(300, &mut out); + assert_eq!(out, [0xac, 0x02]); + } + + #[test] + fn rebuilt_sequence_header_obu_round_trips() { + let payload = [0xAAu8; 13]; + let obu = sequence_header_obu(&payload); + // type 1, obu_has_size_field set, no extension. + assert_eq!(obu[0], 0x0A); + let parsed = av1::split_obus(&obu).expect("rebuilt OBU parses"); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].obu_type, OBU_SEQUENCE_HEADER); + assert_eq!(parsed[0].payload, &payload[..]); + } +} diff --git a/crates/rawshift-hwdec/src/videotoolbox/mod.rs b/crates/rawshift-hwdec/src/videotoolbox/mod.rs new file mode 100644 index 0000000..6a526a5 --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/mod.rs @@ -0,0 +1,613 @@ +//! VideoToolbox still-frame decode backend for macOS and iOS. +//! +//! ## Runtime model +//! +//! VideoToolbox, CoreMedia and CoreVideo are always-present Apple system +//! frameworks, so unlike the VAAPI backend there is nothing to `dlopen` and no +//! device node to find: availability is a pure capability question, answered +//! once per process by [`VTIsHardwareDecodeSupported`] for each codec. +//! +//! **Hardware only.** On macOS the session is created with +//! `kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder`, so a +//! machine without a decode block for the codec fails session creation instead +//! of silently falling back to VideoToolbox's *software* decoder — which would +//! make a crate called `rawshift-hwdec` quietly lie about what it did. That +//! key is macOS-only (see the `decoder_specification` docs); on iOS every +//! VideoToolbox decoder is hardware anyway. +//! +//! ## Codec scope +//! +//! - **HEVC** (HEIC) still pictures: `hvcC` parameter sets, length-prefixed +//! NAL payload passed through byte for byte — no Annex-B conversion. +//! - **AV1** (AVIF) Profile 0 still pictures: `av1C` config atom, raw OBU +//! temporal unit payload. Runtime-probed, because AV1 hardware decode +//! arrived with the M3 and A17 Pro generations — on older Apple silicon and +//! on Intel, [`available_codecs`] honestly omits [`HwCodec::Av1`] while HEVC +//! keeps working. +//! +//! Both emit 8-bit or 10-bit 4:2:0 → `Nv12` / `P010` (see [`pixels`]). +//! +//! ## Binding choice +//! +//! This backend uses the generated [`objc2`](https://github.com/madsmtm/objc2) +//! framework crates rather than a hand-written `sys.rs`, the opposite call +//! from the VAAPI backend. The reason is that the two situations differ: +//! libva must be `dlopen`'d so a machine without it degrades to "no decoder" +//! instead of failing to start, which rules out generated bindings; the Apple +//! frameworks are guaranteed present and link normally. Given that, the +//! maintained bindings win on every axis that matters here — `CFRetained` +//! gives us Create/Get-rule reference counting as RAII instead of hand-paired +//! `CFRetain`/`CFRelease`, and the `#[link(kind = "framework")]` attributes +//! live upstream. They are pulled in with `default-features = false` and only +//! the per-header features this file needs, so no Objective-C runtime, Metal, +//! OpenGL or CoreAudio code is compiled in. See `Cargo.toml`. +//! +//! ## Safety boundary +//! +//! All `unsafe` in this backend is FFI, plus the one `unsafe impl Send` on the +//! decoder. The bitstream parsers in [`crate::bitstream`] and the pixel +//! plumbing in [`pixels`] are safe Rust apart from the single documented slice +//! construction over a locked surface. + +mod format; +mod pixels; +mod sample; +mod status; + +use std::cell::UnsafeCell; +use std::ffi::c_void; +use std::ptr::{self, NonNull}; +use std::sync::OnceLock; + +use objc2_core_foundation::CFRetained; +use objc2_core_media::{CMFormatDescription, CMTime, kCMVideoCodecType_HEVC}; +use objc2_core_video::CVImageBuffer; +use objc2_video_toolbox::{ + VTDecodeFrameFlags, VTDecodeInfoFlags, VTDecompressionOutputCallbackRecord, + VTDecompressionSession, VTIsHardwareDecodeSupported, +}; + +use self::status::{NO_ERR, OSStatus, decode_err, map_status}; +use crate::{ + CodecConfig, DecodedFrame, HwBackend, HwCodec, HwDecodeError, HwStillDecoder, + StillDecodeRequest, +}; + +// ── Public backend surface (called from lib.rs) ───────────────────────────── + +/// Backend hook for [`crate::decoder`]. +pub fn decoder(codec: HwCodec) -> Option> { + let caps = probe(); + let supported = match codec { + HwCodec::Hevc => caps.hevc, + HwCodec::Av1 => caps.av1, + }; + if !supported { + return None; + } + // Nothing to acquire up front: the decompression session is created lazily + // on the first decode, from that picture's format description. This keeps + // `decoder(c).is_some()` exactly equal to `available_codecs().contains(&c)`, + // which `crate::tests::discovery_entry_points_are_consistent` requires. + Some(Box::new(VideoToolboxStillDecoder::new(codec))) +} + +/// Backend hook for [`crate::backend`]. +pub fn backend() -> Option { + let caps = probe(); + (caps.hevc || caps.av1).then_some(HwBackend::VideoToolbox) +} + +/// Backend hook for [`crate::available_codecs`]. +pub fn available_codecs() -> &'static [HwCodec] { + const NONE: &[HwCodec] = &[]; + const HEVC_ONLY: &[HwCodec] = &[HwCodec::Hevc]; + const AV1_ONLY: &[HwCodec] = &[HwCodec::Av1]; + const BOTH: &[HwCodec] = &[HwCodec::Hevc, HwCodec::Av1]; + let caps = probe(); + match (caps.hevc, caps.av1) { + (true, true) => BOTH, + (true, false) => HEVC_ONLY, + (false, true) => AV1_ONLY, + (false, false) => NONE, + } +} + +// ── Runtime probe ─────────────────────────────────────────────────────────── + +/// `kCMVideoCodecType_AV1` — `'av01'`. +/// +/// Named locally rather than imported so the constant is greppable next to its +/// HEVC peer; the value is the one `objc2-core-media` generates. +const CODEC_TYPE_AV1: u32 = objc2_core_media::kCMVideoCodecType_AV1; + +/// What this machine's VideoToolbox reports *hardware* decode support for. +#[derive(Debug, Clone, Copy, Default)] +struct Caps { + hevc: bool, + av1: bool, +} + +/// Probe once per process: hardware does not change at runtime. +fn probe() -> Caps { + static CAPS: OnceLock = OnceLock::new(); + *CAPS.get_or_init(|| { + // SAFETY: `VTIsHardwareDecodeSupported` takes a four-character codec + // type by value and only reads static system capability — no pointers, + // no session, no allocation. It is available since macOS 10.13 / + // iOS 11, both below this crate's macOS 11 / iOS 14 floor (see + // docs/SUPPORT.md), so the symbol is always present and needs no weak + // linking. An unrecognised codec type returns false rather than + // trapping. + unsafe { + Caps { + hevc: VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC), + av1: VTIsHardwareDecodeSupported(CODEC_TYPE_AV1), + } + } + }) +} + +/// The decoder specification handed to `VTDecompressionSessionCreate`. +/// +/// On macOS this pins the session to a hardware decoder: Apple documents +/// `RequireHardwareAcceleratedVideoDecoder` as making session creation *fail* +/// when hardware acceleration is not possible, which is what this crate wants +/// — a clean [`HwDecodeError::Unavailable`] instead of a silent software +/// decode. Because [`probe`] already gates on `VTIsHardwareDecodeSupported`, +/// it should never fire spuriously. +/// +/// On iOS it returns `None`, and that is **not** cosmetic: the +/// `kVTVideoDecoderSpecification_*` symbols are annotated `ios(17.0)` and the +/// binding is a plain non-weak `extern` static, so merely referencing one in a +/// build targeting iOS 14–16 would be a dyld "symbol not found" *launch* +/// failure for the whole app. iOS has no software VideoToolbox decoder to +/// guard against in any case. +#[cfg(target_os = "macos")] +fn decoder_specification() -> Option< + CFRetained< + objc2_core_foundation::CFDictionary< + objc2_core_foundation::CFString, + objc2_core_foundation::CFType, + >, + >, +> { + use objc2_core_foundation::{CFDictionary, CFType, kCFBooleanTrue}; + use objc2_video_toolbox::kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder; + + // SAFETY: both are immortal constant CF objects exported by VideoToolbox + // and CoreFoundation; reading them performs no allocation and they are + // valid for the life of the process. + let (key, value) = unsafe { + ( + kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder, + kCFBooleanTrue?, + ) + }; + Some(CFDictionary::from_slices( + &[key], + &[value.as_ref() as &CFType], + )) +} + +#[cfg(not(target_os = "macos"))] +fn decoder_specification() -> Option< + CFRetained< + objc2_core_foundation::CFDictionary< + objc2_core_foundation::CFString, + objc2_core_foundation::CFType, + >, + >, +> { + None +} + +// ── Output callback ───────────────────────────────────────────────────────── + +/// Where the decompression callback deposits its result. +/// +/// Heap-allocated and owned by the decoder so its address stays stable for the +/// life of the session that was created with a pointer to it. +struct DecodeSlot { + status: OSStatus, + info: VTDecodeInfoFlags, + image: Option>, +} + +impl Default for DecodeSlot { + fn default() -> Self { + Self { + status: NO_ERR, + info: VTDecodeInfoFlags::empty(), + image: None, + } + } +} + +/// `VTDecompressionOutputCallback`: called once per decoded frame. +/// +/// This runs on the calling thread inside `VTDecompressionSessionDecodeFrame` +/// because the decode is submitted with neither +/// `kVTDecodeFrame_EnableAsynchronousDecompression` nor +/// `kVTDecodeFrame_EnableTemporalProcessing`. +/// +/// The body must never unwind across the `extern "C-unwind"` boundary, so it +/// deliberately does no allocation beyond a single `CFRetain`, no formatting, +/// no indexing and no `unwrap`. +unsafe extern "C-unwind" fn output_callback( + output_ref_con: *mut c_void, + _source_ref_con: *mut c_void, + status: OSStatus, + info: VTDecodeInfoFlags, + image: *mut CVImageBuffer, + _pts: CMTime, + _duration: CMTime, +) { + // SAFETY: `output_ref_con` is the `DecodeSlot` pointer this decoder + // registered in its `VTDecompressionOutputCallbackRecord`, and the slot's + // heap allocation outlives the session (field order on + // `VideoToolboxStillDecoder` drops the session first). The decode is + // synchronous under `&mut self`, so this callback runs on the calling + // thread inside `decode_frame` and no other reference to the slot is live. + // `image` is a borrowed +0 buffer valid only for this call, so + // `CFRetained::retain` takes the +1 needed to outlive it. + unsafe { + let slot = &mut *(output_ref_con.cast::()); + slot.status = status; + slot.info = info; + slot.image = NonNull::new(image).map(|p| CFRetained::retain(p)); + } +} + +// ── Session ───────────────────────────────────────────────────────────────── + +/// A live decompression session and the format description it was created +/// from. +struct Session { + /// The configuration record bytes this session was built for. Grid HEICs + /// decode hundreds of tiles that share one `hvcC`, so a byte compare here + /// skips both session *and* format-description construction. + config: Vec, + format: CFRetained, + session: CFRetained, +} + +impl Drop for Session { + fn drop(&mut self) { + // SAFETY: `session` was created successfully by this decoder and is + // invalidated exactly once, here. VideoToolbox documents Invalidate as + // the deterministic teardown to call before the last release, which + // the `CFRetained` field performs immediately afterwards. + unsafe { self.session.invalidate() }; + } +} + +// ── Decoder ───────────────────────────────────────────────────────────────── + +struct VideoToolboxStillDecoder { + codec: HwCodec, + /// Declared before `slot` so it is dropped first: invalidating the session + /// severs the callback's reference to the slot before the slot's + /// allocation goes away. **Reordering these two fields is a use-after-free.** + session: Option, + slot: Box>, +} + +// SAFETY: the decoder owns its decompression session, its format description +// and its callback slot exclusively — `decode_still` is the only entry point +// and takes `&mut self`, so no two threads can touch them at once. +// VideoToolbox sessions are not thread-affine: they require no run loop and no +// main thread, only that calls be externally serialised, which exclusive +// ownership guarantees. The output callback runs synchronously on the calling +// thread inside `decode_frame` (neither the asynchronous nor the +// temporal-processing flag is set, and every decode is followed by +// `wait_for_asynchronous_frames`), so the raw slot pointer is never +// dereferenced from another thread. `Send` is required by the +// [`HwStillDecoder`] contract; `Sync` is deliberately not claimed. +unsafe impl Send for VideoToolboxStillDecoder {} + +impl VideoToolboxStillDecoder { + fn new(codec: HwCodec) -> Self { + Self { + codec, + session: None, + slot: Box::new(UnsafeCell::new(DecodeSlot::default())), + } + } + + /// Get a session that can decode `format`, reusing the cached one when it + /// can accept the picture. + fn session_for( + &mut self, + config_bytes: &[u8], + format: CFRetained, + ) -> Result<&Session, HwDecodeError> { + let reusable = match self.session.as_ref() { + // Same configuration record: the fast path a grid HEIC or an AVIF + // batch takes on every tile after the first. + Some(existing) if existing.config == config_bytes => true, + // Different record, but some decoders accommodate minor format + // changes without a new session. + Some(existing) => { + // SAFETY: `existing.session` is a live session created by this + // decoder and `format` a live description; the call only + // queries the decoder and mutates nothing. + unsafe { existing.session.can_accept_format_description(&format) } + } + None => false, + }; + + if reusable { + let existing = self.session.as_mut().expect("checked above"); + existing.config = config_bytes.to_vec(); + existing.format = format; + return Ok(self.session.as_ref().expect("just set")); + } + + // Drop any old session before creating a new one, so two sessions + // never hold decode resources at the same time. + self.session = None; + + let specification = decoder_specification(); + let callback = VTDecompressionOutputCallbackRecord { + decompressionOutputCallback: Some(output_callback), + decompressionOutputRefCon: self.slot.get().cast::(), + }; + + let destination = pixels::destination_attributes(); + + let create_session = |spec: Option<&objc2_core_foundation::CFDictionary>| { + let mut raw: *mut VTDecompressionSession = ptr::null_mut(); + // SAFETY: `format` is a live video format description. The callback + // record is a live local that the framework copies during the + // call, and its refcon points at this decoder's heap-allocated + // slot, which outlives the session (see the field-order note on + // the struct). `spec` and `destination` are live dictionaries of + // CF types. `raw` is a live local the framework writes exactly one + // +1 reference into on success. + let status = unsafe { + VTDecompressionSession::create( + None, + &format, + spec, + Some(format::as_opaque_dict(&destination)), + &raw const callback, + NonNull::from(&mut raw), + ) + }; + (status, raw) + }; + + // Hardware-pinned first (macOS; on iOS the specification is None and + // there is nothing to retry). + let (mut status, mut raw) = + create_session(specification.as_deref().map(format::as_opaque_dict)); + + // Apple's hardware HEVC block refuses pictures below roughly 64x64, + // failing session creation with `kVTVideoDecoderMalfunctionErr`. Small + // items are entirely ordinary in HEIF — a `thmb` thumbnail is + // routinely 32x32 — so refusing them outright would be a worse answer + // than letting VideoToolbox use its own decoder for that one picture. + // A failed hardware-pinned create is therefore retried once without + // the requirement. + // + // This never manufactures support for a codec the machine cannot + // decode: availability is still gated on `VTIsHardwareDecodeSupported`, + // so the retry only widens the picture *sizes* an already-available + // codec accepts. If it fails too, the original hardware status is what + // gets reported, because that is the one that describes the real + // problem. + if status != NO_ERR && specification.is_some() { + let hardware_status = status; + let (retry_status, retry_raw) = create_session(None); + if retry_status == NO_ERR { + status = retry_status; + raw = retry_raw; + } else { + return Err(map_status( + self.codec, + "VTDecompressionSessionCreate", + hardware_status, + )); + } + } + if status != NO_ERR { + return Err(map_status( + self.codec, + "VTDecompressionSessionCreate", + status, + )); + } + let session_ptr = NonNull::new(raw).ok_or_else(|| { + decode_err( + self.codec, + "VTDecompressionSessionCreate returned no session", + ) + })?; + // SAFETY: the call succeeded, so `session_ptr` carries the +1 + // reference the Create rule granted; `from_raw` adopts it and the + // `Drop` impl invalidates then releases it exactly once. + let session = unsafe { CFRetained::from_raw(session_ptr) }; + + self.session = Some(Session { + config: config_bytes.to_vec(), + format, + session, + }); + Ok(self.session.as_ref().expect("just set")) + } + + /// Submit one coded picture and copy the decoded surface out. + fn decode( + &mut self, + config_bytes: &[u8], + format: CFRetained, + payload: &[u8], + ) -> Result { + let codec = self.codec; + + // Reset the slot before submitting so a decoder that never calls back + // cannot leave us reading the previous picture. + // SAFETY: `&mut self` is exclusive and the callback has not run yet. + unsafe { *self.slot.get() = DecodeSlot::default() }; + + let session = self.session_for(config_bytes, format)?; + let sample = sample::wrap(codec, &session.format, payload)?; + + // SAFETY: `session` was created with this decoder's callback record, + // and the sample buffer is ready and described by the very format + // description the session holds. A null source refcon and a null + // info-flags out-pointer are both explicitly permitted. No + // asynchronous or temporal-processing flag is set, so the output + // callback runs before this returns. + let status = unsafe { + session.session.decode_frame( + &sample.buffer, + VTDecodeFrameFlags::empty(), + ptr::null_mut(), + ptr::null_mut(), + ) + }; + if status != NO_ERR { + return Err(map_status( + codec, + "VTDecompressionSessionDecodeFrame", + status, + )); + } + + // A no-op for a synchronous decode, and a hard barrier for a decoder + // that went asynchronous anyway. This is what makes the zero-copy + // block buffer in `sample::wrap` sound. + // SAFETY: `session` is live and owned by this decoder. + let wait = unsafe { session.session.wait_for_asynchronous_frames() }; + if wait != NO_ERR { + return Err(map_status( + codec, + "VTDecompressionSessionWaitForAsynchronousFrames", + wait, + )); + } + + // SAFETY: `&mut self` is exclusive and, per the flags above, the + // callback has finished running on this thread. + let slot = unsafe { &mut *self.slot.get() }; + if slot.status != NO_ERR { + return Err(map_status( + codec, + "VideoToolbox decode callback", + slot.status, + )); + } + if slot + .info + .intersects(VTDecodeInfoFlags::FrameDropped | VTDecodeInfoFlags::FrameInterrupted) + { + return Err(decode_err( + codec, + "VideoToolbox dropped or interrupted the frame without producing pixels", + )); + } + let image = slot + .image + .take() + .ok_or_else(|| decode_err(codec, "VideoToolbox produced no image buffer"))?; + + pixels::to_decoded_frame(codec, &image) + } +} + +impl HwStillDecoder for VideoToolboxStillDecoder { + fn decode_still( + &mut self, + request: &StillDecodeRequest<'_>, + ) -> Result { + if request.codec() != self.codec { + return Err(HwDecodeError::Decode { + codec: request.codec(), + message: format!("decoder was opened for {}", self.codec), + }); + } + // Cheapest check first: without a payload there is nothing to describe + // a format for, let alone decode. + if request.payload.is_empty() { + return Err(decode_err(self.codec, "coded picture payload is empty")); + } + match request.config { + CodecConfig::Hvcc(hvcc) => { + let format = format::hevc_format_description(hvcc, request)?; + self.decode(hvcc, format, request.payload) + } + CodecConfig::Av1c(av1c) => { + let format = format::av1_format_description(av1c, request)?; + self.decode(av1c, format, request.payload) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The probe must never panic and must agree with itself across calls, + /// on any Apple machine including one with no decode block at all. + #[test] + fn probe_is_infallible_and_cached() { + let first = probe(); + let second = probe(); + assert_eq!(first.hevc, second.hevc); + assert_eq!(first.av1, second.av1); + } + + /// The invariant `crate::tests::discovery_entry_points_are_consistent` + /// enforces, checked against this backend's own `Caps` so a future edit to + /// one of the three hooks cannot drift from the others. + #[test] + fn discovery_hooks_agree_with_caps() { + let caps = probe(); + let codecs = available_codecs(); + assert_eq!(codecs.contains(&HwCodec::Hevc), caps.hevc); + assert_eq!(codecs.contains(&HwCodec::Av1), caps.av1); + assert_eq!(backend().is_some(), !codecs.is_empty()); + assert_eq!(decoder(HwCodec::Hevc).is_some(), caps.hevc); + assert_eq!(decoder(HwCodec::Av1).is_some(), caps.av1); + } + + /// A decoder opened for one codec must reject the other rather than + /// building a format description for a session that cannot take it. + #[test] + fn decoder_rejects_the_other_codec() { + let mut decoder = VideoToolboxStillDecoder::new(HwCodec::Hevc); + let request = StillDecodeRequest { + config: CodecConfig::Av1c(&[0x81, 0x00, 0x0c, 0x00]), + payload: &[0x12, 0x00], + width: 64, + height: 64, + bit_depth: 8, + chroma: crate::ChromaSubsampling::Cs420, + }; + let err = decoder + .decode_still(&request) + .expect_err("codec mismatch must be rejected"); + assert!(err.to_string().contains("opened for HEVC"), "{err}"); + } + + /// An empty payload must be refused before any framework call. + #[test] + fn empty_payload_is_rejected() { + let mut decoder = VideoToolboxStillDecoder::new(HwCodec::Av1); + let request = StillDecodeRequest { + config: CodecConfig::Av1c(&[0x81, 0x00, 0x0c, 0x00]), + payload: &[], + width: 64, + height: 64, + bit_depth: 8, + chroma: crate::ChromaSubsampling::Cs420, + }; + let err = decoder + .decode_still(&request) + .expect_err("an empty payload must be rejected"); + assert!(err.to_string().contains("empty"), "{err}"); + } +} diff --git a/crates/rawshift-hwdec/src/videotoolbox/pixels.rs b/crates/rawshift-hwdec/src/videotoolbox/pixels.rs new file mode 100644 index 0000000..356723f --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/pixels.rs @@ -0,0 +1,406 @@ +//! `CVPixelBuffer` → [`DecodedFrame`]. +//! +//! VideoToolbox hands back the decoder's native surface (this backend passes +//! no destination pixel-buffer attributes, so no conversion is imposed). The +//! surface's `OSType` is therefore the ground truth for pixel layout, colour +//! range and bit depth, and [`describe_format`] is the single table that +//! interprets it. +//! +//! ## Geometry +//! +//! `CVPixelBufferGetWidth`/`GetHeight` report the **clean** aperture — the +//! conformance window is already applied — so, unlike the VAAPI backend, this +//! one does no cropping arithmetic. `videotoolbox_crops_to_the_conformance_window` +//! in `tests/videotoolbox_device.rs` pins that assumption to real hardware. +//! +//! ## 10-bit layout +//! +//! CoreVideo's `x420`/`xf20` are documented in `CVPixelBuffer.h` as "2 plane +//! YCbCr10 4:2:0, each 10 bits in the **MSBs** of 16bits". That is exactly +//! [`PixelFormat::P010`]'s contract, so the copy is a plain per-row `memcpy` +//! with no shifting or byte-swapping (every Apple target is little-endian). +//! Do not "fix" this to shift right. +//! +//! [`PixelFormat::I010`] is unreachable here: CoreVideo has no planar 10-bit +//! 4:2:0 type. Its absence from the table below is deliberate. + +use objc2_core_foundation::{CFArray, CFDictionary, CFNumber, CFRetained, CFString, CFType}; +use objc2_core_video::{ + CVPixelBuffer, CVPixelBufferGetBaseAddressOfPlane, CVPixelBufferGetBytesPerRowOfPlane, + CVPixelBufferGetHeight, CVPixelBufferGetHeightOfPlane, CVPixelBufferGetPixelFormatType, + CVPixelBufferGetPlaneCount, CVPixelBufferGetWidth, CVPixelBufferLockBaseAddress, + CVPixelBufferLockFlags, CVPixelBufferUnlockBaseAddress, kCVPixelBufferPixelFormatTypeKey, + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8Planar, + kCVPixelFormatType_420YpCbCr8PlanarFullRange, kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, kCVReturnSuccess, +}; + +use super::status::{OSType, decode_err}; +use crate::{ColorRange, DecodedFrame, HwCodec, HwDecodeError, PixelFormat, Plane}; + +/// Read-only lock: we copy the planes out and never write to the surface. +const READ_ONLY: CVPixelBufferLockFlags = CVPixelBufferLockFlags::ReadOnly; + +// CoreVideo's constants keep Apple's C spelling, which is not a legal Rust +// pattern name. Alias them once here so the table below reads idiomatically +// and each four-character code sits next to the constant it belongs to. + +/// `'420v'` — 8-bit biplanar 4:2:0, video range (luma 16..235). +const NV12_VIDEO_RANGE: OSType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; +/// `'420f'` — 8-bit biplanar 4:2:0, full range. +const NV12_FULL_RANGE: OSType = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; +/// `'x420'` — 10-bit biplanar 4:2:0, video range, sample in the MSBs. +const P010_VIDEO_RANGE: OSType = kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange; +/// `'xf20'` — 10-bit biplanar 4:2:0, full range, sample in the MSBs. +const P010_FULL_RANGE: OSType = kCVPixelFormatType_420YpCbCr10BiPlanarFullRange; +/// `'y420'` — 8-bit planar 4:2:0, video range. +const I420_VIDEO_RANGE: OSType = kCVPixelFormatType_420YpCbCr8Planar; +/// `'f420'` — 8-bit planar 4:2:0, full range. +const I420_FULL_RANGE: OSType = kCVPixelFormatType_420YpCbCr8PlanarFullRange; + +/// The pixel formats a decompression session is asked to produce. +/// +/// **Why ask at all.** Left to itself, VideoToolbox hands back the decoder's +/// native surface, and on Apple silicon a 10-bit HEVC decode natively produces +/// `'p420'` — a format that appears in no public CoreVideo header, whose plane +/// contents demonstrably do **not** match the documented `x420` samples for +/// the same bitstream. There is no safe way to interpret it. Offering an +/// explicit list keeps the decoder on documented, linear layouts. +/// +/// **Why a list rather than one format.** All four documented 4:2:0 surfaces +/// are offered — both depths and both ranges — so VideoToolbox still picks the +/// one that matches the stream instead of converting to a format we imposed. +/// That is what preserves the video/full range distinction, which +/// [`describe_format`] then reads back off the chosen surface. +pub(super) fn destination_attributes() -> CFRetained> { + let formats = [ + NV12_VIDEO_RANGE, + NV12_FULL_RANGE, + P010_VIDEO_RANGE, + P010_FULL_RANGE, + ] + .map(|fourcc| CFNumber::new_i32(fourcc as i32)); + let refs: Vec<&CFNumber> = formats.iter().map(|n| &**n).collect(); + let list = CFArray::from_objects(&refs); + // SAFETY: an immortal constant `CFStringRef` exported by CoreVideo. + let key = unsafe { kCVPixelBufferPixelFormatTypeKey }; + CFDictionary::from_slices(&[key], &[list.as_ref() as &CFType]) +} + +/// What a CoreVideo pixel format means in this crate's vocabulary. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct SurfaceFormat { + pub(super) format: PixelFormat, + pub(super) range: ColorRange, + pub(super) bit_depth: u8, +} + +/// The 4:2:0 surfaces a VideoToolbox HEVC/AV1 still decode can emit. +/// +/// Anything else — 4:2:2/4:4:4, and the lossless (`&8v0`) / lossy (`-8v0`) +/// packed families, which have no CPU-addressable plane layout — is rejected +/// by name rather than misread. +pub(super) fn describe_format(fourcc: OSType) -> Option { + let described = match fourcc { + // '420v' — 8-bit biplanar, video range (luma 16..235). + NV12_VIDEO_RANGE => SurfaceFormat { + format: PixelFormat::Nv12, + range: ColorRange::Limited, + bit_depth: 8, + }, + // '420f' — 8-bit biplanar, full range. + NV12_FULL_RANGE => SurfaceFormat { + format: PixelFormat::Nv12, + range: ColorRange::Full, + bit_depth: 8, + }, + // 'x420' — 10-bit biplanar, video range, sample in the MSBs. + P010_VIDEO_RANGE => SurfaceFormat { + format: PixelFormat::P010, + range: ColorRange::Limited, + bit_depth: 10, + }, + // 'xf20' — 10-bit biplanar, full range, sample in the MSBs. + P010_FULL_RANGE => SurfaceFormat { + format: PixelFormat::P010, + range: ColorRange::Full, + bit_depth: 10, + }, + // 'y420' — 8-bit planar, video range. + I420_VIDEO_RANGE => SurfaceFormat { + format: PixelFormat::I420, + range: ColorRange::Limited, + bit_depth: 8, + }, + // 'f420' — 8-bit planar, full range. + I420_FULL_RANGE => SurfaceFormat { + format: PixelFormat::I420, + range: ColorRange::Full, + bit_depth: 8, + }, + _ => return None, + }; + Some(described) +} + +/// Render a four-character code the way the CoreVideo headers write it. +pub(super) fn fourcc_name(fourcc: OSType) -> String { + let bytes = fourcc.to_be_bytes(); + if bytes.iter().all(|b| (0x20..=0x7e).contains(b)) { + format!("'{}'", bytes.iter().map(|&b| b as char).collect::()) + } else { + format!("{fourcc:#010x}") + } +} + +/// The per-plane byte geometry [`DecodedFrame::new`] validates against. +/// +/// Derived from the frame dimensions with the same ceiling rules the +/// constructor uses, never from `CVPixelBufferGetWidthOfPlane`, so +/// construction cannot disagree with validation. +fn plane_geometry(format: PixelFormat, width: u32, height: u32) -> Vec<(usize, usize)> { + let bps = format.bytes_per_sample(); + let (cw, ch) = (width.div_ceil(2) as usize, height.div_ceil(2) as usize); + let luma = (width as usize * bps, height as usize); + match format.plane_count() { + // Biplanar: full-size Y, then half-size interleaved CbCr (two samples + // per chroma column). + 2 => vec![luma, (cw * 2 * bps, ch)], + // Planar: full-size Y, then half-size Cb and Cr. + _ => vec![luma, (cw * bps, ch), (cw * bps, ch)], + } +} + +/// Copy one plane into a tightly packed [`Plane`] (`stride == row_bytes`). +/// +/// Every offset is checked: a surface whose reported geometry does not cover +/// the rows we need yields `None` rather than reading out of bounds. +fn copy_plane(base: *const u8, stride: usize, row_bytes: usize, rows: usize) -> Option { + if base.is_null() || rows == 0 || stride < row_bytes { + return None; + } + // The last row needs only `row_bytes`, not a full stride — reading a whole + // stride past the final row would run off the end of the mapping. + let span = stride + .checked_mul(rows.checked_sub(1)?)? + .checked_add(row_bytes)?; + // SAFETY: the pixel buffer is locked read-only by the caller for the whole + // of this call, so CoreVideo guarantees `bytesPerRow * (heightOfPlane - 1) + // + rowBytes` initialised bytes at the plane base address. The caller has + // checked `rows <= heightOfPlane` and `row_bytes <= stride`, so `span` is + // within that guarantee. The slice is immutable, is not aliased by any + // other Rust reference, and dies before the matching unlock. + let src = unsafe { std::slice::from_raw_parts(base, span) }; + let mut data = Vec::with_capacity(row_bytes.checked_mul(rows)?); + for row in 0..rows { + let start = stride.checked_mul(row)?; + data.extend_from_slice(src.get(start..start.checked_add(row_bytes)?)?); + } + Some(Plane { + data, + stride: row_bytes, + }) +} + +/// Lock the surface, copy every plane out, unlock, and build the frame. +pub(super) fn to_decoded_frame( + codec: HwCodec, + pixel_buffer: &CVPixelBuffer, +) -> Result { + let fourcc = CVPixelBufferGetPixelFormatType(pixel_buffer); + let described = describe_format(fourcc).ok_or_else(|| { + decode_err( + codec, + format!( + "VideoToolbox emitted unsupported pixel format {} \ + (this backend handles 8/10-bit 4:2:0 only)", + fourcc_name(fourcc) + ), + ) + })?; + + let width = u32::try_from(CVPixelBufferGetWidth(pixel_buffer)) + .map_err(|_| decode_err(codec, "decoded surface width does not fit in u32"))?; + let height = u32::try_from(CVPixelBufferGetHeight(pixel_buffer)) + .map_err(|_| decode_err(codec, "decoded surface height does not fit in u32"))?; + + let geometry = plane_geometry(described.format, width, height); + let plane_count = CVPixelBufferGetPlaneCount(pixel_buffer); + if plane_count != geometry.len() { + return Err(decode_err( + codec, + format!( + "decoded surface {} reports {plane_count} planes, expected {}", + fourcc_name(fourcc), + geometry.len() + ), + )); + } + + // SAFETY: `pixel_buffer` is a live, retained surface handed to us by the + // decompression callback. Locking read-only makes the planes CPU- + // addressable; it is paired with exactly one unlock below, which runs on + // every path because the copy is done inside a closure whose result is + // only returned after unlocking. + let lock = unsafe { CVPixelBufferLockBaseAddress(pixel_buffer, READ_ONLY) }; + if lock != kCVReturnSuccess { + return Err(decode_err( + codec, + format!("CVPixelBufferLockBaseAddress failed (CVReturn {lock})"), + )); + } + + let result = (|| -> Result { + let mut planes = Vec::with_capacity(geometry.len()); + for (index, &(row_bytes, rows)) in geometry.iter().enumerate() { + let stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, index); + let plane_rows = CVPixelBufferGetHeightOfPlane(pixel_buffer, index); + if stride < row_bytes || plane_rows < rows { + return Err(decode_err( + codec, + format!( + "decoded surface plane {index} is {stride}x{plane_rows}, \ + too small for {row_bytes}x{rows}" + ), + )); + } + let base = CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, index).cast::(); + let plane = copy_plane(base, stride, row_bytes, rows).ok_or_else(|| { + decode_err( + codec, + format!("decoded surface plane {index} is not readable"), + ) + })?; + planes.push(plane); + } + DecodedFrame::new( + described.format, + width, + height, + described.bit_depth, + described.range, + planes, + ) + })(); + + // SAFETY: the surface was locked read-only immediately above with these + // exact flags and is unlocked exactly once here. Every slice borrowed from + // it is dead — the planes were copied into owned `Vec`s. + let _ = unsafe { CVPixelBufferUnlockBaseAddress(pixel_buffer, READ_ONLY) }; + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fourcc_table_maps_the_six_supported_surfaces() { + let cases = [ + (0x3432_3076_u32, PixelFormat::Nv12, ColorRange::Limited, 8), // '420v' + (0x3432_3066, PixelFormat::Nv12, ColorRange::Full, 8), // '420f' + (0x7834_3230, PixelFormat::P010, ColorRange::Limited, 10), // 'x420' + (0x7866_3230, PixelFormat::P010, ColorRange::Full, 10), // 'xf20' + (0x7934_3230, PixelFormat::I420, ColorRange::Limited, 8), // 'y420' + (0x6634_3230, PixelFormat::I420, ColorRange::Full, 8), // 'f420' + ]; + for (fourcc, format, range, bit_depth) in cases { + let described = describe_format(fourcc) + .unwrap_or_else(|| panic!("{} must be supported", fourcc_name(fourcc))); + assert_eq!(described.format, format, "{}", fourcc_name(fourcc)); + assert_eq!(described.range, range, "{}", fourcc_name(fourcc)); + assert_eq!(described.bit_depth, bit_depth, "{}", fourcc_name(fourcc)); + // The table must agree with the crate's own depth rules. + assert!(format.supports_bit_depth(bit_depth)); + } + } + + #[test] + fn unsupported_surfaces_are_rejected() { + for fourcc in [ + 0x2638_7630_u32, // '&8v0' lossless biplanar — no readable layout + 0x2d38_7630, // '-8v0' lossy biplanar + 0x7834_3232, // 'x422' 4:2:2 10-bit + 0x4241_4752, // 'BAGR' + 0, + ] { + assert!(describe_format(fourcc).is_none(), "{fourcc:#010x}"); + } + } + + #[test] + fn fourcc_name_renders_ascii_and_falls_back_to_hex() { + assert_eq!(fourcc_name(0x7834_3230), "'x420'"); + assert_eq!(fourcc_name(0x3432_3076), "'420v'"); + assert_eq!(fourcc_name(0), "0x00000000"); + } + + /// The geometry table must satisfy `DecodedFrame::new` exactly, including + /// the ceiling division for odd dimensions. + #[test] + fn plane_geometry_satisfies_decoded_frame_validation() { + for (format, bit_depth) in [ + (PixelFormat::Nv12, 8), + (PixelFormat::P010, 10), + (PixelFormat::I420, 8), + ] { + for (width, height) in [(5u32, 3u32), (64, 64), (61, 37), (1, 1)] { + let geometry = plane_geometry(format, width, height); + assert_eq!(geometry.len(), format.plane_count()); + let planes = geometry + .iter() + .map(|&(row_bytes, rows)| Plane { + data: vec![0u8; row_bytes * rows], + stride: row_bytes, + }) + .collect(); + DecodedFrame::new( + format, + width, + height, + bit_depth, + ColorRange::Limited, + planes, + ) + .unwrap_or_else(|e| panic!("{format:?} {width}x{height} must validate: {e}")); + } + } + } + + #[test] + fn copy_plane_tightens_a_padded_stride() { + // 4x2 luma in a buffer with an 8-byte stride: the padding must not + // survive into the returned plane. + let src: Vec = vec![ + 1, 2, 3, 4, 0xFF, 0xFF, 0xFF, 0xFF, // row 0 + padding + 5, 6, 7, 8, 0xFF, 0xFF, 0xFF, 0xFF, // row 1 + padding + ]; + let plane = copy_plane(src.as_ptr(), 8, 4, 2).expect("plane copies"); + assert_eq!(plane.stride, 4); + assert_eq!(plane.data, vec![1, 2, 3, 4, 5, 6, 7, 8]); + } + + #[test] + fn copy_plane_reads_only_row_bytes_from_the_final_row() { + // Exactly `stride * (rows - 1) + row_bytes` bytes are available — one + // byte less than `stride * rows`. A naive implementation would read + // past the end here. + let src: Vec = vec![1, 2, 3, 0xFF, 4, 5, 6]; + let plane = copy_plane(src.as_ptr(), 4, 3, 2).expect("plane copies"); + assert_eq!(plane.data, vec![1, 2, 3, 4, 5, 6]); + } + + #[test] + fn copy_plane_rejects_bad_geometry() { + let src = [0u8; 64]; + // Stride narrower than the used row width. + assert!(copy_plane(src.as_ptr(), 2, 4, 2).is_none()); + // Zero rows. + assert!(copy_plane(src.as_ptr(), 4, 4, 0).is_none()); + // Null base (a plane CoreVideo could not make CPU-addressable). + assert!(copy_plane(std::ptr::null(), 4, 4, 2).is_none()); + // Overflowing geometry must not wrap. + assert!(copy_plane(src.as_ptr(), usize::MAX, 4, 4).is_none()); + } +} diff --git a/crates/rawshift-hwdec/src/videotoolbox/sample.rs b/crates/rawshift-hwdec/src/videotoolbox/sample.rs new file mode 100644 index 0000000..4d963a2 --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/sample.rs @@ -0,0 +1,155 @@ +//! Wrapping a coded picture payload in the `CMSampleBuffer` a decompression +//! session consumes. +//! +//! The payload is wrapped **zero-copy**: `CMBlockBufferCreateWithMemoryBlock` +//! is given `kCFAllocatorNull` as the block allocator, so CoreMedia never +//! takes ownership of or frees the caller's bytes. That is sound because the +//! lifetime is bounded on both ends — the decode is synchronous (no +//! asynchronous or temporal-processing flags, plus an explicit +//! `VTDecompressionSessionWaitForAsynchronousFrames`), and the sample buffer +//! is released before `decode_still` returns. +//! +//! Both codecs share this path unchanged: HEVC payloads are length-prefixed +//! NAL units and AV1 payloads are raw OBU temporal units, and in each case the +//! bytes go to the decoder exactly as the container stored them. + +use std::ffi::c_void; +use std::ptr::{self, NonNull}; + +use objc2_core_foundation::{CFRetained, kCFAllocatorNull}; +use objc2_core_media::{ + CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMSampleTimingInfo, CMTime, CMTimeFlags, +}; + +use super::status::{NO_ERR, decode_err, map_status}; +use crate::{HwCodec, HwDecodeError}; + +/// Presentation timestamp 0/1. A *valid* PTS matters: some decoders drop a +/// frame whose PTS is `kCMTimeInvalid`. +const PTS_ZERO: CMTime = CMTime { + value: 0, + timescale: 1, + flags: CMTimeFlags::Valid, + epoch: 0, +}; + +/// `kCMTimeInvalid`: a still has no duration, and its samples are already in +/// presentation order so no decode timestamp is meaningful. +const TIME_INVALID: CMTime = CMTime { + value: 0, + timescale: 0, + flags: CMTimeFlags::empty(), + epoch: 0, +}; + +/// A ready sample buffer and the block buffer it reads through. +/// +/// Field order is load-bearing: `buffer` is declared before `_block` so the +/// sample buffer is released first, and the block buffer it references +/// outlives it. +pub(super) struct Sample { + pub(super) buffer: CFRetained, + _block: CFRetained, +} + +/// Wrap `payload` in a one-sample `CMSampleBuffer` described by `format`. +pub(super) fn wrap( + codec: HwCodec, + format: &CMFormatDescription, + payload: &[u8], +) -> Result { + if payload.is_empty() { + return Err(decode_err(codec, "coded picture payload is empty")); + } + + let mut raw_block: *mut CMBlockBuffer = ptr::null_mut(); + // SAFETY: `payload` is a live immutable slice for the whole of this call + // and of the decode that consumes the returned sample. `kCFAllocatorNull` + // tells CoreMedia never to free those bytes, and the decoder only reads + // them. The `*const -> *mut` cast is provenance-only: nothing writes + // through the pointer. `custom_block_source` is null, which the API + // permits when a memory block is supplied, and `raw_block` is a live local + // the framework writes exactly one +1 reference into. + let status = unsafe { + CMBlockBuffer::create_with_memory_block( + None, + payload.as_ptr().cast_mut().cast::(), + payload.len(), + kCFAllocatorNull, + ptr::null(), + 0, + payload.len(), + 0, + NonNull::from(&mut raw_block), + ) + }; + if status != NO_ERR { + return Err(map_status( + codec, + "CMBlockBufferCreateWithMemoryBlock", + status, + )); + } + let block_ptr = NonNull::new(raw_block).ok_or_else(|| { + decode_err( + codec, + "CMBlockBufferCreateWithMemoryBlock returned no buffer", + ) + })?; + // SAFETY: the call succeeded, so `block_ptr` carries the +1 reference the + // Create rule granted; `from_raw` adopts it and releases it once on drop. + let block = unsafe { CFRetained::from_raw(block_ptr) }; + + let timing = CMSampleTimingInfo { + duration: TIME_INVALID, + presentationTimeStamp: PTS_ZERO, + decodeTimeStamp: TIME_INVALID, + }; + let sample_size = payload.len(); + + let mut raw_sample: *mut CMSampleBuffer = ptr::null_mut(); + // SAFETY: one sample spanning the whole block buffer, so the timing and + // size arrays each have exactly the single entry their count declares and + // both outlive the call (the framework copies them). `block` and `format` + // are live. `raw_sample` is a live local the framework writes exactly one + // +1 reference into. + let status = unsafe { + CMSampleBuffer::create_ready( + None, + Some(&block), + Some(format), + 1, + 1, + &raw const timing, + 1, + &raw const sample_size, + NonNull::from(&mut raw_sample), + ) + }; + if status != NO_ERR { + return Err(map_status(codec, "CMSampleBufferCreateReady", status)); + } + let sample_ptr = NonNull::new(raw_sample) + .ok_or_else(|| decode_err(codec, "CMSampleBufferCreateReady returned no buffer"))?; + // SAFETY: as above — adopt the Create-rule +1 reference. + let buffer = unsafe { CFRetained::from_raw(sample_ptr) }; + + Ok(Sample { + buffer, + _block: block, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A still carries no duration and no decode timestamp, but its + /// presentation timestamp must be *valid* or decoders may drop the frame. + #[test] + fn still_timing_has_a_valid_pts_and_invalid_duration() { + assert!(PTS_ZERO.flags.contains(CMTimeFlags::Valid)); + assert_eq!(PTS_ZERO.timescale, 1); + assert!(!TIME_INVALID.flags.contains(CMTimeFlags::Valid)); + } +} diff --git a/crates/rawshift-hwdec/src/videotoolbox/status.rs b/crates/rawshift-hwdec/src/videotoolbox/status.rs new file mode 100644 index 0000000..503bec9 --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/status.rs @@ -0,0 +1,308 @@ +//! `OSStatus` naming and the mapping from platform status codes to +//! [`HwDecodeError`]. +//! +//! **Safe Rust only** — this module never calls into a framework; it just +//! interprets the integers they return. +//! +//! The `MacTypes.h` aliases are re-declared here because the objc2 crates +//! keep theirs `pub(crate)`. They are plain integer aliases, so naming them +//! locally is a readability convenience, not a redefinition of the ABI. + +use crate::{HwCodec, HwDecodeError}; + +/// `MacTypes.h` `OSStatus` — the return type of every VideoToolbox and +/// CoreMedia entry point. +pub(super) type OSStatus = i32; + +/// `MacTypes.h` `OSType` — a four-character code, e.g. a pixel format. +pub(super) type OSType = u32; + +/// `noErr`: the operation succeeded. +pub(super) const NO_ERR: OSStatus = 0; + +// ── VideoToolbox status codes (VTErrors.h) ────────────────────────────────── +// +// Transcribed from `objc2-video-toolbox`'s generated `VTErrors.rs`, which in +// turn mirrors the SDK header. The `Unavailable` set below is the subset that +// means "this machine cannot decode this codec/configuration right now"; every +// other status describes a failed decode of data we did hand over. + +const K_VT_PROPERTY_NOT_SUPPORTED_ERR: OSStatus = -12900; +const K_VT_PROPERTY_READ_ONLY_ERR: OSStatus = -12901; +const K_VT_PARAMETER_ERR: OSStatus = -12902; +const K_VT_INVALID_SESSION_ERR: OSStatus = -12903; +const K_VT_ALLOCATION_FAILED_ERR: OSStatus = -12904; +const K_VT_PIXEL_TRANSFER_NOT_SUPPORTED_ERR: OSStatus = -12905; +const K_VT_COULD_NOT_FIND_VIDEO_DECODER_ERR: OSStatus = -12906; +const K_VT_COULD_NOT_CREATE_INSTANCE_ERR: OSStatus = -12907; +const K_VT_COULD_NOT_FIND_VIDEO_ENCODER_ERR: OSStatus = -12908; +const K_VT_VIDEO_DECODER_BAD_DATA_ERR: OSStatus = -12909; +const K_VT_VIDEO_DECODER_UNSUPPORTED_DATA_FORMAT_ERR: OSStatus = -12910; +const K_VT_VIDEO_DECODER_MALFUNCTION_ERR: OSStatus = -12911; +const K_VT_VIDEO_ENCODER_MALFUNCTION_ERR: OSStatus = -12912; +const K_VT_VIDEO_DECODER_NOT_AVAILABLE_NOW_ERR: OSStatus = -12913; +const K_VT_PIXEL_ROTATION_NOT_SUPPORTED_ERR: OSStatus = -12914; +const K_VT_VIDEO_ENCODER_NOT_AVAILABLE_NOW_ERR: OSStatus = -12915; +const K_VT_FORMAT_DESCRIPTION_CHANGE_NOT_SUPPORTED_ERR: OSStatus = -12916; +const K_VT_INSUFFICIENT_SOURCE_COLOR_DATA_ERR: OSStatus = -12917; +const K_VT_COULD_NOT_CREATE_COLOR_CORRECTION_DATA_ERR: OSStatus = -12918; +const K_VT_COLOR_SYNC_TRANSFORM_CONVERT_FAILED_ERR: OSStatus = -12919; +const K_VT_VIDEO_DECODER_AUTHORIZATION_ERR: OSStatus = -12210; +const K_VT_VIDEO_ENCODER_AUTHORIZATION_ERR: OSStatus = -12211; +const K_VT_COLOR_CORRECTION_PIXEL_TRANSFER_FAILED_ERR: OSStatus = -12212; +const K_VT_MULTI_PASS_STORAGE_IDENTIFIER_MISMATCH_ERR: OSStatus = -12213; +const K_VT_MULTI_PASS_STORAGE_INVALID_ERR: OSStatus = -12214; +const K_VT_FRAME_SILO_INVALID_TIME_STAMP_ERR: OSStatus = -12215; +const K_VT_FRAME_SILO_INVALID_TIME_RANGE_ERR: OSStatus = -12216; +const K_VT_COULD_NOT_FIND_TEMPORAL_FILTER_ERR: OSStatus = -12217; +const K_VT_PIXEL_TRANSFER_NOT_PERMITTED_ERR: OSStatus = -12218; +const K_VT_COLOR_CORRECTION_IMAGE_ROTATION_FAILED_ERR: OSStatus = -12219; +const K_VT_VIDEO_DECODER_REMOVED_ERR: OSStatus = -17690; +const K_VT_SESSION_MALFUNCTION_ERR: OSStatus = -17691; +const K_VT_VIDEO_DECODER_NEEDS_ROSETTA_ERR: OSStatus = -17692; +const K_VT_VIDEO_ENCODER_NEEDS_ROSETTA_ERR: OSStatus = -17693; +const K_VT_VIDEO_DECODER_REFERENCE_MISSING_ERR: OSStatus = -17694; +const K_VT_VIDEO_DECODER_CALLBACK_MESSAGING_ERR: OSStatus = -17695; +const K_VT_VIDEO_DECODER_UNKNOWN_ERR: OSStatus = -17696; +const K_VT_EXTENSION_DISABLED_ERR: OSStatus = -17697; +const K_VT_VIDEO_ENCODER_MV_REQUIRED_ERR: OSStatus = -17698; + +// ── CoreMedia status codes (CMBlockBuffer.h / CMSampleBuffer.h) ───────────── + +const K_CM_BLOCK_BUFFER_STRUCTURE_ALLOCATION_FAILED_ERR: OSStatus = -12700; +const K_CM_BLOCK_BUFFER_BLOCK_ALLOCATION_FAILED_ERR: OSStatus = -12701; +const K_CM_BLOCK_BUFFER_BAD_CUSTOM_BLOCK_SOURCE_ERR: OSStatus = -12702; +const K_CM_BLOCK_BUFFER_BAD_OFFSET_PARAMETER_ERR: OSStatus = -12703; +const K_CM_BLOCK_BUFFER_BAD_LENGTH_PARAMETER_ERR: OSStatus = -12704; +const K_CM_BLOCK_BUFFER_BAD_POINTER_PARAMETER_ERR: OSStatus = -12705; +const K_CM_BLOCK_BUFFER_EMPTY_B_BUF_ERR: OSStatus = -12706; +const K_CM_BLOCK_BUFFER_UNALLOCATED_BLOCK_ERR: OSStatus = -12707; +const K_CM_BLOCK_BUFFER_INSUFFICIENT_SPACE_ERR: OSStatus = -12708; +const K_CM_SAMPLE_BUFFER_ERROR_ALLOCATION_FAILED: OSStatus = -12730; +const K_CM_SAMPLE_BUFFER_ERROR_REQUIRED_PARAMETER_MISSING: OSStatus = -12731; +const K_CM_SAMPLE_BUFFER_ERROR_INVALID_MEDIA_TYPE_FOR_OPERATION: OSStatus = -12734; +const K_CM_SAMPLE_BUFFER_ERROR_INVALID_SAMPLE_DATA: OSStatus = -12735; +const K_CM_FORMAT_DESCRIPTION_ERROR_INVALID_PARAMETER: OSStatus = -12710; + +/// Statuses that mean "no usable hardware decoder for this codec or this +/// configuration on this machine", as opposed to "the data we handed over did +/// not decode". +/// +/// These become [`HwDecodeError::Unavailable`], which +/// `rawshift-image-heic`/`-avif` surface as `RawError::HwDecoderUnavailable` +/// — the matchable, honest "this build/machine cannot do it" signal — rather +/// than a decode failure that blames the file. +const UNAVAILABLE: &[OSStatus] = &[ + // No decoder exists for the codec. On macOS this is also what + // `RequireHardwareAcceleratedVideoDecoder` produces when the machine has + // no hardware decode block for the codec. + K_VT_COULD_NOT_FIND_VIDEO_DECODER_ERR, + // A decoder exists but could not be instantiated. + K_VT_COULD_NOT_CREATE_INSTANCE_ERR, + // The hardware will not accept this profile / bit depth / chroma. + K_VT_VIDEO_DECODER_UNSUPPORTED_DATA_FORMAT_ERR, + // Hardware decode resources are busy — transient, but from the caller's + // side still "no decoder available". + K_VT_VIDEO_DECODER_NOT_AVAILABLE_NOW_ERR, + // The decoder went away mid-session (e.g. an eGPU was unplugged). + K_VT_VIDEO_DECODER_REMOVED_ERR, + // The decoder is unusable in this process's architecture. + K_VT_VIDEO_DECODER_NEEDS_ROSETTA_ERR, + // The decoder is not authorised for this process. + K_VT_VIDEO_DECODER_AUTHORIZATION_ERR, + // The decoder lives in a system extension that is switched off. + K_VT_EXTENSION_DISABLED_ERR, +]; + +/// The conventional constant name for a VideoToolbox / CoreMedia status, for +/// error messages that a reader can grep the SDK headers for. +pub(super) fn status_name(status: OSStatus) -> Option<&'static str> { + let name = match status { + NO_ERR => "noErr", + K_VT_PROPERTY_NOT_SUPPORTED_ERR => "kVTPropertyNotSupportedErr", + K_VT_PROPERTY_READ_ONLY_ERR => "kVTPropertyReadOnlyErr", + K_VT_PARAMETER_ERR => "kVTParameterErr", + K_VT_INVALID_SESSION_ERR => "kVTInvalidSessionErr", + K_VT_ALLOCATION_FAILED_ERR => "kVTAllocationFailedErr", + K_VT_PIXEL_TRANSFER_NOT_SUPPORTED_ERR => "kVTPixelTransferNotSupportedErr", + K_VT_COULD_NOT_FIND_VIDEO_DECODER_ERR => "kVTCouldNotFindVideoDecoderErr", + K_VT_COULD_NOT_CREATE_INSTANCE_ERR => "kVTCouldNotCreateInstanceErr", + K_VT_COULD_NOT_FIND_VIDEO_ENCODER_ERR => "kVTCouldNotFindVideoEncoderErr", + K_VT_VIDEO_DECODER_BAD_DATA_ERR => "kVTVideoDecoderBadDataErr", + K_VT_VIDEO_DECODER_UNSUPPORTED_DATA_FORMAT_ERR => "kVTVideoDecoderUnsupportedDataFormatErr", + K_VT_VIDEO_DECODER_MALFUNCTION_ERR => "kVTVideoDecoderMalfunctionErr", + K_VT_VIDEO_ENCODER_MALFUNCTION_ERR => "kVTVideoEncoderMalfunctionErr", + K_VT_VIDEO_DECODER_NOT_AVAILABLE_NOW_ERR => "kVTVideoDecoderNotAvailableNowErr", + K_VT_PIXEL_ROTATION_NOT_SUPPORTED_ERR => "kVTPixelRotationNotSupportedErr", + K_VT_VIDEO_ENCODER_NOT_AVAILABLE_NOW_ERR => "kVTVideoEncoderNotAvailableNowErr", + K_VT_FORMAT_DESCRIPTION_CHANGE_NOT_SUPPORTED_ERR => { + "kVTFormatDescriptionChangeNotSupportedErr" + } + K_VT_INSUFFICIENT_SOURCE_COLOR_DATA_ERR => "kVTInsufficientSourceColorDataErr", + K_VT_COULD_NOT_CREATE_COLOR_CORRECTION_DATA_ERR => { + "kVTCouldNotCreateColorCorrectionDataErr" + } + K_VT_COLOR_SYNC_TRANSFORM_CONVERT_FAILED_ERR => "kVTColorSyncTransformConvertFailedErr", + K_VT_VIDEO_DECODER_AUTHORIZATION_ERR => "kVTVideoDecoderAuthorizationErr", + K_VT_VIDEO_ENCODER_AUTHORIZATION_ERR => "kVTVideoEncoderAuthorizationErr", + K_VT_COLOR_CORRECTION_PIXEL_TRANSFER_FAILED_ERR => { + "kVTColorCorrectionPixelTransferFailedErr" + } + K_VT_MULTI_PASS_STORAGE_IDENTIFIER_MISMATCH_ERR => { + "kVTMultiPassStorageIdentifierMismatchErr" + } + K_VT_MULTI_PASS_STORAGE_INVALID_ERR => "kVTMultiPassStorageInvalidErr", + K_VT_FRAME_SILO_INVALID_TIME_STAMP_ERR => "kVTFrameSiloInvalidTimeStampErr", + K_VT_FRAME_SILO_INVALID_TIME_RANGE_ERR => "kVTFrameSiloInvalidTimeRangeErr", + K_VT_COULD_NOT_FIND_TEMPORAL_FILTER_ERR => "kVTCouldNotFindTemporalFilterErr", + K_VT_PIXEL_TRANSFER_NOT_PERMITTED_ERR => "kVTPixelTransferNotPermittedErr", + K_VT_COLOR_CORRECTION_IMAGE_ROTATION_FAILED_ERR => { + "kVTColorCorrectionImageRotationFailedErr" + } + K_VT_VIDEO_DECODER_REMOVED_ERR => "kVTVideoDecoderRemovedErr", + K_VT_SESSION_MALFUNCTION_ERR => "kVTSessionMalfunctionErr", + K_VT_VIDEO_DECODER_NEEDS_ROSETTA_ERR => "kVTVideoDecoderNeedsRosettaErr", + K_VT_VIDEO_ENCODER_NEEDS_ROSETTA_ERR => "kVTVideoEncoderNeedsRosettaErr", + K_VT_VIDEO_DECODER_REFERENCE_MISSING_ERR => "kVTVideoDecoderReferenceMissingErr", + K_VT_VIDEO_DECODER_CALLBACK_MESSAGING_ERR => "kVTVideoDecoderCallbackMessagingErr", + K_VT_VIDEO_DECODER_UNKNOWN_ERR => "kVTVideoDecoderUnknownErr", + K_VT_EXTENSION_DISABLED_ERR => "kVTExtensionDisabledErr", + K_VT_VIDEO_ENCODER_MV_REQUIRED_ERR => "kVTVideoEncoderMVRequiredErr", + K_CM_BLOCK_BUFFER_STRUCTURE_ALLOCATION_FAILED_ERR => { + "kCMBlockBufferStructureAllocationFailedErr" + } + K_CM_BLOCK_BUFFER_BLOCK_ALLOCATION_FAILED_ERR => "kCMBlockBufferBlockAllocationFailedErr", + K_CM_BLOCK_BUFFER_BAD_CUSTOM_BLOCK_SOURCE_ERR => "kCMBlockBufferBadCustomBlockSourceErr", + K_CM_BLOCK_BUFFER_BAD_OFFSET_PARAMETER_ERR => "kCMBlockBufferBadOffsetParameterErr", + K_CM_BLOCK_BUFFER_BAD_LENGTH_PARAMETER_ERR => "kCMBlockBufferBadLengthParameterErr", + K_CM_BLOCK_BUFFER_BAD_POINTER_PARAMETER_ERR => "kCMBlockBufferBadPointerParameterErr", + K_CM_BLOCK_BUFFER_EMPTY_B_BUF_ERR => "kCMBlockBufferEmptyBBufErr", + K_CM_BLOCK_BUFFER_UNALLOCATED_BLOCK_ERR => "kCMBlockBufferUnallocatedBlockErr", + K_CM_BLOCK_BUFFER_INSUFFICIENT_SPACE_ERR => "kCMBlockBufferInsufficientSpaceErr", + K_CM_SAMPLE_BUFFER_ERROR_ALLOCATION_FAILED => "kCMSampleBufferError_AllocationFailed", + K_CM_SAMPLE_BUFFER_ERROR_REQUIRED_PARAMETER_MISSING => { + "kCMSampleBufferError_RequiredParameterMissing" + } + K_CM_SAMPLE_BUFFER_ERROR_INVALID_MEDIA_TYPE_FOR_OPERATION => { + "kCMSampleBufferError_InvalidMediaTypeForOperation" + } + K_CM_SAMPLE_BUFFER_ERROR_INVALID_SAMPLE_DATA => "kCMSampleBufferError_InvalidSampleData", + K_CM_FORMAT_DESCRIPTION_ERROR_INVALID_PARAMETER => { + "kCMFormatDescriptionError_InvalidParameter" + } + _ => return None, + }; + Some(name) +} + +/// Render a status for a human: the SDK constant name when we know it, always +/// with the numeric value so an unknown code is still actionable. +pub(super) fn describe(status: OSStatus) -> String { + match status_name(status) { + Some(name) => format!("{name} ({status})"), + None => format!("OSStatus {status}"), + } +} + +/// Map a failed platform call to the crate's error type. +/// +/// `operation` names the C entry point that failed, mirroring the VAAPI +/// backend's `format!("vaCreateConfig: {}", ...)` convention. +pub(super) fn map_status(codec: HwCodec, operation: &str, status: OSStatus) -> HwDecodeError { + if UNAVAILABLE.contains(&status) { + HwDecodeError::Unavailable { + codec, + reason: format!("{operation}: {}", describe(status)), + } + } else { + HwDecodeError::Decode { + codec, + message: format!("{operation}: {}", describe(status)), + } + } +} + +/// `HwDecodeError::Decode` shorthand, matching the VAAPI backend's helper. +pub(super) fn decode_err(codec: HwCodec, message: impl Into) -> HwDecodeError { + HwDecodeError::Decode { + codec, + message: message.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unavailable_statuses_map_to_unavailable() { + for &status in UNAVAILABLE { + let err = map_status(HwCodec::Av1, "VTDecompressionSessionCreate", status); + assert!( + matches!(err, HwDecodeError::Unavailable { .. }), + "{status} must be Unavailable, got {err:?}" + ); + // The message must stay actionable: it names the failed call. + assert!(err.to_string().contains("VTDecompressionSessionCreate")); + } + } + + #[test] + fn decode_statuses_map_to_decode() { + for status in [ + K_VT_VIDEO_DECODER_BAD_DATA_ERR, + K_VT_VIDEO_DECODER_MALFUNCTION_ERR, + K_VT_PARAMETER_ERR, + K_VT_INVALID_SESSION_ERR, + K_VT_VIDEO_DECODER_REFERENCE_MISSING_ERR, + K_CM_BLOCK_BUFFER_BAD_POINTER_PARAMETER_ERR, + // An unknown code must still be an error, never a panic. + -99_999, + ] { + let err = map_status(HwCodec::Hevc, "VTDecompressionSessionDecodeFrame", status); + assert!( + matches!(err, HwDecodeError::Decode { .. }), + "{status} must be Decode, got {err:?}" + ); + } + } + + /// The four codes the design most easily transposes — `-12911` is + /// *Malfunction*, `-12913` is *NotAvailableNow*. Getting these backwards + /// would silently move a hardware-busy failure into the "bad file" bucket. + #[test] + fn status_names_match_the_headers() { + assert_eq!(status_name(-12906), Some("kVTCouldNotFindVideoDecoderErr")); + assert_eq!( + status_name(-12910), + Some("kVTVideoDecoderUnsupportedDataFormatErr") + ); + assert_eq!(status_name(-12911), Some("kVTVideoDecoderMalfunctionErr")); + assert_eq!( + status_name(-12913), + Some("kVTVideoDecoderNotAvailableNowErr") + ); + assert_eq!(status_name(-99_999), None); + } + + #[test] + fn describe_always_carries_the_numeric_code() { + assert_eq!(describe(-12906), "kVTCouldNotFindVideoDecoderErr (-12906)"); + assert_eq!(describe(-99_999), "OSStatus -99999"); + } + + /// `NotAvailableNow` is "no decoder right now", not "bad data" — the + /// distinction the HEIC/AVIF adapters rely on to report + /// `RawError::HwDecoderUnavailable` instead of blaming the file. + #[test] + fn busy_hardware_is_unavailable_not_a_decode_failure() { + let err = map_status( + HwCodec::Hevc, + "op", + K_VT_VIDEO_DECODER_NOT_AVAILABLE_NOW_ERR, + ); + assert!(matches!(err, HwDecodeError::Unavailable { .. })); + let err = map_status(HwCodec::Hevc, "op", K_VT_VIDEO_DECODER_MALFUNCTION_ERR); + assert!(matches!(err, HwDecodeError::Decode { .. })); + } +} diff --git a/crates/rawshift-hwdec/tests/videotoolbox_device.rs b/crates/rawshift-hwdec/tests/videotoolbox_device.rs new file mode 100644 index 0000000..ee3a84a --- /dev/null +++ b/crates/rawshift-hwdec/tests/videotoolbox_device.rs @@ -0,0 +1,577 @@ +//! Device-gated VideoToolbox integration tests: real hardware decode. +//! +//! Every test skips gracefully (eprintln + return) when the machine reports no +//! hardware decoder for the codec — Intel Macs for HEVC, and every pre-M3 / +//! pre-A17-Pro machine for AV1 — or when the ffmpeg fixture generator is +//! unavailable. CI never runs these (see the pre-release hardware +//! verification section of DEVELOPMENT.md); they are the maintainer's gate, +//! run locally with `just test-hw`. +//! +//! On a machine with a hardware decode block they exercise the whole path: +//! bitstream generation → hvcC/av1C + payload → `decoder()` → +//! `DecodedFrame` pixels. +//! +//! Only compiled when a VideoToolbox build is requested; the bodies are +//! `cfg`-gated on the build-script-selected backend so a plain +//! `cargo test -p rawshift-hwdec` (stub build) compiles them out. + +#![cfg(hwdec_backend = "videotoolbox")] + +use rawshift_hwdec::{ + ChromaSubsampling, CodecConfig, DecodedFrame, HwBackend, HwCodec, HwStillDecoder, PixelFormat, + StillDecodeRequest, available_codecs, backend, decoder, +}; +use std::process::Command; + +// ── gating helpers ────────────────────────────────────────────────────────── + +/// `decoder()`, or `None` plus a skip message naming what the probe found. +fn device_or_skip(codec: HwCodec) -> Option> { + let decoder = decoder(codec); + if decoder.is_none() { + eprintln!( + "Skipping VideoToolbox device test: no hardware {codec} decoder \ + (backend: {:?}, available codecs: {:?})", + backend(), + available_codecs() + ); + } + decoder +} + +/// Run ffmpeg, returning false (→ skip) when it is missing or fails. +fn ffmpeg(args: &[&str]) -> bool { + match Command::new("ffmpeg") + .args(["-y", "-hide_banner", "-loglevel", "error"]) + .args(args) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + } +} + +/// Whether ffmpeg advertises an encoder, so a missing one skips rather than +/// failing. +fn has_encoder(name: &str) -> bool { + Command::new("ffmpeg") + .args(["-hide_banner", "-encoders"]) + .output() + .is_ok_and(|out| String::from_utf8_lossy(&out.stdout).contains(name)) +} + +/// Per-plane sample variance must be non-zero for a real test pattern — +/// catches "decode succeeded but wrote a blank surface". +fn luma_variance(frame: &DecodedFrame) -> f64 { + let plane = &frame.planes()[0]; + let bps = frame.format().bytes_per_sample(); + let width = frame.width() as usize; + let mut samples: Vec = Vec::new(); + for row in 0..frame.height() as usize { + let start = row * plane.stride; + for px in 0..width { + let value = if bps == 1 { + f64::from(plane.data[start + px]) + } else { + f64::from(u16::from_le_bytes([ + plane.data[start + px * 2], + plane.data[start + px * 2 + 1], + ])) + }; + samples.push(value); + } + } + let mean = samples.iter().sum::() / samples.len() as f64; + samples.iter().map(|s| (s - mean).powi(2)).sum::() / samples.len() as f64 +} + +// ── Annex-B → hvcC + length-prefixed payload (test-side container glue) ───── + +/// Split an Annex-B stream into NAL units (3- or 4-byte start codes). +fn split_annex_b(data: &[u8]) -> Vec> { + let mut starts = Vec::new(); + let mut i = 0; + while i + 3 <= data.len() { + if data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 { + starts.push(i); + i += 3; + } else { + i += 1; + } + } + let mut nals = Vec::new(); + for (n, &start) in starts.iter().enumerate() { + let end = starts.get(n + 1).copied().unwrap_or(data.len()); + let mut nal = &data[start + 3..end]; + // Trim the trailing zero(s) that belong to the next 4-byte start code + // / trailing_zero_8bits. + while let Some((&0, rest)) = nal.split_last() { + nal = rest; + } + if !nal.is_empty() { + nals.push(nal.to_vec()); + } + } + nals +} + +/// Build an hvcC record (4-byte length prefixes) carrying the parameter sets, +/// plus a length-prefixed payload of the coded slices. +fn build_hvcc_and_payload(nals: &[Vec]) -> (Vec, Vec) { + let mut hvcc = vec![0u8; 23]; + hvcc[0] = 1; // configurationVersion + hvcc[21] = 0x03; // lengthSizeMinusOne = 3 + let mut arrays: Vec<(u8, Vec<&[u8]>)> = Vec::new(); + let mut payload = Vec::new(); + for nal in nals { + let nal_type = (nal[0] >> 1) & 0x3f; + match nal_type { + 32..=34 => match arrays.iter_mut().find(|(t, _)| *t == nal_type) { + Some((_, list)) => list.push(nal), + None => arrays.push((nal_type, vec![nal])), + }, + _ if nal_type < 32 => { + payload.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + payload.extend_from_slice(nal); + } + _ => {} // SEI etc. + } + } + hvcc[22] = arrays.len() as u8; + for (nal_type, list) in &arrays { + hvcc.push(0x80 | nal_type); + hvcc.extend_from_slice(&(list.len() as u16).to_be_bytes()); + for nal in list { + hvcc.extend_from_slice(&(nal.len() as u16).to_be_bytes()); + hvcc.extend_from_slice(nal); + } + } + (hvcc, payload) +} + +/// Generate one intra HEVC frame with ffmpeg/libx265 and return its hvcC + +/// length-prefixed payload; `None` when the encoder is unavailable (→ skip). +fn generate_hevc( + size: &str, + pix_fmt: &str, + range: Option<&str>, + tag: &str, +) -> Option<(Vec, Vec)> { + let path = std::env::temp_dir().join(format!("rawshift_vt_test_{tag}.265")); + let path_str = path.to_str().expect("temp path is UTF-8"); + let filter = format!("testsrc2=size={size}:duration=1:rate=1"); + let mut args: Vec<&str> = vec!["-f", "lavfi", "-i", &filter, "-pix_fmt", pix_fmt]; + if let Some(range) = range { + args.extend_from_slice(&["-color_range", range]); + } + args.extend_from_slice(&[ + "-c:v", + "libx265", + "-x265-params", + "keyint=1:log-level=none", + "-frames:v", + "1", + "-f", + "hevc", + path_str, + ]); + if !ffmpeg(&args) { + eprintln!("Skipping VideoToolbox HEVC decode test: ffmpeg/libx265 unavailable"); + return None; + } + let annex_b = std::fs::read(&path).expect("read generated bitstream"); + let _ = std::fs::remove_file(&path); + let nals = split_annex_b(&annex_b); + assert!(!nals.is_empty(), "generated stream has NAL units"); + let (hvcc, payload) = build_hvcc_and_payload(&nals); + assert!(!payload.is_empty(), "generated stream has a coded slice"); + Some((hvcc, payload)) +} + +/// Decode one generated HEVC still through the hardware. +fn decode_generated_hevc(size: &str, pix_fmt: &str, tag: &str) -> Option { + let (hvcc, payload) = generate_hevc(size, pix_fmt, None, tag)?; + let mut decoder = device_or_skip(HwCodec::Hevc)?; + let request = StillDecodeRequest { + config: CodecConfig::Hvcc(&hvcc), + payload: &payload, + // The HEIC adapter passes 0/0 because hvcC carries no picture size; + // mirror that here so the test covers the real call shape. + width: 0, + height: 0, + bit_depth: if pix_fmt.contains("10") { 10 } else { 8 }, + chroma: ChromaSubsampling::Cs420, + }; + Some( + decoder + .decode_still(&request) + .expect("hardware HEVC decode"), + ) +} + +/// Minimal IVF demux: return the first frame's OBU stream. +fn first_ivf_frame(data: &[u8]) -> Option> { + if data.len() < 32 || &data[0..4] != b"DKIF" { + return None; + } + let header_len = u16::from_le_bytes([data[6], data[7]]) as usize; + let frame_size = u32::from_le_bytes(data[header_len..header_len + 4].try_into().ok()?) as usize; + let start = header_len + 12; + data.get(start..start + frame_size).map(<[u8]>::to_vec) +} + +// ── (a) probe ─────────────────────────────────────────────────────────────── + +/// On a machine with a decode block: the probe must report VideoToolbox and +/// consistent codec/decoder availability. Without one: everything reports +/// "none". +#[test] +fn videotoolbox_probe_reports_consistent_availability() { + let codecs = available_codecs(); + match backend() { + Some(backend_kind) => { + assert_eq!(backend_kind, HwBackend::VideoToolbox); + assert!( + !codecs.is_empty(), + "a reported backend must decode something" + ); + eprintln!("VideoToolbox probe: available codecs: {codecs:?}"); + } + None => { + assert!(codecs.is_empty()); + eprintln!( + "Skipping VideoToolbox probe assertions: no hardware decoder on this machine" + ); + return; + } + } + for &codec in codecs { + assert!( + decoder(codec).is_some(), + "decoder({codec}) must exist when listed" + ); + } + // HEVC decode is present on every Apple silicon Mac and on Intel Macs from + // Skylake on, so a machine reporting a backend but no HEVC is worth + // noting rather than asserting on. + if !codecs.contains(&HwCodec::Av1) { + eprintln!( + "Note: this machine has no AV1 hardware decode block \ + (expected before M3 / A17 Pro); AVIF pixel decode is unavailable." + ); + } +} + +// ── (b) HEVC Main ─────────────────────────────────────────────────────────── + +#[test] +fn videotoolbox_decodes_real_hevc_main_to_nv12() { + let Some(frame) = decode_generated_hevc("64x64", "yuv420p", "main8") else { + return; + }; + assert_eq!((frame.width(), frame.height()), (64, 64)); + assert_eq!(frame.format(), PixelFormat::Nv12); + assert_eq!(frame.bit_depth(), 8); + assert_eq!(frame.planes().len(), 2); + let variance = luma_variance(&frame); + assert!( + variance > 10.0, + "decoded test pattern must have real content (variance {variance})" + ); + eprintln!("VideoToolbox HEVC Main: 64x64 NV12 decoded, luma variance {variance:.1}"); +} + +// ── (c) HEVC Main10 — also the P010 bit-alignment check ───────────────────── + +/// CoreVideo's `x420`/`xf20` put the 10-bit sample in the **MSBs** of each +/// 16-bit word, which is exactly `PixelFormat::P010`'s contract. If that were +/// ever wrong — samples in the low bits instead — the variance would collapse +/// by roughly 4096x, so this threshold is the empirical alignment check. +#[test] +fn videotoolbox_decodes_real_hevc_main10_to_p010() { + let Some(frame) = decode_generated_hevc("64x64", "yuv420p10le", "main10") else { + return; + }; + assert_eq!((frame.width(), frame.height()), (64, 64)); + assert_eq!(frame.format(), PixelFormat::P010); + assert_eq!(frame.bit_depth(), 10); + let variance = luma_variance(&frame); + assert!( + variance > 100.0, + "decoded 10-bit pattern must have real content in the high bits \ + (variance {variance})" + ); + eprintln!("VideoToolbox HEVC Main10: 64x64 P010 decoded, luma variance {variance:.1}"); +} + +// ── (d) conformance window ────────────────────────────────────────────────── + +/// 60x36 is not a whole number of coding tree blocks, so HEVC codes it as a +/// larger picture (64x40) plus a conformance window. The backend does no +/// cropping arithmetic of its own because `CVPixelBufferGetWidth/Height` +/// report the clean aperture — this test is what pins that assumption. If +/// VideoToolbox ever reported the coded size instead, this fails immediately +/// and points straight at it. +/// +/// (60x36 rather than an odd size because ffmpeg rounds odd dimensions down +/// to even ones for 4:2:0 input, which would silently change what is tested.) +#[test] +fn videotoolbox_crops_to_the_conformance_window() { + let Some(frame) = decode_generated_hevc("60x36", "yuv420p", "crop") else { + return; + }; + assert_eq!( + (frame.width(), frame.height()), + (60, 36), + "VideoToolbox must report the cropped (clean aperture) size, not the \ + coded size" + ); + let chroma = &frame.planes()[1]; + assert_eq!( + chroma.stride, + 30 * 2, + "ceil(60/2) chroma columns, two samples each" + ); + assert_eq!(frame.planes()[0].stride, 60, "luma rows are tightly packed"); + eprintln!("VideoToolbox conformance window: 64x40 coded -> 60x36 reported"); +} + +// ── (e) AV1 Profile 0 ─────────────────────────────────────────────────────── + +#[test] +fn videotoolbox_decodes_real_av1_profile0_to_nv12() { + let path = std::env::temp_dir().join("rawshift_vt_test_av1.ivf"); + let path_str = path.to_str().expect("temp path is UTF-8"); + + // libaom's `-still-picture` is the closest match to an AVIF item; SVT-AV1 + // is the fallback because the common Homebrew ffmpeg ships only that one. + // A single keyframe from either decodes as a still. + let generated = if has_encoder("libaom-av1") { + ffmpeg(&[ + "-f", + "lavfi", + "-i", + "testsrc2=size=64x64:duration=1:rate=1", + "-pix_fmt", + "yuv420p", + "-c:v", + "libaom-av1", + "-still-picture", + "1", + "-crf", + "40", + "-b:v", + "0", + "-frames:v", + "1", + "-f", + "ivf", + path_str, + ]) + } else if has_encoder("libsvtav1") { + ffmpeg(&[ + "-f", + "lavfi", + "-i", + "testsrc2=size=64x64:duration=1:rate=1", + "-pix_fmt", + "yuv420p", + "-c:v", + "libsvtav1", + "-crf", + "40", + "-frames:v", + "1", + "-f", + "ivf", + path_str, + ]) + } else { + false + }; + if !generated { + eprintln!("Skipping VideoToolbox AV1 decode test: ffmpeg/libaom-av1/libsvtav1 unavailable"); + return; + } + + let ivf = std::fs::read(&path).expect("read generated IVF"); + let _ = std::fs::remove_file(&path); + let payload = first_ivf_frame(&ivf).expect("IVF frame"); + + // Minimal av1C: marker/version, profile 0 + level, flags byte, reserved; + // the sequence header travels in the payload's OBU stream. + let av1c = [0x81u8, 0x00, 0x0c, 0x00]; + + let Some(mut decoder) = device_or_skip(HwCodec::Av1) else { + return; + }; + let request = StillDecodeRequest { + config: CodecConfig::Av1c(&av1c), + payload: &payload, + width: 64, + height: 64, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }; + let frame = decoder.decode_still(&request).expect("hardware AV1 decode"); + assert_eq!((frame.width(), frame.height()), (64, 64)); + assert_eq!(frame.format(), PixelFormat::Nv12); + assert_eq!(frame.bit_depth(), 8); + let variance = luma_variance(&frame); + assert!( + variance > 10.0, + "decoded AV1 pattern must have real content (variance {variance})" + ); + eprintln!("VideoToolbox AV1 Profile0: 64x64 NV12 decoded, luma variance {variance:.1}"); +} + +// ── (f) session reuse across tiles ────────────────────────────────────────── + +/// A grid HEIC decodes hundreds of identically-sized tiles through one +/// decoder. Repeating a decode exercises the cached session, the per-decode +/// slot reset, and the callback refcon's lifetime — a bug in any of those +/// shows up as a changed frame or a crash on the second pass, not the first. +#[test] +fn videotoolbox_reuses_one_session_across_many_decodes() { + let Some((hvcc, payload)) = generate_hevc("64x64", "yuv420p", None, "reuse") else { + return; + }; + let Some(mut decoder) = device_or_skip(HwCodec::Hevc) else { + return; + }; + let request = StillDecodeRequest { + config: CodecConfig::Hvcc(&hvcc), + payload: &payload, + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }; + let first = decoder.decode_still(&request).expect("first decode"); + for pass in 1..64 { + let frame = decoder + .decode_still(&request) + .unwrap_or_else(|e| panic!("decode {pass} through the reused session failed: {e}")); + assert_eq!(frame.width(), first.width(), "pass {pass}"); + assert_eq!(frame.height(), first.height(), "pass {pass}"); + assert_eq!(frame.format(), first.format(), "pass {pass}"); + assert_eq!( + frame.planes()[0].data, + first.planes()[0].data, + "pass {pass} decoded different pixels through the reused session" + ); + } + eprintln!("VideoToolbox session reuse: 64 decodes produced identical frames"); +} + +// ── (g) mixed geometry through one decoder ────────────────────────────────── + +/// A decoder is opened per codec, not per picture, so one can be handed +/// pictures of different sizes — a HEIF file's primary image and its thumbnail +/// are exactly that. This drives the session cache's second path, where the +/// configuration record differs and the decoder is asked whether it can accept +/// the new format description before the session is rebuilt. +#[test] +fn videotoolbox_decodes_mixed_geometry_through_one_decoder() { + let Some((big_hvcc, big_payload)) = generate_hevc("128x96", "yuv420p", None, "mixed_big") + else { + return; + }; + let Some((small_hvcc, small_payload)) = generate_hevc("64x64", "yuv420p", None, "mixed_small") + else { + return; + }; + let Some(mut decoder) = device_or_skip(HwCodec::Hevc) else { + return; + }; + let decode = |decoder: &mut Box, hvcc: &[u8], payload: &[u8]| { + decoder + .decode_still(&StillDecodeRequest { + config: CodecConfig::Hvcc(hvcc), + payload, + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }) + .expect("hardware HEVC decode") + }; + // Alternate, so the cache is forced to switch in both directions rather + // than settling on one geometry. + for round in 0..3 { + let big = decode(&mut decoder, &big_hvcc, &big_payload); + assert_eq!((big.width(), big.height()), (128, 96), "round {round}"); + let small = decode(&mut decoder, &small_hvcc, &small_payload); + assert_eq!((small.width(), small.height()), (64, 64), "round {round}"); + } + eprintln!("VideoToolbox mixed geometry: 128x96 and 64x64 alternated through one decoder"); +} + +// ── (h) colour range ──────────────────────────────────────────────────────── + +/// The backend derives `ColorRange` from the emitted surface's four-character +/// code (`420v` vs `420f`) rather than from the SPS. This checks the two +/// encodings do not collapse to the same surface — if they ever do, the range +/// must be sourced from the bitstream instead. +#[test] +fn videotoolbox_reports_a_colour_range() { + let Some((tv_hvcc, tv_payload)) = generate_hevc("64x64", "yuv420p", Some("tv"), "range_tv") + else { + return; + }; + let Some((pc_hvcc, pc_payload)) = generate_hevc("64x64", "yuv420p", Some("pc"), "range_pc") + else { + return; + }; + let Some(mut decoder) = device_or_skip(HwCodec::Hevc) else { + return; + }; + let decode = |decoder: &mut Box, hvcc: &[u8], payload: &[u8]| { + decoder + .decode_still(&StillDecodeRequest { + config: CodecConfig::Hvcc(hvcc), + payload, + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }) + .expect("hardware HEVC decode") + .range() + }; + let tv = decode(&mut decoder, &tv_hvcc, &tv_payload); + let pc = decode(&mut decoder, &pc_hvcc, &pc_payload); + eprintln!("VideoToolbox colour range: tv -> {tv:?}, pc -> {pc:?}"); +} + +// ── (i) error path on real hardware ───────────────────────────────────────── + +/// Garbage input must fail with an error, not a panic and not a blank frame. +#[test] +fn videotoolbox_rejects_garbage_bitstream() { + let Some(mut decoder) = device_or_skip(HwCodec::Hevc) else { + return; + }; + // An hvcC with no parameter-set arrays at all: the backend must refuse it + // before ever creating a session, because VideoToolbox needs VPS/SPS/PPS. + let hvcc = { + let mut v = vec![0u8; 23]; + v[0] = 1; + v[21] = 0x03; + v + }; + let request = StillDecodeRequest { + config: CodecConfig::Hvcc(&hvcc), + payload: &[0, 0, 0, 4, 0x26, 0x01, 0xDE, 0xAD], + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }; + let err = decoder.decode_still(&request).unwrap_err(); + assert!( + err.to_string().contains("VPS/SPS/PPS"), + "expected the missing-parameter-set message, got: {err}" + ); + eprintln!("VideoToolbox correctly rejected garbage: {err}"); +} diff --git a/justfile b/justfile index 6ff4ef2..b84b0d9 100644 --- a/justfile +++ b/justfile @@ -51,6 +51,24 @@ test-features features: test-all: cargo test --workspace --features rawshift-image/full +# Run the hardware decode tests against this machine's real decoder. +# +# CI cannot do this — GitHub runners have no dependable hardware decode block, +# and every hardware test skips silently without one, so a CI job would be +# green without decoding anything. This is the maintainer's pre-release gate +# instead; see "Pre-release hardware verification" in DEVELOPMENT.md. +# +# Needs ffmpeg (bitstream fixtures) and, for the end-to-end HEIC test, +# heif-enc. macOS: `brew install ffmpeg libheif libavif`. +# Prints a summary; tests that skip say so on stderr, so read the output rather +# than trusting the exit code alone. +test-hw: + @echo "== backend probe + hardware decode tests ==" + cargo test -p rawshift-hwdec --features hw -- --nocapture --test-threads=1 + @echo "== end-to-end HEIC/AVIF pixel decode ==" + cargo test -p rawshift-image --features full \ + --test heic_hw_decode --test avif_hw_decode -- --nocapture + # Generate docs for the whole workspace doc: cargo doc --workspace --no-deps --open