From 792bc6d9fe36a7687878002360fddda78eb28f1d Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Wed, 30 Sep 2026 18:11:05 +0200 Subject: [PATCH 1/2] feat: add a hardware video decoder factory, with Media Foundation on Windows HardwareVideoDecoderFactory puts the decoders of the GPU in front of the built-in ones where the platform has them, the way HardwareVideoEncoderFactory does for encoders. DefaultVideoDecoderFactory, and a PeerConnectionFactory without a decoder factory, decode as before. On Windows, H.264 and AV1 are decoded by the Media Foundation decoders of Windows on a Direct3D 11 device, so through DXVA on the GPU. A codec is offered only where the GPU has its DXVA decode profile and Windows has a decoder that uses Direct3D 11 (the AV1 Video Extension for AV1). The decoders run synchronously in low-latency mode; each decoded frame is copied from its texture into a staging texture, read back and converted from NV12 to I420, the size taken from the display aperture. A decoder whose output is not in GPU memory decodes in software, which WebRTC's own decoders do better, so it gives up then, as on any failure. FallbackVideoDecoder switches to the software decoder when the hardware one fails to configure or gives up while decoding; the software decoder starts with the frame at hand, and the receiver asks for a key frame if it cannot decode it. The hardware decoders offer the formats the software decoders offer, so negotiation does not change. Linux has no hardware decoders yet; on macOS both factories use VideoToolbox. --- docs/guide/advanced/video-codecs.md | 23 +- webrtc-jni/src/main/cpp/CMakeLists.txt | 2 +- .../include/JNI_HardwareVideoDecoderFactory.h | 36 ++ .../video/codec/DefaultVideoCodecFactories.h | 5 + .../media/video/codec/FallbackVideoDecoder.h | 71 +++ .../video/codec/HardwareVideoDecoderFactory.h | 57 ++ .../video/codec/VideoDecoderFactoryWrapper.h | 3 + .../video/codec/windows/MFDecoderUtils.h | 55 ++ .../video/codec/windows/MFVideoDecoder.h | 113 ++++ .../codec/windows/MFVideoDecoderFactory.h | 55 ++ .../src/JNI_HardwareVideoDecoderFactory.cpp | 35 ++ .../codec/DefaultVideoCodecFactories.cpp | 16 + .../video/codec/FallbackVideoDecoder.cpp | 114 ++++ .../codec/HardwareVideoDecoderFactory.cpp | 94 ++++ .../codec/VideoDecoderFactoryWrapper.cpp | 9 +- ...p => LinuxHardwareVideoCodecFactories.cpp} | 7 + .../video/codec/windows/MFDecoderUtils.cpp | 134 +++++ .../video/codec/windows/MFVideoDecoder.cpp | 523 ++++++++++++++++++ .../codec/windows/MFVideoDecoderFactory.cpp | 128 +++++ ...=> WindowsHardwareVideoCodecFactories.cpp} | 13 + .../codec/DefaultVideoDecoderFactory.java | 5 +- .../codec/HardwareVideoDecoderFactory.java | 106 ++++ .../media/video/codec/NativeVideoDecoder.java | 14 +- .../HardwareVideoDecoderIntegrationTest.java | 164 ++++++ .../java/dev/onvoid/webrtc/TestMediaCall.java | 23 + 25 files changed, 1795 insertions(+), 10 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/JNI_HardwareVideoDecoderFactory.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/FallbackVideoDecoder.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFDecoderUtils.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h create mode 100644 webrtc-jni/src/main/cpp/src/JNI_HardwareVideoDecoderFactory.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/FallbackVideoDecoder.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp rename webrtc-jni/src/main/cpp/src/media/video/codec/linux/{LinuxHardwareVideoEncoderFactories.cpp => LinuxHardwareVideoCodecFactories.cpp} (85%) create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFDecoderUtils.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp rename webrtc-jni/src/main/cpp/src/media/video/codec/windows/{WindowsHardwareVideoEncoderFactories.cpp => WindowsHardwareVideoCodecFactories.cpp} (76%) create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 8c72898d..7aad255b 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -38,6 +38,27 @@ The hardware encoders take over H.264 Constrained Baseline and Baseline with pac Which encoder a stream uses shows in the `encoderImplementation` statistic of its `outbound-rtp` stats, e.g. `NVENC (NVIDIA GeForce RTX 4070)`, `MediaFoundation (AMDav1Encoder)`, `VA-API (Intel iHD driver ...)`, `OpenH264` or `libaom`. +### Hardware Decoding + +`HardwareVideoDecoderFactory` does the same for decoding: + +```java +PeerConnectionFactory factory = PeerConnectionFactory.builder() + .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .setVideoDecoderFactory(new HardwareVideoDecoderFactory()) + .build(); +``` + +| Platform | Hardware decoding | +|---|---| +| Windows | H.264 and AV1, on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | +| Linux | Not yet; decoding is in software | +| macOS | VideoToolbox, as with `DefaultVideoDecoderFactory` | + +AV1 on Windows needs the *AV1 Video Extension*, which Windows 11 includes. Decoded frames are copied from GPU memory back to system memory, where WebRTC's frames are, so hardware decoding pays off mostly at high resolutions and with many streams; at low resolutions WebRTC's software decoders are about as cheap. A hardware decoder that fails, or turns out to decode in software, is replaced by the software decoder, which starts with the next key frame. + +Which decoder a stream uses shows in the `decoderImplementation` statistic of its `inbound-rtp` stats, e.g. `MediaFoundation (Microsoft H264 Video Decoder MFT)` or `MediaFoundation (AV1VideoExtension)`. + ### Native Codecs The encoders and decoders these factories create are `NativeVideoEncoder`s and `NativeVideoDecoder`s. They are placeholders that make WebRTC create the built-in codec, which then runs entirely inside WebRTC, so their methods are not to be called from Java. @@ -189,7 +210,7 @@ Whatever an encoder, decoder or factory method throws is logged and treated as ` - `VideoEncoderFactory`, `VideoDecoderFactory` — create the codecs of a factory. - `VideoEncoder`, `VideoDecoder`, `EncodedImage` — codecs implemented in Java. - `DefaultVideoEncoderFactory`, `DefaultVideoDecoderFactory` — the built-in codecs. -- `HardwareVideoEncoderFactory` — the built-in encoders, on the GPU where possible. +- `HardwareVideoEncoderFactory`, `HardwareVideoDecoderFactory` — the built-in codecs, on the GPU where possible. - `RTCRtpTransceiver.setCodecPreferences()` — chooses among the negotiated codecs. For the full API, see the JavaDoc of the `dev.onvoid.webrtc.media.video.codec` package. diff --git a/webrtc-jni/src/main/cpp/CMakeLists.txt b/webrtc-jni/src/main/cpp/CMakeLists.txt index ce28b20c..e23cb2ed 100644 --- a/webrtc-jni/src/main/cpp/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/CMakeLists.txt @@ -135,7 +135,7 @@ elseif(LINUX) target_link_libraries(${PROJECT_NAME} ${CXX_LIBS}) target_link_libraries(${PROJECT_NAME} dl) elseif(WIN32) - target_link_libraries(${PROJECT_NAME} dwmapi.lib mf.lib mfreadwrite.lib mfplat.lib mfuuid.lib shcore.lib) + target_link_libraries(${PROJECT_NAME} d3d11.lib dwmapi.lib dxguid.lib mf.lib mfreadwrite.lib mfplat.lib mfuuid.lib shcore.lib) endif() install(TARGETS ${PROJECT_NAME} diff --git a/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoDecoderFactory.h new file mode 100644 index 00000000..00a27e44 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoDecoderFactory.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory +#define _Included_dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory + * Method: getSupportedCodecsInternal + * Signature: ()[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; + */ + JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory_getSupportedCodecsInternal + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h index 5a2fa4b0..67a7cacf 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h @@ -34,6 +34,11 @@ namespace jni // macOS the default encoders already use VideoToolbox. std::unique_ptr CreateHardwareVideoEncoderFactory(); std::unique_ptr CreateDefaultVideoDecoderFactory(); + + // The default decoders, with the decoders of the GPU in front of them + // where the platform has them, falling back to the software ones. On + // macOS the default decoders already use VideoToolbox. + std::unique_ptr CreateHardwareVideoDecoderFactory(); } #endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/FallbackVideoDecoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/FallbackVideoDecoder.h new file mode 100644 index 00000000..a7be6c41 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/FallbackVideoDecoder.h @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_FALLBACK_VIDEO_DECODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_FALLBACK_VIDEO_DECODER_H_ + +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_decoder.h" + +#include +#include + +namespace jni +{ + // Decodes with a hardware decoder, and switches to a software decoder of + // the same codec when the hardware one fails to configure, or gives up + // while decoding by returning WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE. The + // switch is for good. The software decoder starts with the frame the + // hardware one gave up on; if that is no key frame, it fails to decode + // it, and the receiver asks the sender for a key frame. + // + // WebRTC has such a wrapper too, but it is not part of the WebRTC + // library this library links. + class FallbackVideoDecoder : public webrtc::VideoDecoder + { + public: + FallbackVideoDecoder(std::unique_ptr hardware, + std::unique_ptr software); + ~FallbackVideoDecoder() override = default; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + private: + // Configures the software decoder with the settings the hardware + // one was given. Returns whether it is ready. + bool StartSoftware(); + + webrtc::VideoDecoder * Active() const; + + private: + const std::unique_ptr hardware; + const std::unique_ptr software; + + bool useSoftware; + + std::optional settings; + webrtc::DecodedImageCallback * callback; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h new file mode 100644 index 00000000..cb650b30 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_HARDWARE_VIDEO_DECODER_FACTORY_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_HARDWARE_VIDEO_DECODER_FACTORY_H_ + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" + +#include +#include + +namespace jni +{ + // Puts hardware decoders in front of software ones, the way + // HardwareVideoEncoderFactory does with encoders: a codec is decoded by + // the first hardware factory that has it, falling back to the next and + // finally to the software decoder. + class HardwareVideoDecoderFactory : public webrtc::VideoDecoderFactory + { + public: + // The hardware factories are in order of preference. + HardwareVideoDecoderFactory(std::vector> hardware, + std::unique_ptr software); + ~HardwareVideoDecoderFactory() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + const std::vector> hardware; + const std::unique_ptr software; + }; + + // Returns the factories for the hardware decoders of the platform that are + // available, in order of preference; none if the platform has none this + // library supports, or no device that can decode. Defined per platform. + std::vector> CreatePlatformHardwareVideoDecoderFactories(); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h index 7ce079e0..3d507d85 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h @@ -59,12 +59,15 @@ namespace jni jclass nativeDecoderClass; jfieldID nativeDecoderCodecInfo; + jfieldID nativeDecoderHardwareAcceleration; }; private: const JavaGlobalRef factory; const std::shared_ptr javaClass; + // The built-in decoders, and those with the GPU's in front. const std::unique_ptr defaultFactory; + const std::unique_ptr hardwareFactory; std::vector supportedFormats; }; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFDecoderUtils.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFDecoderUtils.h new file mode 100644 index 00000000..ea2d3390 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFDecoderUtils.h @@ -0,0 +1,55 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_DECODER_UTILS_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_DECODER_UTILS_H_ + +#include +#include +#include +#include +#include + +#include +#include + +namespace jni +{ + // Creates a Direct3D 11 device on the default adapter that video can be + // decoded on, protected for use from several threads, as a decoder + // transform and the thread reading its output both use it. + HRESULT CreateVideoDevice(Microsoft::WRL::ComPtr & device, + Microsoft::WRL::ComPtr & context); + + // Whether the GPU of the device decodes the given DXVA profile, such as + // D3D11_DECODER_PROFILE_H264_VLD_NOFGT, in hardware. + bool SupportsDecoderProfile(ID3D11Device * device, const GUID & profile); + + // Lists the decoder transforms for the given video format, such as + // MFVideoFormat_H264, best first. These are the synchronous transforms of + // Windows, which decode on the GPU through DXVA when given a Direct3D + // device. Media Foundation has to be started. + HRESULT EnumerateDecoders(const GUID & format, std::vector> & decoders); + + // Activates the first of the decoders that can decode on a Direct3D 11 + // device, and gives it the device manager. The activation object of the + // transform is returned with it, to shut it down with. + HRESULT ActivateDirect3DDecoder(const std::vector> & decoders, + IMFDXGIDeviceManager * manager, Microsoft::WRL::ComPtr & activate, + Microsoft::WRL::ComPtr & transform); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h new file mode 100644 index 00000000..55ae783b --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h @@ -0,0 +1,113 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_VIDEO_DECODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_VIDEO_DECODER_H_ + +#include "platform/windows/MFInitializer.h" + +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_rotation.h" +#include "api/video_codecs/video_decoder.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace jni +{ + // Decodes H.264 or AV1 on the GPU, with the decoder transform of Windows + // for the codec, which decodes through DXVA on the Direct3D 11 device it + // is given. + // + // Decoding is synchronous, and in low-latency mode each frame comes out + // of the transform as soon as it went in. Decoded frames are in GPU + // memory; each one is copied to system memory and converted to I420, + // which is what WebRTC's frames hold. A transform whose output is not in + // GPU memory decodes in software, which WebRTC's own decoders do better, + // so the decoder gives up then, as it does when anything fails, and the + // software decoder takes over. + class MFVideoDecoder : public webrtc::VideoDecoder + { + public: + // The codec is H.264 or AV1. + explicit MFVideoDecoder(webrtc::VideoCodecType codec); + ~MFVideoDecoder() override; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + private: + // What becomes of an encoded frame once it is decoded. + struct PendingFrame + { + uint32_t rtpTimestamp; + int64_t ntpTimeMs; + int64_t renderTimeMs; + webrtc::VideoRotation rotation; + }; + + HRESULT CreateTransform(const Settings & settings); + HRESULT SetOutputType(); + bool DrainOutput(); + bool DeliverOutput(IMFSample * sample); + bool ReadFrame(IMFMediaBuffer * buffer, webrtc::scoped_refptr & frame); + + private: + const webrtc::VideoCodecType codec; + std::string implementationName; + + std::unique_ptr mfInitializer; + + Microsoft::WRL::ComPtr device; + Microsoft::WRL::ComPtr context; + Microsoft::WRL::ComPtr deviceManager; + Microsoft::WRL::ComPtr activate; + Microsoft::WRL::ComPtr transform; + DWORD inputStreamId; + DWORD outputStreamId; + + // The texture decoded frames are copied into, to read them. + Microsoft::WRL::ComPtr staging; + D3D11_TEXTURE2D_DESC stagingDesc; + + // The size of the picture, within the possibly larger frame. + uint32_t width; + uint32_t height; + + webrtc::DecodedImageCallback * callback; + + LONGLONG sampleTime; + std::map pendingFrames; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h new file mode 100644 index 00000000..5dfe2640 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h @@ -0,0 +1,55 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_VIDEO_DECODER_FACTORY_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_VIDEO_DECODER_FACTORY_H_ + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" + +#include +#include + +namespace jni +{ + // Creates the Media Foundation decoders that decode on the GPU, for H.264 + // and AV1, whichever the GPU decodes and Windows has a decoder for that + // uses Direct3D 11. It offers them in the formats WebRTC's software + // decoders offer too: H.264 in all its profiles, and AV1 in profile 0. + class MFVideoDecoderFactory : public webrtc::VideoDecoderFactory + { + public: + // Returns a factory, or null if the GPU decodes neither codec. + static std::unique_ptr Create(); + + ~MFVideoDecoderFactory() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + MFVideoDecoderFactory(bool h264, bool av1); + + private: + const bool h264; + const bool av1; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoDecoderFactory.cpp new file mode 100644 index 00000000..251131e1 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoDecoderFactory.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_HardwareVideoDecoderFactory.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "JavaUtils.h" + +JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory_getSupportedCodecsInternal +(JNIEnv * env, jclass caller) +{ + try { + auto factory = jni::CreateHardwareVideoDecoderFactory(); + + return jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()).release(); + } + catch (...) { + ThrowCxxJavaException(env); + } + + return nullptr; +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp index 686fa58a..6a82dcca 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp @@ -16,6 +16,7 @@ #include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/HardwareVideoDecoderFactory.h" #include "media/video/codec/HardwareVideoEncoderFactory.h" #ifdef __APPLE__ @@ -76,6 +77,21 @@ namespace jni webrtc::LibvpxVp9DecoderTemplateAdapter, webrtc::OpenH264DecoderTemplateAdapter, webrtc::Dav1dDecoderTemplateAdapter>>(); +#endif + } + + std::unique_ptr CreateHardwareVideoDecoderFactory() + { +#ifdef __APPLE__ + return CreateDefaultVideoDecoderFactory(); +#else + std::vector> hardware = CreatePlatformHardwareVideoDecoderFactories(); + + if (hardware.empty()) { + return CreateDefaultVideoDecoderFactory(); + } + + return std::make_unique(std::move(hardware), CreateDefaultVideoDecoderFactory()); #endif } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/FallbackVideoDecoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/FallbackVideoDecoder.cpp new file mode 100644 index 00000000..885ae5f6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/FallbackVideoDecoder.cpp @@ -0,0 +1,114 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/FallbackVideoDecoder.h" + +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace jni +{ + FallbackVideoDecoder::FallbackVideoDecoder(std::unique_ptr hardware, + std::unique_ptr software) : + hardware(std::move(hardware)), + software(std::move(software)), + useSoftware(false), + callback(nullptr) + { + } + + bool FallbackVideoDecoder::Configure(const Settings & decoderSettings) + { + settings = decoderSettings; + + if (!useSoftware) { + if (hardware->Configure(decoderSettings)) { + return true; + } + + RTC_LOG(LS_WARNING) << "Hardware decoder failed to configure, switching to software"; + + hardware->Release(); + useSoftware = true; + } + + return StartSoftware(); + } + + bool FallbackVideoDecoder::StartSoftware() + { + if (!settings || !software->Configure(*settings)) { + RTC_LOG(LS_WARNING) << "Software decoder failed to configure"; + return false; + } + + if (callback != nullptr) { + software->RegisterDecodeCompleteCallback(callback); + } + + return true; + } + + int32_t FallbackVideoDecoder::Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + int32_t result = Active()->Decode(image, renderTimeMs); + + if (useSoftware || result != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) { + return result; + } + + RTC_LOG(LS_WARNING) << "Hardware decoder gave up, switching to software"; + + hardware->Release(); + useSoftware = true; + + if (!StartSoftware()) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + return software->Decode(image, renderTimeMs); + } + + int32_t FallbackVideoDecoder::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + hardware->RegisterDecodeCompleteCallback(decodeCallback); + software->RegisterDecodeCompleteCallback(decodeCallback); + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t FallbackVideoDecoder::Release() + { + return Active()->Release(); + } + + webrtc::VideoDecoder::DecoderInfo FallbackVideoDecoder::GetDecoderInfo() const + { + return Active()->GetDecoderInfo(); + } + + const char * FallbackVideoDecoder::ImplementationName() const + { + return Active()->ImplementationName(); + } + + webrtc::VideoDecoder * FallbackVideoDecoder::Active() const + { + return useSoftware ? software.get() : hardware.get(); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp new file mode 100644 index 00000000..64f2f0ea --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/HardwareVideoDecoderFactory.h" +#include "media/video/codec/FallbackVideoDecoder.h" + +#include + +namespace jni +{ + namespace + { + bool Supports(const webrtc::VideoDecoderFactory & factory, const webrtc::SdpVideoFormat & format) + { + const std::vector formats = factory.GetSupportedFormats(); + + return std::any_of(formats.begin(), formats.end(), [&](const webrtc::SdpVideoFormat & supported) { + return supported.IsSameCodec(format); + }); + } + } + + HardwareVideoDecoderFactory::HardwareVideoDecoderFactory( + std::vector> hardware, + std::unique_ptr software) : + hardware(std::move(hardware)), + software(std::move(software)) + { + } + + std::vector HardwareVideoDecoderFactory::GetSupportedFormats() const + { + // The software formats keep their order, so that hardware support + // does not change what a peer connection negotiates. + std::vector formats = software->GetSupportedFormats(); + + for (const auto & factory : hardware) { + for (const webrtc::SdpVideoFormat & format : factory->GetSupportedFormats()) { + bool listed = std::any_of(formats.begin(), formats.end(), [&](const webrtc::SdpVideoFormat & other) { + return other.IsSameCodec(format); + }); + + if (!listed) { + formats.push_back(format); + } + } + } + + return formats; + } + + std::unique_ptr HardwareVideoDecoderFactory::Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) + { + std::unique_ptr decoder; + + if (Supports(*software, format)) { + decoder = software->Create(env, format); + } + + // Built from the back, so that the most preferred decoder comes first + // and each one falls back to the chain behind it. + for (auto factory = hardware.rbegin(); factory != hardware.rend(); ++factory) { + if (!Supports(**factory, format)) { + continue; + } + + std::unique_ptr hardwareDecoder = (*factory)->Create(env, format); + + if (!hardwareDecoder) { + continue; + } + + decoder = decoder + ? std::make_unique(std::move(hardwareDecoder), std::move(decoder)) + : std::move(hardwareDecoder); + } + + return decoder; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp index 9347c227..cbb9211c 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp @@ -31,7 +31,8 @@ namespace jni VideoDecoderFactoryWrapper::VideoDecoderFactoryWrapper(JNIEnv * env, jobject factory) : factory(env, factory), javaClass(JavaClasses::get(env)), - defaultFactory(CreateDefaultVideoDecoderFactory()) + defaultFactory(CreateDefaultVideoDecoderFactory()), + hardwareFactory(CreateHardwareVideoDecoderFactory()) { JavaLocalRef codecs(env, env->CallObjectMethod(factory, javaClass->getSupportedCodecs)); @@ -69,7 +70,10 @@ namespace jni if (env->IsInstanceOf(decoder, javaClass->nativeDecoderClass)) { JavaLocalRef nativeInfo(env, env->GetObjectField(decoder, javaClass->nativeDecoderCodecInfo)); - return defaultFactory->Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); + const bool hardware = env->GetBooleanField(decoder, javaClass->nativeDecoderHardwareAcceleration) == JNI_TRUE; + webrtc::VideoDecoderFactory & builtIn = hardware ? *hardwareFactory : *defaultFactory; + + return builtIn.Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); } return std::make_unique(env, decoder); @@ -93,5 +97,6 @@ namespace jni nativeDecoderClass = FindClass(env, PKG_CODEC"NativeVideoDecoder"); nativeDecoderCodecInfo = GetFieldID(env, nativeDecoderClass, "codecInfo", "L" PKG_CODEC "VideoCodecInfo;"); + nativeDecoderHardwareAcceleration = GetFieldID(env, nativeDecoderClass, "hardwareAcceleration", "Z"); } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoEncoderFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp similarity index 85% rename from webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoEncoderFactories.cpp rename to webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp index b253de85..6eaab4c1 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoEncoderFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "media/video/codec/HardwareVideoDecoderFactory.h" #include "media/video/codec/HardwareVideoEncoderFactory.h" #include "media/video/codec/linux/VaapiVideoEncoderFactory.h" #include "media/video/codec/nvenc/NvencVideoEncoderFactory.h" @@ -35,4 +36,10 @@ namespace jni return factories; } + + std::vector> CreatePlatformHardwareVideoDecoderFactories() + { + // No hardware decoders on Linux yet. + return {}; + } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFDecoderUtils.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFDecoderUtils.cpp new file mode 100644 index 00000000..731bbb4a --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFDecoderUtils.cpp @@ -0,0 +1,134 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/windows/MFDecoderUtils.h" + +#include +#include + +using Microsoft::WRL::ComPtr; + +namespace jni +{ + HRESULT CreateVideoDevice(ComPtr & device, ComPtr & context) + { + const D3D_FEATURE_LEVEL levels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0 + }; + + HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT, levels, ARRAYSIZE(levels), D3D11_SDK_VERSION, + &device, nullptr, &context); + + if (FAILED(hr)) { + return hr; + } + + ComPtr multithread; + hr = device.As(&multithread); + + if (SUCCEEDED(hr)) { + multithread->SetMultithreadProtected(TRUE); + } + + return hr; + } + + bool SupportsDecoderProfile(ID3D11Device * device, const GUID & profile) + { + ComPtr videoDevice; + + if (FAILED(device->QueryInterface(IID_PPV_ARGS(&videoDevice)))) { + return false; + } + + const UINT count = videoDevice->GetVideoDecoderProfileCount(); + + for (UINT i = 0; i < count; i++) { + GUID supported; + + if (SUCCEEDED(videoDevice->GetVideoDecoderProfile(i, &supported)) && IsEqualGUID(supported, profile)) { + BOOL nv12 = FALSE; + + // Decoded into NV12, the format that is read back. + return SUCCEEDED(videoDevice->CheckVideoDecoderFormat(&profile, DXGI_FORMAT_NV12, &nv12)) && nv12; + } + } + + return false; + } + + HRESULT EnumerateDecoders(const GUID & format, std::vector> & decoders) + { + MFT_REGISTER_TYPE_INFO input = { MFMediaType_Video, format }; + + IMFActivate ** activates = nullptr; + UINT32 count = 0; + + HRESULT hr = MFTEnumEx(MFT_CATEGORY_VIDEO_DECODER, + MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_LOCALMFT | MFT_ENUM_FLAG_SORTANDFILTER, + &input, nullptr, &activates, &count); + + if (FAILED(hr)) { + return hr; + } + + for (UINT32 i = 0; i < count; i++) { + ComPtr activate; + activate.Attach(activates[i]); + + decoders.push_back(activate); + } + + CoTaskMemFree(activates); + + return S_OK; + } + + HRESULT ActivateDirect3DDecoder(const std::vector> & decoders, IMFDXGIDeviceManager * manager, + ComPtr & activate, ComPtr & transform) + { + for (const ComPtr & candidate : decoders) { + ComPtr candidateTransform; + + if (FAILED(candidate->ActivateObject(IID_PPV_ARGS(&candidateTransform)))) { + continue; + } + + ComPtr attributes; + UINT32 aware = FALSE; + + if (SUCCEEDED(candidateTransform->GetAttributes(&attributes)) && + SUCCEEDED(attributes->GetUINT32(MF_SA_D3D11_AWARE, &aware)) && aware && + SUCCEEDED(candidateTransform->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, + reinterpret_cast(manager)))) + { + activate = candidate; + transform = candidateTransform; + + return S_OK; + } + + candidateTransform.Reset(); + candidate->ShutdownObject(); + } + + return MF_E_TOPO_CODEC_NOT_FOUND; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp new file mode 100644 index 00000000..2bbd88fc --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp @@ -0,0 +1,523 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/windows/MFVideoDecoder.h" +#include "media/video/codec/windows/MFDecoderUtils.h" +#include "media/video/codec/windows/MFEncoderUtils.h" +#include "platform/windows/ComInitializer.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +#include + +#include + +using Microsoft::WRL::ComPtr; + +namespace jni +{ + namespace + { + // Initializes COM for the calling thread, once. Media Foundation + // transforms are free-threaded, so the apartment a thread already + // has does as well. + bool EnsureComInitialized() + { + thread_local std::unique_ptr initializer; + + if (!initializer) { + try { + initializer = std::make_unique(); + } + catch (...) { + return false; + } + } + + return true; + } + + const GUID & InputFormat(webrtc::VideoCodecType codec) + { + return codec == webrtc::kVideoCodecAV1 ? MFVideoFormat_AV1 : MFVideoFormat_H264; + } + } + + MFVideoDecoder::MFVideoDecoder(webrtc::VideoCodecType codec) : + codec(codec), + implementationName("MediaFoundation"), + inputStreamId(0), + outputStreamId(0), + stagingDesc(), + width(0), + height(0), + callback(nullptr), + sampleTime(0) + { + } + + MFVideoDecoder::~MFVideoDecoder() + { + Release(); + } + + bool MFVideoDecoder::Configure(const Settings & settings) + { + Release(); + + if (!EnsureComInitialized()) { + return false; + } + + try { + mfInitializer = std::make_unique(); + } + catch (...) { + return false; + } + + HRESULT hr = CreateTransform(settings); + + if (FAILED(hr)) { + RTC_LOG(LS_WARNING) << "Media Foundation decoder failed to configure, hr=" << hr; + + Release(); + + return false; + } + + RTC_LOG(LS_INFO) << implementationName << " configured"; + + return true; + } + + HRESULT MFVideoDecoder::CreateTransform(const Settings & settings) + { + HRESULT hr = CreateVideoDevice(device, context); + + if (FAILED(hr)) { + return hr; + } + + UINT resetToken = 0; + hr = MFCreateDXGIDeviceManager(&resetToken, &deviceManager); + + if (SUCCEEDED(hr)) { + hr = deviceManager->ResetDevice(device.Get(), resetToken); + } + if (FAILED(hr)) { + return hr; + } + + std::vector> decoders; + hr = EnumerateDecoders(InputFormat(codec), decoders); + + if (SUCCEEDED(hr)) { + hr = ActivateDirect3DDecoder(decoders, deviceManager.Get(), activate, transform); + } + if (FAILED(hr)) { + return hr; + } + + implementationName = "MediaFoundation (" + GetTransformName(activate.Get()) + ")"; + + ComPtr attributes; + + if (SUCCEEDED(transform->GetAttributes(&attributes))) { + // Each frame out as soon as it is in, rather than after a few + // frames that could reorder it. + attributes->SetUINT32(MF_LOW_LATENCY, TRUE); + } + + hr = transform->GetStreamIDs(1, &inputStreamId, 1, &outputStreamId); + + if (hr == E_NOTIMPL) { + inputStreamId = 0; + outputStreamId = 0; + } + + ComPtr inputType; + hr = MFCreateMediaType(&inputType); + + if (FAILED(hr)) { + return hr; + } + + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + inputType->SetGUID(MF_MT_SUBTYPE, InputFormat(codec)); + inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + + webrtc::RenderResolution resolution = settings.max_render_resolution(); + + if (resolution.Valid()) { + MFSetAttributeSize(inputType.Get(), MF_MT_FRAME_SIZE, resolution.Width(), resolution.Height()); + } + + hr = transform->SetInputType(inputStreamId, inputType.Get(), 0); + + if (SUCCEEDED(hr)) { + hr = SetOutputType(); + } + if (SUCCEEDED(hr)) { + hr = transform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0); + } + if (SUCCEEDED(hr)) { + hr = transform->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); + } + + return hr; + } + + HRESULT MFVideoDecoder::SetOutputType() + { + for (DWORD index = 0;; index++) { + ComPtr type; + HRESULT hr = transform->GetOutputAvailableType(outputStreamId, index, &type); + + if (FAILED(hr)) { + // No NV12 among the types, or no more types. + return hr == MF_E_NO_MORE_TYPES ? MF_E_INVALIDMEDIATYPE : hr; + } + + GUID subtype = GUID_NULL; + + if (FAILED(type->GetGUID(MF_MT_SUBTYPE, &subtype)) || !IsEqualGUID(subtype, MFVideoFormat_NV12)) { + continue; + } + + hr = transform->SetOutputType(outputStreamId, type.Get(), 0); + + if (FAILED(hr)) { + return hr; + } + + UINT32 frameWidth = 0; + UINT32 frameHeight = 0; + MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &frameWidth, &frameHeight); + + // The frame may be larger than the picture, e.g. 1088 rows for + // 1080; the aperture says what is picture. + MFVideoArea aperture = {}; + UINT32 size = 0; + + if (SUCCEEDED(type->GetBlob(MF_MT_MINIMUM_DISPLAY_APERTURE, reinterpret_cast(&aperture), + sizeof(aperture), &size)) && size == sizeof(aperture) && aperture.Area.cx > 0 && aperture.Area.cy > 0) + { + width = static_cast(aperture.Area.cx); + height = static_cast(aperture.Area.cy); + } + else { + width = frameWidth; + height = frameHeight; + } + + return S_OK; + } + } + + int32_t MFVideoDecoder::Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + if (!transform || callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (image.size() == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + ComPtr buffer; + HRESULT hr = MFCreateMemoryBuffer(static_cast(image.size()), &buffer); + + BYTE * data = nullptr; + + if (SUCCEEDED(hr)) { + hr = buffer->Lock(&data, nullptr, nullptr); + } + if (FAILED(hr)) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + std::memcpy(data, image.data(), image.size()); + + buffer->Unlock(); + buffer->SetCurrentLength(static_cast(image.size())); + + ComPtr sample; + hr = MFCreateSample(&sample); + + if (SUCCEEDED(hr)) { + hr = sample->AddBuffer(buffer.Get()); + } + if (FAILED(hr)) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // The sample time identifies the frame when it comes out decoded. + sampleTime += 10000; + sample->SetSampleTime(sampleTime); + + if (image.frame_type() == webrtc::VideoFrameType::kVideoFrameKey) { + sample->SetUINT32(MFSampleExtension_CleanPoint, TRUE); + } + + pendingFrames[sampleTime] = PendingFrame { + image.RtpTimestamp(), + image.ntp_time_ms_, + renderTimeMs, + image.rotation_ + }; + + hr = transform->ProcessInput(inputStreamId, sample.Get(), 0); + + if (hr == MF_E_NOTACCEPTING) { + // Output that is still waiting has to be taken first. + if (!DrainOutput()) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + hr = transform->ProcessInput(inputStreamId, sample.Get(), 0); + } + + if (FAILED(hr)) { + RTC_LOG(LS_WARNING) << implementationName << " rejected a frame, hr=" << hr; + pendingFrames.erase(sampleTime); + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + return DrainOutput() ? WEBRTC_VIDEO_CODEC_OK : WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + bool MFVideoDecoder::DrainOutput() + { + for (;;) { + MFT_OUTPUT_STREAM_INFO info = {}; + + if (FAILED(transform->GetOutputStreamInfo(outputStreamId, &info))) { + return false; + } + + ComPtr ownSample; + + if (!(info.dwFlags & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES))) { + ComPtr buffer; + + if (FAILED(MFCreateSample(&ownSample)) || FAILED(MFCreateMemoryBuffer(info.cbSize, &buffer)) || + FAILED(ownSample->AddBuffer(buffer.Get()))) + { + return false; + } + } + + MFT_OUTPUT_DATA_BUFFER output = {}; + output.dwStreamID = outputStreamId; + output.pSample = ownSample.Get(); + + DWORD status = 0; + HRESULT hr = transform->ProcessOutput(0, 1, &output, &status); + + if (output.pEvents != nullptr) { + output.pEvents->Release(); + } + + // A sample the transform provided is ours to release. + ComPtr sample; + + if (ownSample) { + sample = ownSample; + } + else if (output.pSample != nullptr) { + sample.Attach(output.pSample); + } + + if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) { + return true; + } + if (hr == MF_E_TRANSFORM_STREAM_CHANGE) { + // The stream settled its size; the output type has to be set + // again before frames come out. + if (FAILED(SetOutputType())) { + return false; + } + + staging.Reset(); + continue; + } + if (FAILED(hr) || !sample) { + RTC_LOG(LS_WARNING) << implementationName << " failed to produce output, hr=" << hr; + return false; + } + + if (!DeliverOutput(sample.Get())) { + return false; + } + } + } + + bool MFVideoDecoder::DeliverOutput(IMFSample * sample) + { + LONGLONG time = 0; + sample->GetSampleTime(&time); + + auto found = pendingFrames.find(time); + + if (found == pendingFrames.end()) { + RTC_LOG(LS_WARNING) << implementationName << " produced a frame for no input, time " << time; + return true; + } + + const PendingFrame pending = found->second; + + // Frames the decoder skipped come out never. + pendingFrames.erase(pendingFrames.begin(), std::next(found)); + + ComPtr buffer; + + if (FAILED(sample->GetBufferByIndex(0, &buffer))) { + return false; + } + + webrtc::scoped_refptr i420; + + if (!ReadFrame(buffer.Get(), i420)) { + return false; + } + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420) + .set_rtp_timestamp(pending.rtpTimestamp) + .set_timestamp_ms(pending.renderTimeMs) + .set_ntp_time_ms(pending.ntpTimeMs) + .set_rotation(pending.rotation) + .build(); + + callback->Decoded(frame, std::nullopt, std::nullopt); + + return true; + } + + bool MFVideoDecoder::ReadFrame(IMFMediaBuffer * buffer, webrtc::scoped_refptr & frame) + { + ComPtr dxgiBuffer; + + if (FAILED(buffer->QueryInterface(IID_PPV_ARGS(&dxgiBuffer)))) { + // Decoded in system memory, that is in software. + RTC_LOG(LS_WARNING) << implementationName << " does not decode on the GPU"; + return false; + } + + ComPtr texture; + UINT subresource = 0; + + if (FAILED(dxgiBuffer->GetResource(IID_PPV_ARGS(&texture))) || + FAILED(dxgiBuffer->GetSubresourceIndex(&subresource))) + { + return false; + } + + D3D11_TEXTURE2D_DESC desc = {}; + texture->GetDesc(&desc); + + if (desc.Format != DXGI_FORMAT_NV12 || desc.Width < width || desc.Height < height) { + return false; + } + + if (!staging || stagingDesc.Width != desc.Width || stagingDesc.Height != desc.Height) { + stagingDesc = desc; + stagingDesc.ArraySize = 1; + stagingDesc.MipLevels = 1; + stagingDesc.Usage = D3D11_USAGE_STAGING; + stagingDesc.BindFlags = 0; + stagingDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + stagingDesc.MiscFlags = 0; + + staging.Reset(); + + if (FAILED(device->CreateTexture2D(&stagingDesc, nullptr, &staging))) { + return false; + } + } + + context->CopySubresourceRegion(staging.Get(), 0, 0, 0, 0, texture.Get(), subresource, nullptr); + + D3D11_MAPPED_SUBRESOURCE mapped = {}; + + if (FAILED(context->Map(staging.Get(), 0, D3D11_MAP_READ, 0, &mapped))) { + return false; + } + + const uint8_t * y = static_cast(mapped.pData); + // The chroma plane follows all rows of the texture, not only those of + // the picture. + const uint8_t * uv = y + static_cast(mapped.RowPitch) * desc.Height; + + frame = webrtc::I420Buffer::Create(static_cast(width), static_cast(height)); + + libyuv::NV12ToI420(y, static_cast(mapped.RowPitch), uv, static_cast(mapped.RowPitch), + frame->MutableDataY(), frame->StrideY(), frame->MutableDataU(), frame->StrideU(), + frame->MutableDataV(), frame->StrideV(), static_cast(width), static_cast(height)); + + context->Unmap(staging.Get(), 0); + + return true; + } + + int32_t MFVideoDecoder::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t MFVideoDecoder::Release() + { + if (transform) { + transform->ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0); + transform->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); + transform.Reset(); + } + if (activate) { + activate->ShutdownObject(); + activate.Reset(); + } + + staging.Reset(); + deviceManager.Reset(); + context.Reset(); + device.Reset(); + mfInitializer.reset(); + + pendingFrames.clear(); + + return WEBRTC_VIDEO_CODEC_OK; + } + + webrtc::VideoDecoder::DecoderInfo MFVideoDecoder::GetDecoderInfo() const + { + DecoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + + return info; + } + + const char * MFVideoDecoder::ImplementationName() const + { + return implementationName.c_str(); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp new file mode 100644 index 00000000..4f6defed --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp @@ -0,0 +1,128 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/windows/MFVideoDecoderFactory.h" +#include "media/video/codec/windows/MFDecoderUtils.h" +#include "media/video/codec/windows/MFVideoDecoder.h" +#include "platform/windows/ComInitializer.h" +#include "platform/windows/MFInitializer.h" + +#include "api/video_codecs/video_codec.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "rtc_base/logging.h" + +using Microsoft::WRL::ComPtr; + +namespace jni +{ + namespace + { + // Whether Windows has a decoder for the format that decodes on a + // Direct3D 11 device. + bool HasDirect3DDecoder(const GUID & format, IMFDXGIDeviceManager * manager) + { + std::vector> decoders; + + if (FAILED(EnumerateDecoders(format, decoders)) || decoders.empty()) { + return false; + } + + ComPtr activate; + ComPtr transform; + + if (FAILED(ActivateDirect3DDecoder(decoders, manager, activate, transform))) { + return false; + } + + transform.Reset(); + activate->ShutdownObject(); + + return true; + } + } + + std::unique_ptr MFVideoDecoderFactory::Create() + { + bool h264 = false; + bool av1 = false; + + try { + ComInitializer comInitializer; + MFInitializer mfInitializer; + + ComPtr device; + ComPtr context; + + if (FAILED(CreateVideoDevice(device, context))) { + RTC_LOG(LS_INFO) << "Media Foundation decoders: no Direct3D 11 video device"; + return nullptr; + } + + ComPtr manager; + UINT resetToken = 0; + + if (FAILED(MFCreateDXGIDeviceManager(&resetToken, &manager)) || + FAILED(manager->ResetDevice(device.Get(), resetToken))) + { + return nullptr; + } + + // The GPU has to decode the codec, and Windows has to have a + // decoder that lets it. + h264 = SupportsDecoderProfile(device.Get(), D3D11_DECODER_PROFILE_H264_VLD_NOFGT) + && HasDirect3DDecoder(MFVideoFormat_H264, manager.Get()); + av1 = SupportsDecoderProfile(device.Get(), D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0) + && HasDirect3DDecoder(MFVideoFormat_AV1, manager.Get()); + } + catch (...) { + return nullptr; + } + + RTC_LOG(LS_INFO) << "Media Foundation hardware decoders, H.264: " << h264 << ", AV1: " << av1; + + if (!h264 && !av1) { + return nullptr; + } + + return std::unique_ptr(new MFVideoDecoderFactory(h264, av1)); + } + + MFVideoDecoderFactory::MFVideoDecoderFactory(bool h264, bool av1) : + h264(h264), + av1(av1) + { + } + + std::vector MFVideoDecoderFactory::GetSupportedFormats() const + { + std::vector formats; + + if (h264) { + formats = webrtc::SupportedH264DecoderCodecs(); + } + if (av1) { + formats.push_back(webrtc::SdpVideoFormat::AV1Profile0()); + } + + return formats; + } + + std::unique_ptr MFVideoDecoderFactory::Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) + { + return std::make_unique(webrtc::PayloadStringToCodecType(format.name)); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/WindowsHardwareVideoEncoderFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/WindowsHardwareVideoCodecFactories.cpp similarity index 76% rename from webrtc-jni/src/main/cpp/src/media/video/codec/windows/WindowsHardwareVideoEncoderFactories.cpp rename to webrtc-jni/src/main/cpp/src/media/video/codec/windows/WindowsHardwareVideoCodecFactories.cpp index 768ea91d..d651eb55 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/WindowsHardwareVideoEncoderFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/WindowsHardwareVideoCodecFactories.cpp @@ -14,8 +14,10 @@ * limitations under the License. */ +#include "media/video/codec/HardwareVideoDecoderFactory.h" #include "media/video/codec/HardwareVideoEncoderFactory.h" #include "media/video/codec/nvenc/NvencVideoEncoderFactory.h" +#include "media/video/codec/windows/MFVideoDecoderFactory.h" #include "media/video/codec/windows/MFVideoEncoderFactory.h" namespace jni @@ -36,4 +38,15 @@ namespace jni return factories; } + + std::vector> CreatePlatformHardwareVideoDecoderFactories() + { + std::vector> factories; + + if (auto mediaFoundation = MFVideoDecoderFactory::Create()) { + factories.push_back(std::move(mediaFoundation)); + } + + return factories; + } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java index 95ef3cc0..029a8057 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java @@ -31,6 +31,9 @@ * WebRTC's default factories, with H.264 through VideoToolbox. * {@link #getSupportedCodecs()} lists what the platform has. *

+ * To decode on the GPU where the platform supports it, use a {@link + * HardwareVideoDecoderFactory} instead. + *

* A factory of one's own can hand out these decoders next to its own ones * by delegating to an instance of this class. The decoders it creates are * {@link NativeVideoDecoder}s, which run inside WebRTC and are not to be @@ -80,7 +83,7 @@ public NativeVideoDecoder createDecoder(VideoCodecInfo info) { for (VideoCodecInfo codec : supportedCodecs) { if (codec.getName().equalsIgnoreCase(info.getName())) { - return new NativeVideoDecoder(info); + return new NativeVideoDecoder(info, false); } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java new file mode 100644 index 00000000..e42dcdd8 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -0,0 +1,106 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.internal.NativeLoader; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * The built-in video decoders, with the decoders of the GPU in front of them + * where the platform has them. Set it on a {@link + * dev.onvoid.webrtc.PeerConnectionFactory PeerConnectionFactory} to decode in + * hardware: + * 

{@code
+ * PeerConnectionFactory factory = PeerConnectionFactory.builder()
+ *     .setVideoDecoderFactory(new HardwareVideoDecoderFactory())
+ *     .build();
+ * }
+ *

+ * On Windows, H.264 and AV1 are decoded on the GPU through Direct3D 11 by the + * Media Foundation decoders of Windows, where the GPU decodes the codec. + * Decoded frames are copied back to system memory, which WebRTC's frames are + * in, so hardware decoding saves CPU mostly at high resolutions. A hardware + * decoder that fails to start, or fails while decoding, is replaced by the + * software decoder of the same codec, which starts with the next key frame. + * Where there is no hardware decoder, this factory decodes like a {@link + * DefaultVideoDecoderFactory}. On macOS, that already uses VideoToolbox. + * Linux has no hardware decoders yet. + *

+ * The hardware decoders take over only codecs the software decoders have too, + * so the supported codecs are the same as those of a {@link + * DefaultVideoDecoderFactory}, and what a peer connection negotiates does not + * depend on the GPU. + * + * @author Alex Andres + */ +public final class HardwareVideoDecoderFactory implements VideoDecoderFactory { + + static { + try { + NativeLoader.loadLibrary("webrtc-java"); + } + catch (Exception e) { + throw new RuntimeException("Load library 'webrtc-java' failed", e); + } + } + + + private final List supportedCodecs; + + + /** + * Creates a factory for the hardware video decoders. + */ + public HardwareVideoDecoderFactory() { + supportedCodecs = Collections.unmodifiableList( + Arrays.asList(getSupportedCodecsInternal())); + } + + @Override + public List getSupportedCodecs() { + return supportedCodecs; + } + + /** + * Returns a built-in decoder for the given codec, decoding on the GPU if + * it can. + * + * @param info The codec to decode, as negotiated. + * + * @return A native decoder, or {@code null} if there is no built-in + * decoder for the codec. + */ + @Override + public NativeVideoDecoder createDecoder(VideoCodecInfo info) { + Objects.requireNonNull(info, "VideoCodecInfo is null"); + + for (VideoCodecInfo codec : supportedCodecs) { + if (codec.getName().equalsIgnoreCase(info.getName())) { + return new NativeVideoDecoder(info, true); + } + } + + return null; + } + + private static native VideoCodecInfo[] getSupportedCodecsInternal(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java index e2b941b0..49984a1a 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java @@ -17,10 +17,10 @@ package dev.onvoid.webrtc.media.video.codec; /** - * A built-in decoder, created by a {@link DefaultVideoDecoderFactory}. It is a - * placeholder: returned from a {@link VideoDecoderFactory}, it makes WebRTC - * create the native decoder for its codec, which then runs entirely inside - * WebRTC. Its methods are therefore not to be called from Java, and throw + * A built-in decoder, created by a {@link DefaultVideoDecoderFactory} or a + * {@link HardwareVideoDecoderFactory}. It is a placeholder: returned from a + * {@link VideoDecoderFactory}, it makes WebRTC create the native decoder for + * its codec, which then runs entirely inside WebRTC. Its methods are therefore not to be called from Java, and throw * {@link UnsupportedOperationException}. * * @author Alex Andres @@ -30,9 +30,13 @@ public final class NativeVideoDecoder implements VideoDecoder { /** The codec to create the native decoder for; read by native code. */ private final VideoCodecInfo codecInfo; + /** Whether the decoder may use the GPU; read by native code. */ + private final boolean hardwareAcceleration; - NativeVideoDecoder(VideoCodecInfo codecInfo) { + + NativeVideoDecoder(VideoCodecInfo codecInfo, boolean hardwareAcceleration) { this.codecInfo = codecInfo; + this.hardwareAcceleration = hardwareAcceleration; } /** diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java new file mode 100644 index 00000000..9bea9be3 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java @@ -0,0 +1,164 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoDecoderFactory; + +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests the hardware decoders of the {@link HardwareVideoDecoderFactory}. + *

+ * A machine without a hardware decoder skips the tests that need one, as CI + * runners do. Set the system property {@code webrtc.test.hardwareDecoder} to + * {@code true} on a machine that has one, to make those tests fail instead, + * and {@code webrtc.test.hardwareAv1Decoder} for a GPU that decodes AV1. + */ +@Execution(ExecutionMode.SAME_THREAD) +class HardwareVideoDecoderIntegrationTest extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + private static final boolean HARDWARE_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareDecoder"); + + private static final boolean HARDWARE_AV1_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareAv1Decoder"); + + private static final String OS = System.getProperty("os.name").toLowerCase(Locale.ROOT); + + private static final Predicate H264 = codec -> + "H264".equalsIgnoreCase(codec.getName()) + && "1".equals(codec.getSDPFmtp().get("packetization-mode")) + && codec.getSDPFmtp().getOrDefault("profile-level-id", "").startsWith("42e0"); + + private static final Predicate AV1 = codec -> "AV1".equalsIgnoreCase(codec.getName()); + + + @Test + void hardwareKeepsCodecs() { + // Hardware decoding must not change what is negotiated. + assertEquals(new DefaultVideoDecoderFactory().getSupportedCodecs(), + new HardwareVideoDecoderFactory().getSupportedCodecs()); + } + + @Test + void hardwareDecodesH264() throws Exception { + assertHardwareDecodes(H264, HARDWARE_REQUIRED); + } + + @Test + void hardwareDecodesAv1() throws Exception { + assertHardwareDecodes(AV1, HARDWARE_AV1_REQUIRED); + } + + private void assertHardwareDecodes(Predicate codec, boolean required) throws Exception { + assumeTrue(OS.contains("win"), "hardware decoders are implemented on Windows only"); + + PeerConnectionFactory hardware = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoDecoderFactory(new HardwareVideoDecoderFactory()) + .build(); + + CountDownLatch received = new CountDownLatch(10); + AtomicReference wrongSize = new AtomicReference<>(); + String implementation; + + try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + // The call sends 320x240; a decoder that returned the padded + // frame rather than the picture would be larger. + int width = frame.buffer.getWidth(); + int height = frame.buffer.getHeight(); + + if (width != 320 || height != 240) { + wrongSize.compareAndSet(null, width + "x" + height); + } + + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); + + implementation = decoderImplementation(call); + + track.removeSink(sink); + receiver.dispose(); + } + finally { + hardware.dispose(); + } + + boolean hardwareUsed = implementation.contains("MediaFoundation"); + + if (required) { + assertTrue(hardwareUsed, implementation); + } + else { + assumeTrue(hardwareUsed, "no hardware decoder: " + implementation); + } + + assertNull(wrongSize.get(), "decoded frames of the wrong size: " + wrongSize.get()); + } + + /** + * Returns what the receiver reports its decoder to be, once the + * statistics have caught up with it. + */ + private static String decoderImplementation(TestMediaCall call) throws InterruptedException { + Object implementation = null; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (implementation == null && System.nanoTime() < deadline) { + Map inbound = call.getInboundVideoStats(); + + if (inbound != null) { + implementation = inbound.get("decoderImplementation"); + } + if (implementation == null) { + Thread.sleep(100); + } + } + + return String.valueOf(implementation); + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java index dc9e6af5..ebc4e243 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java @@ -166,6 +166,29 @@ Map getOutboundVideoStats() throws InterruptedException { return outbound.get(); } + /** + * Returns the callee's statistics of its inbound video stream, or null if + * there are none yet. + */ + Map getInboundVideoStats() throws InterruptedException { + CountDownLatch done = new CountDownLatch(1); + AtomicReference> inbound = new AtomicReference<>(); + + callee.getPeerConnection().getStats(report -> { + for (RTCStats stats : report.getStats().values()) { + if (stats.getType() == RTCStatsType.INBOUND_RTP + && "video".equals(stats.getAttributes().get("kind"))) { + inbound.set(stats.getAttributes()); + } + } + done.countDown(); + }); + + done.await(5, TimeUnit.SECONDS); + + return inbound.get(); + } + RTCRtpSender getVideoSender() { return videoSender; } From 04c9f2a2449c48a2dd7f2e1ea597150551cb629a Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 1 Oct 2026 23:48:29 +0200 Subject: [PATCH 2/2] test: cover H.264 through VideoToolbox on macOS --- .../HardwareVideoDecoderIntegrationTest.java | 68 +++++++++++++++---- .../HardwareVideoEncoderIntegrationTest.java | 22 ++++++ 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java index 9bea9be3..b2b9e8c1 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java @@ -89,11 +89,59 @@ private void assertHardwareDecodes(Predicate codec, boole .setVideoDecoderFactory(new HardwareVideoDecoderFactory()) .build(); + String implementation; + + try { + implementation = decoderImplementation(hardware, codec); + } + finally { + hardware.dispose(); + } + + boolean hardwareUsed = implementation.contains("MediaFoundation"); + + if (required) { + assertTrue(hardwareUsed, implementation); + } + else { + assumeTrue(hardwareUsed, "no hardware decoder: " + implementation); + } + } + + @Test + void macDecodesH264WithVideoToolbox() throws Exception { + assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only"); + + // The default decoders use VideoToolbox on macOS. + assertTrue(decoderImplementation(factory, H264).contains("VideoToolbox")); + + // The hardware factory has nothing of its own there, and hands over to + // the default decoders. + PeerConnectionFactory hardware = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoDecoderFactory(new HardwareVideoDecoderFactory()) + .build(); + + try { + assertTrue(decoderImplementation(hardware, H264).contains("VideoToolbox")); + } + finally { + hardware.dispose(); + } + } + + /** + * Receives video in the preferred codec through a call, checks that the + * decoded frames are the size that was sent, and returns what the receiver + * reports its decoder to be. + */ + private static String decoderImplementation(PeerConnectionFactory factory, + Predicate codec) throws Exception { CountDownLatch received = new CountDownLatch(10); AtomicReference wrongSize = new AtomicReference<>(); String implementation; - try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { + try (TestMediaCall call = new TestMediaCall(factory, true, false, codec)) { call.negotiate(); RTCRtpReceiver receiver = call.getReceiver("video"); @@ -118,32 +166,22 @@ private void assertHardwareDecodes(Predicate codec, boole assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); - implementation = decoderImplementation(call); + implementation = decoderImplementationOf(call); track.removeSink(sink); receiver.dispose(); } - finally { - hardware.dispose(); - } - - boolean hardwareUsed = implementation.contains("MediaFoundation"); - - if (required) { - assertTrue(hardwareUsed, implementation); - } - else { - assumeTrue(hardwareUsed, "no hardware decoder: " + implementation); - } assertNull(wrongSize.get(), "decoded frames of the wrong size: " + wrongSize.get()); + + return implementation; } /** * Returns what the receiver reports its decoder to be, once the * statistics have caught up with it. */ - private static String decoderImplementation(TestMediaCall call) throws InterruptedException { + private static String decoderImplementationOf(TestMediaCall call) throws InterruptedException { Object implementation = null; long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 6e80037a..95d70a06 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -122,6 +122,28 @@ void defaultEncodesH264InSoftware() throws Exception { assertTrue(implementation.contains("OpenH264"), implementation); } + @Test + void macEncodesH264WithVideoToolbox() throws Exception { + assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only"); + + // The default encoders use VideoToolbox on macOS. + assertTrue(encoderImplementation(factory, H264).contains("VideoToolbox")); + + // The hardware factory has nothing of its own there, and hands over to + // the default encoders. + PeerConnectionFactory hardware = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .build(); + + try { + assertTrue(encoderImplementation(hardware, H264).contains("VideoToolbox")); + } + finally { + hardware.dispose(); + } + } + private static void assertHardware(String implementation, boolean required) { boolean hardwareUsed = implementation.startsWith("NVENC") || implementation.startsWith("VA-API")