diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 9c88d61e..34b6a9b7 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -14,6 +14,28 @@ for (VideoCodecInfo codec : new DefaultVideoEncoderFactory().getSupportedCodecs( } ``` +### Hardware Encoding + +`HardwareVideoEncoderFactory` offers the same codecs as `DefaultVideoEncoderFactory`, but encodes on the GPU where the platform supports it: + +```java +PeerConnectionFactory factory = PeerConnectionFactory.builder() + .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .build(); +``` + +| Platform | Hardware encoding | +|---|---| +| Windows | H.264 through the Media Foundation encoder of the GPU driver (NVIDIA, AMD, Intel) | +| macOS | H.264 through VideoToolbox, as with `DefaultVideoEncoderFactory` | +| Linux | Not yet; encoding is in software | + +On Windows, the hardware encoder takes over H.264 Constrained Baseline and Baseline with packetization mode 1, formats the software encoder offers too, so encoding in hardware never changes what is negotiated. When the hardware encoder fails to start, for example because the GPU has no encoder sessions left, or fails while encoding, the stream switches to the software encoder and continues with a key frame. On a machine without a hardware encoder, the factory encodes like `DefaultVideoEncoderFactory`. + +Which encoder a stream uses shows in the `encoderImplementation` statistic of its `outbound-rtp` stats, e.g. `MediaFoundation (AMDh264Encoder)` or `OpenH264`. + +### Native Codecs + The encoders and decoders these factories create are `NativeVideoEncoder`s and `NativeVideoDecoder`s. They are placeholders that make WebRTC create the built-in codec, which then runs entirely inside WebRTC, so their methods are not to be called from Java. ## Setting the Factories @@ -163,6 +185,7 @@ Whatever an encoder, decoder or factory method throws is logged and treated as ` - `VideoEncoderFactory`, `VideoDecoderFactory` — create the codecs of a factory. - `VideoEncoder`, `VideoDecoder`, `EncodedImage` — codecs implemented in Java. - `DefaultVideoEncoderFactory`, `DefaultVideoDecoderFactory` — the built-in codecs. +- `HardwareVideoEncoderFactory` — the built-in encoders, on the GPU where possible. - `RTCRtpTransceiver.setCodecPreferences()` — chooses among the negotiated codecs. For the full API, see the JavaDoc of the `dev.onvoid.webrtc.media.video.codec` package. diff --git a/webrtc-jni/src/main/cpp/CMakeLists.txt b/webrtc-jni/src/main/cpp/CMakeLists.txt index 2c83e39d..e5467a44 100644 --- a/webrtc-jni/src/main/cpp/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/CMakeLists.txt @@ -56,6 +56,7 @@ file(GLOB SOURCES_MEDIA_AUDIO "src/media/audio/*.cpp") file(GLOB SOURCES_MEDIA_AUDIO_OS "src/media/audio/${SOURCE_TARGET}/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO "src/media/video/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_CODEC "src/media/video/codec/*.cpp") +file(GLOB SOURCES_MEDIA_VIDEO_CODEC_OS "src/media/video/codec/${SOURCE_TARGET}/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_DESKTOP "src/media/video/desktop/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_DESKTOP_OS "src/media/video/desktop/${SOURCE_TARGET}/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_OS "src/media/video/${SOURCE_TARGET}/*.cpp") @@ -71,6 +72,7 @@ list(APPEND SOURCES ${SOURCES_MEDIA_AUDIO_OS} ${SOURCES_MEDIA_VIDEO} ${SOURCES_MEDIA_VIDEO_CODEC} + ${SOURCES_MEDIA_VIDEO_CODEC_OS} ${SOURCES_MEDIA_VIDEO_DESKTOP} ${SOURCES_MEDIA_VIDEO_DESKTOP_OS} ${SOURCES_MEDIA_VIDEO_OS} diff --git a/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoEncoderFactory.h new file mode 100644 index 00000000..dba3732e --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoEncoderFactory.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory +#define _Included_dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory + * Method: getSupportedCodecsInternal + * Signature: ()[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; + */ + JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory_getSupportedCodecsInternal + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h index c3d43f52..5a2fa4b0 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h @@ -28,6 +28,11 @@ namespace jni // software, or on macOS those of WebRTC's default Objective-C factories, // which use VideoToolbox. std::unique_ptr CreateDefaultVideoEncoderFactory(); + + // The default encoders, with the encoders of the GPU in front of them + // where the platform has them, falling back to the software ones. On + // macOS the default encoders already use VideoToolbox. + std::unique_ptr CreateHardwareVideoEncoderFactory(); std::unique_ptr CreateDefaultVideoDecoderFactory(); } diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/FallbackVideoEncoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/FallbackVideoEncoder.h new file mode 100644 index 00000000..eb77272f --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/FallbackVideoEncoder.h @@ -0,0 +1,74 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_FALLBACK_VIDEO_ENCODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_FALLBACK_VIDEO_ENCODER_H_ + +#include "api/video/video_frame.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" + +#include +#include +#include + +namespace jni +{ + // Encodes with a hardware encoder, and switches to a software encoder of + // the same codec when the hardware one fails to initialize, or gives up + // while encoding by returning WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE. The + // switch is for good; the software encoder starts with a key frame. + // + // WebRTC has such a wrapper too, but it is not part of the WebRTC + // library this library links. + class FallbackVideoEncoder : public webrtc::VideoEncoder + { + public: + FallbackVideoEncoder(std::unique_ptr hardware, + std::unique_ptr software); + ~FallbackVideoEncoder() override = default; + + int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override; + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) override; + void SetRates(const RateControlParameters & parameters) override; + void OnPacketLossRateUpdate(float packetLossRate) override; + void OnRttUpdate(int64_t rttMs) override; + EncoderInfo GetEncoderInfo() const override; + + private: + // Initializes the software encoder with the settings the hardware + // one was given. Returns whether it is ready. + bool StartSoftware(); + + webrtc::VideoEncoder * Active() const; + + private: + const std::unique_ptr hardware; + const std::unique_ptr software; + + bool useSoftware; + bool initialized; + + std::optional codecSettings; + std::optional encoderSettings; + std::optional rates; + webrtc::EncodedImageCallback * callback; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h new file mode 100644 index 00000000..3f1524c8 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_HARDWARE_VIDEO_ENCODER_FACTORY_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_HARDWARE_VIDEO_ENCODER_FACTORY_H_ + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" + +#include +#include + +namespace jni +{ + // Puts hardware encoders in front of software ones. A codec both have is + // encoded in hardware, inside a wrapper that switches to the software + // encoder when the hardware one fails to initialize or gives up while + // encoding, for example once a GPU runs out of encoder sessions. A codec + // only one of them has is encoded by that one. + class HardwareVideoEncoderFactory : public webrtc::VideoEncoderFactory + { + public: + HardwareVideoEncoderFactory(std::unique_ptr hardware, + std::unique_ptr software); + ~HardwareVideoEncoderFactory() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + const std::unique_ptr hardware; + const std::unique_ptr software; + }; + + // Returns a factory for the hardware encoders of the platform, or null if + // the platform has none this library supports, or no device that can + // encode. Defined per platform. + std::unique_ptr CreatePlatformHardwareVideoEncoderFactory(); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h index e1b9ef9a..663d2f61 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h @@ -59,12 +59,15 @@ namespace jni jclass nativeEncoderClass; jfieldID nativeEncoderCodecInfo; + jfieldID nativeEncoderHardwareAcceleration; }; private: const JavaGlobalRef factory; const std::shared_ptr javaClass; + // The built-in encoders, and those with the GPU's in front. const std::unique_ptr defaultFactory; + const std::unique_ptr hardwareFactory; std::vector supportedFormats; }; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFEncoderUtils.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFEncoderUtils.h new file mode 100644 index 00000000..85cf8e37 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFEncoderUtils.h @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_ENCODER_UTILS_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_ENCODER_UTILS_H_ + +#include +#include +#include + +#include +#include + +namespace jni +{ + // Lists the hardware H.264 encoder transforms that take NV12, best + // first. Media Foundation has to be started. + HRESULT EnumerateHardwareH264Encoders(std::vector> & encoders); + + // Returns the name the driver gives a transform, e.g. "AMDh264Encoder". + std::string GetTransformName(IMFActivate * activate); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFH264Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFH264Encoder.h new file mode 100644 index 00000000..42958e79 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFH264Encoder.h @@ -0,0 +1,133 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_H264_ENCODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_H264_ENCODER_H_ + +#include "media/video/codec/windows/MFTransformEvents.h" +#include "platform/windows/MFInitializer.h" + +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "modules/video_coding/codecs/h264/include/h264_globals.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace jni +{ + // Encodes H.264 with the hardware encoder of the GPU, through the Media + // Foundation transform its driver provides. These transforms are + // asynchronous: they ask for input and announce output through events, + // which arrive on a Media Foundation thread, so encoded frames are + // handed to WebRTC from there. + // + // Frames are passed in system memory as NV12; the transform uploads them. + // Anything that fails makes the encoder give up, so that WebRTC switches + // to the software encoder. + class MFH264Encoder : public webrtc::VideoEncoder, public MFTransformEventListener + { + public: + explicit MFH264Encoder(const webrtc::SdpVideoFormat & format); + ~MFH264Encoder() override; + + int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override; + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) override; + void SetRates(const RateControlParameters & parameters) override; + EncoderInfo GetEncoderInfo() const override; + + void OnTransformEvent(MediaEventType type, HRESULT status) override; + + private: + // What becomes of an input frame once its output arrives. + struct PendingFrame + { + uint32_t rtpTimestamp; + int64_t captureTimeMs; + int64_t ntpTimeMs; + webrtc::VideoRotation rotation; + }; + + HRESULT CreateTransform(); + HRESULT ConfigureTypes(); + void ConfigureCodec(); + void SetBitrate(uint32_t bitrateBps); + HRESULT CreateInputSample(const webrtc::VideoFrame & frame, IMFSample ** sample); + void ProcessOutput(); + void DeliverOutput(IMFSample * sample); + void ShutdownTransform(); + + private: + const webrtc::SdpVideoFormat format; + std::string implementationName; + + std::unique_ptr mfInitializer; + + // Activated anew for each initialization, since an activation + // object cannot be relied on to activate again once shut down. + Microsoft::WRL::ComPtr activate; + Microsoft::WRL::ComPtr transform; + Microsoft::WRL::ComPtr codecApi; + Microsoft::WRL::ComPtr events; + DWORD inputStreamId; + DWORD outputStreamId; + + webrtc::VideoCodec codecSettings; + uint32_t bitrateBps; + uint32_t framerate; + + std::atomic callback; + + // Set once the transform fails, from any thread. + std::atomic failed; + + // Input the transform asked for and has not been given. + std::mutex inputMutex; + std::condition_variable inputRequested; + int inputRequests; + + // Frames given to the transform, by sample time. Used on the + // encoder thread and the event thread. + std::mutex pendingMutex; + std::map pendingFrames; + + // Used on the encoder thread only. + LONGLONG lastSampleTime; + bool keyFrameRequested; + + // Used on the event thread only. + std::vector parameterSets; + webrtc::H264BitstreamParser bitstreamParser; + webrtc::H264PacketizationMode packetizationMode; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFTransformEvents.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFTransformEvents.h new file mode 100644 index 00000000..83990daf --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFTransformEvents.h @@ -0,0 +1,82 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_TRANSFORM_EVENTS_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_TRANSFORM_EVENTS_H_ + +#include +#include +#include +#include + +#include +#include + +namespace jni +{ + // Receives the events of an asynchronous Media Foundation transform. + class MFTransformEventListener + { + public: + virtual ~MFTransformEventListener() = default; + + // Called on a Media Foundation work queue thread, one event at a + // time. + virtual void OnTransformEvent(MediaEventType type, HRESULT status) = 0; + }; + + // Pumps the events of an asynchronous Media Foundation transform, which + // hardware encoders are: the transform asks for input and announces + // output through events rather than accepting frames on demand. + // + // Media Foundation keeps a reference to this object while an event + // request is pending, so it may outlive its listener. Stop() detaches + // the listener; once it returns, the listener is not called again. + class MFTransformEvents : public IMFAsyncCallback + { + public: + // Starts pumping the events of the transform to the listener. + static HRESULT Start(IMFMediaEventGenerator * generator, MFTransformEventListener * listener, + Microsoft::WRL::ComPtr * events); + + // Detaches the listener, waiting for an event it is handling. + void Stop(); + + // IUnknown + STDMETHODIMP QueryInterface(REFIID iid, void ** object) override; + STDMETHODIMP_(ULONG) AddRef() override; + STDMETHODIMP_(ULONG) Release() override; + + // IMFAsyncCallback + STDMETHODIMP GetParameters(DWORD * flags, DWORD * queue) override; + STDMETHODIMP Invoke(IMFAsyncResult * result) override; + + private: + MFTransformEvents(IMFMediaEventGenerator * generator, MFTransformEventListener * listener); + virtual ~MFTransformEvents() = default; + + private: + std::atomic references; + + const Microsoft::WRL::ComPtr generator; + + // Guards the listener, and is held while it handles an event. + std::mutex mutex; + MFTransformEventListener * listener; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoEncoderFactory.h new file mode 100644 index 00000000..0e729d33 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoEncoderFactory.h @@ -0,0 +1,52 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_VIDEO_ENCODER_FACTORY_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_MF_VIDEO_ENCODER_FACTORY_H_ + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" + +#include +#include + +namespace jni +{ + // Creates the Media Foundation hardware encoders of the GPU. It offers + // H.264 in the profiles WebRTC's software encoder offers too, and only + // with packetization mode 1: mode 0 needs each NAL unit to fit a packet, + // which hardware encoders cannot be relied on to keep to. + class MFVideoEncoderFactory : public webrtc::VideoEncoderFactory + { + public: + // Returns a factory, or null if there is no hardware H.264 + // encoder on this system. + static std::unique_ptr Create(); + + ~MFVideoEncoderFactory() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + MFVideoEncoderFactory() = default; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoEncoderFactory.cpp new file mode 100644 index 00000000..f7cc8a12 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoEncoderFactory.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_HardwareVideoEncoderFactory.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "JavaUtils.h" + +JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory_getSupportedCodecsInternal +(JNIEnv * env, jclass caller) +{ + try { + auto factory = jni::CreateHardwareVideoEncoderFactory(); + + return jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()).release(); + } + catch (...) { + ThrowCxxJavaException(env); + } + + return nullptr; +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp index 9fd32ac5..845fbf6e 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp @@ -16,6 +16,8 @@ #include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/HardwareVideoEncoderFactory.h" + #ifdef __APPLE__ #include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" #include "sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h" @@ -49,6 +51,21 @@ namespace jni #endif } + std::unique_ptr CreateHardwareVideoEncoderFactory() + { +#ifdef __APPLE__ + return CreateDefaultVideoEncoderFactory(); +#else + std::unique_ptr hardware = CreatePlatformHardwareVideoEncoderFactory(); + + if (!hardware) { + return CreateDefaultVideoEncoderFactory(); + } + + return std::make_unique(std::move(hardware), CreateDefaultVideoEncoderFactory()); +#endif + } + std::unique_ptr CreateDefaultVideoDecoderFactory() { #ifdef __APPLE__ diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/FallbackVideoEncoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/FallbackVideoEncoder.cpp new file mode 100644 index 00000000..3e8d3432 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/FallbackVideoEncoder.cpp @@ -0,0 +1,159 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/FallbackVideoEncoder.h" + +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace jni +{ + FallbackVideoEncoder::FallbackVideoEncoder(std::unique_ptr hardware, + std::unique_ptr software) : + hardware(std::move(hardware)), + software(std::move(software)), + useSoftware(false), + initialized(false), + callback(nullptr) + { + } + + int32_t FallbackVideoEncoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) + { + codecSettings = *settings; + this->encoderSettings = encoderSettings; + rates.reset(); + + if (!useSoftware) { + int32_t result = hardware->InitEncode(settings, encoderSettings); + + if (result == WEBRTC_VIDEO_CODEC_OK) { + initialized = true; + return result; + } + if (result == WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED) { + // WebRTC then encodes each stream with an encoder of its own. + return result; + } + + RTC_LOG(LS_WARNING) << "Hardware encoder failed to initialize (" << result + << "), switching to software"; + + hardware->Release(); + useSoftware = true; + } + + initialized = StartSoftware(); + + return initialized ? WEBRTC_VIDEO_CODEC_OK : WEBRTC_VIDEO_CODEC_ERROR; + } + + bool FallbackVideoEncoder::StartSoftware() + { + if (!codecSettings || !encoderSettings) { + return false; + } + + if (software->InitEncode(&*codecSettings, *encoderSettings) != WEBRTC_VIDEO_CODEC_OK) { + RTC_LOG(LS_WARNING) << "Software encoder failed to initialize"; + return false; + } + + if (callback != nullptr) { + software->RegisterEncodeCompleteCallback(callback); + } + if (rates) { + software->SetRates(*rates); + } + + return true; + } + + int32_t FallbackVideoEncoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) + { + callback = encodeCallback; + + hardware->RegisterEncodeCompleteCallback(encodeCallback); + software->RegisterEncodeCompleteCallback(encodeCallback); + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t FallbackVideoEncoder::Release() + { + initialized = false; + + return Active()->Release(); + } + + int32_t FallbackVideoEncoder::Encode(const webrtc::VideoFrame & frame, + const std::vector * frameTypes) + { + if (!initialized) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + int32_t result = Active()->Encode(frame, frameTypes); + + if (useSoftware || result != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) { + return result; + } + + RTC_LOG(LS_WARNING) << "Hardware encoder gave up, switching to software"; + + hardware->Release(); + useSoftware = true; + initialized = StartSoftware(); + + if (!initialized) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // The receiver has to start over with what the software encoder + // produces, so that begins with a key frame. + std::vector keyFrames(frameTypes != nullptr ? frameTypes->size() : 1, + webrtc::VideoFrameType::kVideoFrameKey); + + return software->Encode(frame, &keyFrames); + } + + void FallbackVideoEncoder::SetRates(const RateControlParameters & parameters) + { + rates = parameters; + + Active()->SetRates(parameters); + } + + void FallbackVideoEncoder::OnPacketLossRateUpdate(float packetLossRate) + { + Active()->OnPacketLossRateUpdate(packetLossRate); + } + + void FallbackVideoEncoder::OnRttUpdate(int64_t rttMs) + { + Active()->OnRttUpdate(rttMs); + } + + webrtc::VideoEncoder::EncoderInfo FallbackVideoEncoder::GetEncoderInfo() const + { + return Active()->GetEncoderInfo(); + } + + webrtc::VideoEncoder * FallbackVideoEncoder::Active() const + { + return useSoftware ? software.get() : hardware.get(); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp new file mode 100644 index 00000000..3b0a17f6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/HardwareVideoEncoderFactory.h" +#include "media/video/codec/FallbackVideoEncoder.h" + +#include + +namespace jni +{ + namespace + { + bool Supports(const webrtc::VideoEncoderFactory & factory, const webrtc::SdpVideoFormat & format) + { + const std::vector formats = factory.GetSupportedFormats(); + + return std::any_of(formats.begin(), formats.end(), [&](const webrtc::SdpVideoFormat & supported) { + return supported.IsSameCodec(format); + }); + } + } + + HardwareVideoEncoderFactory::HardwareVideoEncoderFactory(std::unique_ptr hardware, + std::unique_ptr software) : + hardware(std::move(hardware)), + software(std::move(software)) + { + } + + std::vector HardwareVideoEncoderFactory::GetSupportedFormats() const + { + // The software formats keep their order, so that hardware support + // does not change which codec a peer connection prefers. + std::vector formats = software->GetSupportedFormats(); + + for (const webrtc::SdpVideoFormat & format : hardware->GetSupportedFormats()) { + if (!Supports(*software, format)) { + formats.push_back(format); + } + } + + return formats; + } + + std::unique_ptr HardwareVideoEncoderFactory::Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) + { + std::unique_ptr hardwareEncoder; + std::unique_ptr softwareEncoder; + + if (Supports(*hardware, format)) { + hardwareEncoder = hardware->Create(env, format); + } + if (Supports(*software, format)) { + softwareEncoder = software->Create(env, format); + } + + if (hardwareEncoder && softwareEncoder) { + return std::make_unique(std::move(hardwareEncoder), std::move(softwareEncoder)); + } + + return hardwareEncoder ? std::move(hardwareEncoder) : std::move(softwareEncoder); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp index 3c6a536d..2e2a4b12 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp @@ -31,7 +31,8 @@ namespace jni VideoEncoderFactoryWrapper::VideoEncoderFactoryWrapper(JNIEnv * env, jobject factory) : factory(env, factory), javaClass(JavaClasses::get(env)), - defaultFactory(CreateDefaultVideoEncoderFactory()) + defaultFactory(CreateDefaultVideoEncoderFactory()), + hardwareFactory(CreateHardwareVideoEncoderFactory()) { JavaLocalRef codecs(env, env->CallObjectMethod(factory, javaClass->getSupportedCodecs)); @@ -68,8 +69,10 @@ namespace jni if (env->IsInstanceOf(encoder, javaClass->nativeEncoderClass)) { JavaLocalRef nativeInfo(env, env->GetObjectField(encoder, javaClass->nativeEncoderCodecInfo)); + const bool hardware = env->GetBooleanField(encoder, javaClass->nativeEncoderHardwareAcceleration) == JNI_TRUE; + webrtc::VideoEncoderFactory & builtIn = hardware ? *hardwareFactory : *defaultFactory; - return defaultFactory->Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); + return builtIn.Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); } return std::make_unique(env, encoder, format); @@ -93,5 +96,6 @@ namespace jni nativeEncoderClass = FindClass(env, PKG_CODEC"NativeVideoEncoder"); nativeEncoderCodecInfo = GetFieldID(env, nativeEncoderClass, "codecInfo", "L" PKG_CODEC "VideoCodecInfo;"); + nativeEncoderHardwareAcceleration = GetFieldID(env, nativeEncoderClass, "hardwareAcceleration", "Z"); } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoEncoderFactory.cpp new file mode 100644 index 00000000..2f47f427 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoEncoderFactory.cpp @@ -0,0 +1,26 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/HardwareVideoEncoderFactory.h" + +namespace jni +{ + std::unique_ptr CreatePlatformHardwareVideoEncoderFactory() + { + // No hardware encoders on Linux yet: VA-API and NVENC are to come. + return nullptr; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFEncoderUtils.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFEncoderUtils.cpp new file mode 100644 index 00000000..88def1cb --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFEncoderUtils.cpp @@ -0,0 +1,69 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/windows/MFEncoderUtils.h" +#include "platform/windows/WinUtils.h" + +#include + +using Microsoft::WRL::ComPtr; + +namespace jni +{ + HRESULT EnumerateHardwareH264Encoders(std::vector> & encoders) + { + MFT_REGISTER_TYPE_INFO input = { MFMediaType_Video, MFVideoFormat_NV12 }; + MFT_REGISTER_TYPE_INFO output = { MFMediaType_Video, MFVideoFormat_H264 }; + + IMFActivate ** activates = nullptr; + UINT32 count = 0; + + // Sorted by merit, and restricted to what the system trusts. + HRESULT hr = MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, + &input, &output, &activates, &count); + + if (FAILED(hr)) { + return hr; + } + + for (UINT32 i = 0; i < count; i++) { + ComPtr activate; + activate.Attach(activates[i]); + + encoders.push_back(activate); + } + + CoTaskMemFree(activates); + + return S_OK; + } + + std::string GetTransformName(IMFActivate * activate) + { + LPWSTR name = nullptr; + UINT32 length = 0; + + if (FAILED(activate->GetAllocatedString(MFT_FRIENDLY_NAME_Attribute, &name, &length)) || name == nullptr) { + return "unknown"; + } + + std::string result = WideStrToStr(name); + + CoTaskMemFree(name); + + return result; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFH264Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFH264Encoder.cpp new file mode 100644 index 00000000..cfdaf464 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFH264Encoder.cpp @@ -0,0 +1,783 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/windows/MFH264Encoder.h" +#include "media/video/codec/windows/MFEncoderUtils.h" +#include "platform/windows/ComInitializer.h" + +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "common_video/h264/h264_common.h" +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "third_party/libyuv/include/libyuv/convert_from.h" + +#include +#include + +#include +#include +#include + +using Microsoft::WRL::ComPtr; + +namespace jni +{ + namespace + { + // How long a frame may wait for the transform to take input before + // it is dropped. + constexpr std::chrono::milliseconds kInputTimeout(20); + + // A key frame every 100 seconds at 30 fps. WebRTC asks for key + // frames whenever a receiver needs one, so they are rarely due to + // this. + constexpr UINT32 kGopSize = 3000; + + // The QP thresholds of WebRTC's own H.264 encoder. + constexpr int kLowH264QpThreshold = 24; + constexpr int kHighH264QpThreshold = 37; + + // Media Foundation counts time in 100 ns units. + constexpr LONGLONG kUnitsPerSecond = 10000000; + + // Initializes COM for the calling thread, once. Media Foundation + // transforms are free-threaded, so the apartment a thread already + // has does as well. + bool EnsureComInitialized() + { + thread_local std::unique_ptr initializer; + + if (!initializer) { + try { + initializer = std::make_unique(); + } + catch (...) { + return false; + } + } + + return true; + } + + void SetCodecValue(ICodecAPI * codecApi, const GUID & api, UINT32 value) + { + VARIANT variant; + VariantInit(&variant); + variant.vt = VT_UI4; + variant.ulVal = value; + + // Not every transform supports every setting; those it does + // not support keep their default. + HRESULT hr = codecApi->SetValue(&api, &variant); + + if (FAILED(hr)) { + RTC_LOG(LS_INFO) << "Media Foundation encoder: a codec setting is not supported, hr=" << hr; + } + } + + void SetCodecFlag(ICodecAPI * codecApi, const GUID & api, bool value) + { + VARIANT variant; + VariantInit(&variant); + variant.vt = VT_BOOL; + variant.boolVal = value ? VARIANT_TRUE : VARIANT_FALSE; + + HRESULT hr = codecApi->SetValue(&api, &variant); + + if (FAILED(hr)) { + RTC_LOG(LS_INFO) << "Media Foundation encoder: a codec flag is not supported, hr=" << hr; + } + } + } + + MFH264Encoder::MFH264Encoder(const webrtc::SdpVideoFormat & format) : + format(format), + implementationName("MediaFoundation"), + inputStreamId(0), + outputStreamId(0), + codecSettings(), + bitrateBps(0), + framerate(30), + callback(nullptr), + failed(false), + inputRequests(0), + lastSampleTime(-1), + keyFrameRequested(false), + packetizationMode(webrtc::H264PacketizationMode::NonInterleaved) + { + auto mode = format.parameters.find("packetization-mode"); + + if (mode == format.parameters.end() || mode->second != "1") { + packetizationMode = webrtc::H264PacketizationMode::SingleNalUnit; + } + } + + MFH264Encoder::~MFH264Encoder() + { + Release(); + } + + int32_t MFH264Encoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) + { + if (settings == nullptr || settings->width == 0 || settings->height == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (settings->numberOfSimulcastStreams > 1) { + return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; + } + // NV12 has chroma at half the resolution in both directions. + if (settings->width % 2 != 0 || settings->height % 2 != 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + Release(); + + if (!EnsureComInitialized()) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + try { + mfInitializer = std::make_unique(); + } + catch (...) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + codecSettings = *settings; + bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; + framerate = std::max(1u, codecSettings.maxFramerate); + + HRESULT hr = CreateTransform(); + + if (SUCCEEDED(hr)) { + hr = ConfigureTypes(); + } + if (SUCCEEDED(hr)) { + ConfigureCodec(); + + ComPtr generator; + hr = transform.As(&generator); + + if (SUCCEEDED(hr)) { + hr = MFTransformEvents::Start(generator.Get(), this, &events); + } + } + if (SUCCEEDED(hr)) { + hr = transform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0); + } + if (SUCCEEDED(hr)) { + hr = transform->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); + } + + if (FAILED(hr)) { + RTC_LOG(LS_WARNING) << "Media Foundation encoder failed to initialize, hr=" << hr; + + Release(); + + return WEBRTC_VIDEO_CODEC_ERROR; + } + + RTC_LOG(LS_INFO) << "Media Foundation encoder " << implementationName << " initialized: " + << codecSettings.width << "x" << codecSettings.height << " at " << bitrateBps << " bps"; + + return WEBRTC_VIDEO_CODEC_OK; + } + + HRESULT MFH264Encoder::CreateTransform() + { + std::vector> encoders; + + HRESULT hr = EnumerateHardwareH264Encoders(encoders); + + if (FAILED(hr)) { + return hr; + } + if (encoders.empty()) { + return MF_E_TOPO_CODEC_NOT_FOUND; + } + + // The first is the one the system ranks best. + activate = encoders.front(); + implementationName = "MediaFoundation (" + GetTransformName(activate.Get()) + ")"; + + hr = activate->ActivateObject(IID_PPV_ARGS(&transform)); + + if (FAILED(hr)) { + return hr; + } + + ComPtr attributes; + hr = transform->GetAttributes(&attributes); + + if (FAILED(hr)) { + return hr; + } + + UINT32 async = FALSE; + attributes->GetUINT32(MF_TRANSFORM_ASYNC, &async); + + // Hardware transforms are asynchronous, and that is all this + // encoder drives. + if (!async) { + return E_NOTIMPL; + } + + hr = attributes->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, TRUE); + + if (FAILED(hr)) { + return hr; + } + + attributes->SetUINT32(MF_LOW_LATENCY, TRUE); + + // Optional: without it, the encoder keeps its defaults. + transform.As(&codecApi); + + hr = transform->GetStreamIDs(1, &inputStreamId, 1, &outputStreamId); + + if (hr == E_NOTIMPL) { + // Fixed stream IDs, which start at 0. + inputStreamId = 0; + outputStreamId = 0; + hr = S_OK; + } + + return hr; + } + + HRESULT MFH264Encoder::ConfigureTypes() + { + const UINT32 width = codecSettings.width; + const UINT32 height = codecSettings.height; + + // An encoder takes its output type first. + ComPtr outputType; + HRESULT hr = MFCreateMediaType(&outputType); + + if (FAILED(hr)) { + return hr; + } + + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + outputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + outputType->SetUINT32(MF_MT_AVG_BITRATE, bitrateBps); + outputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + MFSetAttributeSize(outputType.Get(), MF_MT_FRAME_SIZE, width, height); + MFSetAttributeRatio(outputType.Get(), MF_MT_FRAME_RATE, framerate, 1); + MFSetAttributeRatio(outputType.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + + // Constrained Baseline is what peers expect, and is a subset of + // Baseline too. Older encoders only know Baseline, which peers + // decode as well, since WebRTC encoders use no Baseline-only tools. + outputType->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_ConstrainedBase); + + hr = transform->SetOutputType(outputStreamId, outputType.Get(), 0); + + if (FAILED(hr)) { + outputType->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base); + + hr = transform->SetOutputType(outputStreamId, outputType.Get(), 0); + } + if (FAILED(hr)) { + return hr; + } + + ComPtr inputType; + hr = MFCreateMediaType(&inputType); + + if (FAILED(hr)) { + return hr; + } + + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); + inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, width); + MFSetAttributeSize(inputType.Get(), MF_MT_FRAME_SIZE, width, height); + MFSetAttributeRatio(inputType.Get(), MF_MT_FRAME_RATE, framerate, 1); + MFSetAttributeRatio(inputType.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + + return transform->SetInputType(inputStreamId, inputType.Get(), 0); + } + + void MFH264Encoder::ConfigureCodec() + { + if (!codecApi) { + return; + } + + // One frame in, one frame out, and no frames that reference later + // ones: what real-time communication needs. + SetCodecFlag(codecApi.Get(), CODECAPI_AVLowLatencyMode, true); + SetCodecValue(codecApi.Get(), CODECAPI_AVEncMPVDefaultBPictureCount, 0); + SetCodecValue(codecApi.Get(), CODECAPI_AVEncMPVGOPSize, kGopSize); + SetCodecValue(codecApi.Get(), CODECAPI_AVEncCommonRateControlMode, eAVEncCommonRateControlMode_CBR); + SetCodecValue(codecApi.Get(), CODECAPI_AVEncCommonMeanBitRate, bitrateBps); + } + + int32_t MFH264Encoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) + { + callback = encodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t MFH264Encoder::Release() + { + ShutdownTransform(); + + mfInitializer.reset(); + + { + std::lock_guard lock(inputMutex); + inputRequests = 0; + } + { + std::lock_guard lock(pendingMutex); + pendingFrames.clear(); + } + + failed = false; + lastSampleTime = -1; + keyFrameRequested = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + void MFH264Encoder::ShutdownTransform() + { + // Detached first, so that no event handler touches the transform + // while it shuts down, nor this encoder afterwards. + if (events) { + events->Stop(); + events.Reset(); + } + + if (transform) { + transform->ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0); + transform->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); + } + + codecApi.Reset(); + transform.Reset(); + + if (activate) { + activate->ShutdownObject(); + activate.Reset(); + } + } + + int32_t MFH264Encoder::Encode(const webrtc::VideoFrame & frame, + const std::vector * frameTypes) + { + webrtc::EncodedImageCallback * encodeCallback = callback.load(); + + if (!transform || encodeCallback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (failed) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + if (frameTypes != nullptr) { + for (webrtc::VideoFrameType type : *frameTypes) { + if (type == webrtc::VideoFrameType::kVideoFrameKey) { + keyFrameRequested = true; + } + } + } + + { + std::unique_lock lock(inputMutex); + + inputRequested.wait_for(lock, kInputTimeout, [this] { + return inputRequests > 0 || failed; + }); + + if (failed) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + if (inputRequests == 0) { + // Still busy with earlier frames. A key frame asked for is + // kept for the next frame that gets through. + lock.unlock(); + + encodeCallback->OnFrameDropped(frame.rtp_timestamp(), 0, true); + + return WEBRTC_VIDEO_CODEC_OK; + } + + inputRequests--; + } + + if (keyFrameRequested && codecApi) { + SetCodecValue(codecApi.Get(), CODECAPI_AVEncVideoForceKeyFrame, 1); + } + keyFrameRequested = false; + + ComPtr sample; + HRESULT hr = CreateInputSample(frame, &sample); + + if (FAILED(hr)) { + RTC_LOG(LS_WARNING) << "Media Foundation encoder failed to create a sample, hr=" << hr; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + LONGLONG sampleTime = 0; + sample->GetSampleTime(&sampleTime); + + { + std::lock_guard lock(pendingMutex); + pendingFrames[sampleTime] = PendingFrame { + frame.rtp_timestamp(), + frame.render_time_ms(), + frame.ntp_time_ms(), + frame.rotation() + }; + } + + hr = transform->ProcessInput(inputStreamId, sample.Get(), 0); + + if (FAILED(hr)) { + RTC_LOG(LS_WARNING) << "Media Foundation encoder rejected a frame, hr=" << hr; + + { + std::lock_guard lock(pendingMutex); + pendingFrames.erase(sampleTime); + } + + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + return WEBRTC_VIDEO_CODEC_OK; + } + + HRESULT MFH264Encoder::CreateInputSample(const webrtc::VideoFrame & frame, IMFSample ** sample) + { + webrtc::scoped_refptr i420 = frame.video_frame_buffer()->ToI420(); + + if (!i420) { + return E_FAIL; + } + + const int width = static_cast(codecSettings.width); + const int height = static_cast(codecSettings.height); + + // WebRTC initializes the encoder again when the frame size changes, + // so a frame of another size is one that raced with that. + if (i420->width() != width || i420->height() != height) { + webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); + scaled->ScaleFrom(*i420); + i420 = scaled; + } + + const DWORD size = static_cast(width * height * 3 / 2); + + ComPtr buffer; + HRESULT hr = MFCreateMemoryBuffer(size, &buffer); + + if (FAILED(hr)) { + return hr; + } + + BYTE * data = nullptr; + hr = buffer->Lock(&data, nullptr, nullptr); + + if (FAILED(hr)) { + return hr; + } + + libyuv::I420ToNV12(i420->DataY(), i420->StrideY(), i420->DataU(), i420->StrideU(), + i420->DataV(), i420->StrideV(), data, width, data + width * height, width, width, height); + + buffer->Unlock(); + buffer->SetCurrentLength(size); + + ComPtr input; + hr = MFCreateSample(&input); + + if (FAILED(hr)) { + return hr; + } + + hr = input->AddBuffer(buffer.Get()); + + if (FAILED(hr)) { + return hr; + } + + // The sample time identifies the frame when its output arrives, so + // it has to be unique. + LONGLONG sampleTime = frame.timestamp_us() * 10; + + if (sampleTime <= lastSampleTime) { + sampleTime = lastSampleTime + 1; + } + + lastSampleTime = sampleTime; + + input->SetSampleTime(sampleTime); + input->SetSampleDuration(kUnitsPerSecond / framerate); + + *sample = input.Detach(); + + return S_OK; + } + + void MFH264Encoder::SetRates(const RateControlParameters & parameters) + { + const uint32_t bitrate = parameters.bitrate.get_sum_bps(); + + // A zero bitrate pauses the stream; the encoder gets no frames then. + if (bitrate == 0) { + return; + } + + bitrateBps = bitrate; + + if (parameters.framerate_fps >= 1.0) { + framerate = static_cast(parameters.framerate_fps + 0.5); + } + + if (codecApi) { + SetCodecValue(codecApi.Get(), CODECAPI_AVEncCommonMeanBitRate, bitrateBps); + } + } + + webrtc::VideoEncoder::EncoderInfo MFH264Encoder::GetEncoderInfo() const + { + EncoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + info.supports_native_handle = false; + info.supports_simulcast = false; + info.scaling_settings = ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold); + // NV12 needs even dimensions. + info.requested_resolution_alignment = 2; + + return info; + } + + void MFH264Encoder::OnTransformEvent(MediaEventType type, HRESULT status) + { + switch (type) { + case METransformNeedInput: + { + std::lock_guard lock(inputMutex); + inputRequests++; + } + inputRequested.notify_one(); + break; + + case METransformHaveOutput: + ProcessOutput(); + break; + + case MEError: + RTC_LOG(LS_WARNING) << "Media Foundation encoder failed, hr=" << status; + + failed = true; + inputRequested.notify_one(); + break; + + default: + break; + } + } + + void MFH264Encoder::ProcessOutput() + { + MFT_OUTPUT_STREAM_INFO info = {}; + HRESULT hr = transform->GetOutputStreamInfo(outputStreamId, &info); + + if (FAILED(hr)) { + failed = true; + return; + } + + ComPtr ownSample; + + if (!(info.dwFlags & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES))) { + ComPtr buffer; + + if (FAILED(MFCreateSample(&ownSample)) || FAILED(MFCreateMemoryBuffer(info.cbSize, &buffer)) || + FAILED(ownSample->AddBuffer(buffer.Get()))) + { + failed = true; + return; + } + } + + MFT_OUTPUT_DATA_BUFFER output = {}; + output.dwStreamID = outputStreamId; + output.pSample = ownSample.Get(); + + DWORD status = 0; + hr = transform->ProcessOutput(0, 1, &output, &status); + + if (output.pEvents != nullptr) { + output.pEvents->Release(); + } + + // A sample the transform provided is ours to release. + ComPtr sample; + + if (ownSample) { + sample = ownSample; + } + else if (output.pSample != nullptr) { + sample.Attach(output.pSample); + } + + if (hr == MF_E_TRANSFORM_STREAM_CHANGE) { + // The transform settled its output format; it has to be set + // again before output continues. + ComPtr type; + + if (FAILED(transform->GetOutputAvailableType(outputStreamId, 0, &type)) || + FAILED(transform->SetOutputType(outputStreamId, type.Get(), 0))) + { + failed = true; + } + return; + } + if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) { + return; + } + if (FAILED(hr) || !sample) { + RTC_LOG(LS_WARNING) << "Media Foundation encoder failed to produce output, hr=" << hr; + + failed = true; + return; + } + + DeliverOutput(sample.Get()); + } + + void MFH264Encoder::DeliverOutput(IMFSample * sample) + { + LONGLONG sampleTime = 0; + sample->GetSampleTime(&sampleTime); + + UINT32 cleanPoint = FALSE; + bool keyFrame = SUCCEEDED(sample->GetUINT32(MFSampleExtension_CleanPoint, &cleanPoint)) && cleanPoint; + + ComPtr buffer; + BYTE * data = nullptr; + DWORD length = 0; + + if (FAILED(sample->ConvertToContiguousBuffer(&buffer)) || FAILED(buffer->Lock(&data, nullptr, &length))) { + failed = true; + return; + } + + std::span bitstream(data, length); + + // Encoders put the parameter sets in front of the first key frame, + // not necessarily in front of every one, but a receiver that joins + // later needs them with the key frame it starts from. + std::vector parameterSetsFound; + bool hasSps = false; + bool hasPps = false; + + for (const webrtc::H264::NaluIndex & nalu : webrtc::H264::FindNaluIndices(bitstream)) { + const size_t end = nalu.payload_start_offset + nalu.payload_size; + const webrtc::H264::NaluType type = webrtc::H264::ParseNaluType(data[nalu.payload_start_offset]); + + if (type == webrtc::H264::NaluType::kSps || type == webrtc::H264::NaluType::kPps) { + hasSps |= type == webrtc::H264::NaluType::kSps; + hasPps |= type == webrtc::H264::NaluType::kPps; + + parameterSetsFound.insert(parameterSetsFound.end(), data + nalu.start_offset, data + end); + } + else if (type == webrtc::H264::NaluType::kIdr) { + keyFrame = true; + } + } + + if (hasSps && hasPps) { + parameterSets = std::move(parameterSetsFound); + } + + const bool prependParameterSets = keyFrame && !hasSps && !parameterSets.empty(); + const size_t size = length + (prependParameterSets ? parameterSets.size() : 0); + + webrtc::scoped_refptr encoded = webrtc::EncodedImageBuffer::Create(size); + uint8_t * target = encoded->data(); + + if (prependParameterSets) { + std::copy(parameterSets.begin(), parameterSets.end(), target); + target += parameterSets.size(); + } + + std::copy(data, data + length, target); + + buffer->Unlock(); + + PendingFrame pending; + + { + std::lock_guard lock(pendingMutex); + + auto found = pendingFrames.find(sampleTime); + + if (found == pendingFrames.end()) { + RTC_LOG(LS_WARNING) << "Media Foundation encoder produced a frame for no input, time " << sampleTime; + return; + } + + pending = found->second; + + // Frames the encoder skipped come out never. + pendingFrames.erase(pendingFrames.begin(), std::next(found)); + } + + webrtc::EncodedImage image; + image.SetEncodedData(encoded); + image._encodedWidth = codecSettings.width; + image._encodedHeight = codecSettings.height; + image.SetRtpTimestamp(pending.rtpTimestamp); + image.capture_time_ms_ = pending.captureTimeMs; + image.ntp_time_ms_ = pending.ntpTimeMs; + image.rotation_ = pending.rotation; + image.set_frame_type(keyFrame + ? webrtc::VideoFrameType::kVideoFrameKey + : webrtc::VideoFrameType::kVideoFrameDelta); + + bitstreamParser.ParseBitstream(std::span(encoded->data(), encoded->size())); + image.qp_ = bitstreamParser.GetLastSliceQp().value_or(-1); + + webrtc::CodecSpecificInfo info; + info.codecType = webrtc::kVideoCodecH264; + info.codecSpecific.H264.packetization_mode = packetizationMode; + info.codecSpecific.H264.temporal_idx = webrtc::kNoTemporalIdx; + info.codecSpecific.H264.base_layer_sync = false; + info.codecSpecific.H264.idr_frame = keyFrame; + + webrtc::EncodedImageCallback * encodeCallback = callback.load(); + + if (encodeCallback != nullptr) { + encodeCallback->OnEncodedImage(image, &info); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFTransformEvents.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFTransformEvents.cpp new file mode 100644 index 00000000..68bb6bee --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFTransformEvents.cpp @@ -0,0 +1,127 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/windows/MFTransformEvents.h" + +using Microsoft::WRL::ComPtr; + +namespace jni +{ + HRESULT MFTransformEvents::Start(IMFMediaEventGenerator * generator, MFTransformEventListener * listener, + ComPtr * events) + { + ComPtr instance; + instance.Attach(new MFTransformEvents(generator, listener)); + + HRESULT hr = generator->BeginGetEvent(instance.Get(), nullptr); + + if (SUCCEEDED(hr)) { + *events = instance; + } + + return hr; + } + + MFTransformEvents::MFTransformEvents(IMFMediaEventGenerator * generator, MFTransformEventListener * listener) : + references(1), + generator(generator), + listener(listener) + { + } + + void MFTransformEvents::Stop() + { + std::lock_guard lock(mutex); + + listener = nullptr; + } + + STDMETHODIMP MFTransformEvents::QueryInterface(REFIID iid, void ** object) + { + if (object == nullptr) { + return E_POINTER; + } + + if (iid == __uuidof(IUnknown) || iid == __uuidof(IMFAsyncCallback)) { + *object = static_cast(this); + AddRef(); + + return S_OK; + } + + *object = nullptr; + + return E_NOINTERFACE; + } + + STDMETHODIMP_(ULONG) MFTransformEvents::AddRef() + { + return ++references; + } + + STDMETHODIMP_(ULONG) MFTransformEvents::Release() + { + ULONG count = --references; + + if (count == 0) { + delete this; + } + + return count; + } + + STDMETHODIMP MFTransformEvents::GetParameters(DWORD * flags, DWORD * queue) + { + // The default work queue. + return E_NOTIMPL; + } + + STDMETHODIMP MFTransformEvents::Invoke(IMFAsyncResult * result) + { + ComPtr event; + + // Fails with MF_E_SHUTDOWN once the transform is shut down, which + // ends the pump. + HRESULT hr = generator->EndGetEvent(result, &event); + + if (FAILED(hr)) { + return S_OK; + } + + MediaEventType type = MEUnknown; + HRESULT status = S_OK; + + event->GetType(&type); + event->GetStatus(&status); + + std::lock_guard lock(mutex); + + if (listener == nullptr) { + return S_OK; + } + + listener->OnTransformEvent(type, status); + + // Asks for the next event only while someone listens. + hr = generator->BeginGetEvent(this, nullptr); + + if (FAILED(hr)) { + listener->OnTransformEvent(MEError, hr); + } + + return S_OK; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoderFactory.cpp new file mode 100644 index 00000000..dad259c3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoderFactory.cpp @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/windows/MFVideoEncoderFactory.h" +#include "media/video/codec/windows/MFEncoderUtils.h" +#include "media/video/codec/windows/MFH264Encoder.h" +#include "media/video/codec/HardwareVideoEncoderFactory.h" +#include "platform/windows/ComInitializer.h" +#include "platform/windows/MFInitializer.h" + +#include "api/video_codecs/h264_profile_level_id.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "rtc_base/logging.h" + +using Microsoft::WRL::ComPtr; + +namespace jni +{ + std::unique_ptr MFVideoEncoderFactory::Create() + { + std::vector> encoders; + + try { + ComInitializer comInitializer; + MFInitializer mfInitializer; + + if (FAILED(EnumerateHardwareH264Encoders(encoders))) { + return nullptr; + } + } + catch (...) { + return nullptr; + } + + if (encoders.empty()) { + RTC_LOG(LS_INFO) << "No Media Foundation hardware H.264 encoder found"; + + return nullptr; + } + + RTC_LOG(LS_INFO) << "Media Foundation hardware H.264 encoder: " << GetTransformName(encoders.front().Get()); + + return std::unique_ptr(new MFVideoEncoderFactory()); + } + + std::vector MFVideoEncoderFactory::GetSupportedFormats() const + { + return { + webrtc::CreateH264Format(webrtc::H264Profile::kProfileConstrainedBaseline, webrtc::H264Level::kLevel3_1, "1"), + webrtc::CreateH264Format(webrtc::H264Profile::kProfileBaseline, webrtc::H264Level::kLevel3_1, "1") + }; + } + + std::unique_ptr MFVideoEncoderFactory::Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) + { + return std::make_unique(format); + } + + std::unique_ptr CreatePlatformHardwareVideoEncoderFactory() + { + return MFVideoEncoderFactory::Create(); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java index 3bae9f79..192f2039 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java @@ -31,6 +31,9 @@ * WebRTC's default factories, with H.264 through VideoToolbox. * {@link #getSupportedCodecs()} lists what the platform has. *

+ * To encode on the GPU where the platform supports it, use a {@link + * HardwareVideoEncoderFactory} instead. + *

* A factory of one's own can hand out these encoders next to its own ones * by delegating to an instance of this class. The encoders it creates are * {@link NativeVideoEncoder}s, which run inside WebRTC and are not to be @@ -80,7 +83,7 @@ public NativeVideoEncoder createEncoder(VideoCodecInfo info) { for (VideoCodecInfo codec : supportedCodecs) { if (codec.getName().equalsIgnoreCase(info.getName())) { - return new NativeVideoEncoder(info); + return new NativeVideoEncoder(info, false); } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java new file mode 100644 index 00000000..f3388a1c --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -0,0 +1,104 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.internal.NativeLoader; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * The built-in video encoders, with the encoders of the GPU in front of them + * where the platform has them. Set it on a {@link + * dev.onvoid.webrtc.PeerConnectionFactory PeerConnectionFactory} to encode in + * hardware: + * 

{@code
+ * PeerConnectionFactory factory = PeerConnectionFactory.builder()
+ *     .setVideoEncoderFactory(new HardwareVideoEncoderFactory())
+ *     .build();
+ * }
+ *

+ * On Windows, H.264 is encoded by the Media Foundation encoder of the GPU + * driver. A hardware encoder that fails to start, for example because the GPU + * has no encoder sessions left, or fails while encoding, is replaced by the + * software encoder of the same codec, so a stream keeps going. Where there is + * no hardware encoder, this factory encodes like a {@link + * DefaultVideoEncoderFactory}. On macOS, that already uses VideoToolbox. + * Linux has no hardware encoders yet. + *

+ * The hardware encoders take over only codecs the software encoders have too, + * so the supported codecs are the same as those of a {@link + * DefaultVideoEncoderFactory}, and what a peer connection negotiates does not + * depend on the GPU. + * + * @author Alex Andres + */ +public final class HardwareVideoEncoderFactory implements VideoEncoderFactory { + + static { + try { + NativeLoader.loadLibrary("webrtc-java"); + } + catch (Exception e) { + throw new RuntimeException("Load library 'webrtc-java' failed", e); + } + } + + + private final List supportedCodecs; + + + /** + * Creates a factory for the hardware video encoders. + */ + public HardwareVideoEncoderFactory() { + supportedCodecs = Collections.unmodifiableList( + Arrays.asList(getSupportedCodecsInternal())); + } + + @Override + public List getSupportedCodecs() { + return supportedCodecs; + } + + /** + * Returns a built-in encoder for the given codec, encoding on the GPU if + * it can. + * + * @param info The codec to encode, as negotiated. + * + * @return A native encoder, or {@code null} if there is no built-in + * encoder for the codec. + */ + @Override + public NativeVideoEncoder createEncoder(VideoCodecInfo info) { + Objects.requireNonNull(info, "VideoCodecInfo is null"); + + for (VideoCodecInfo codec : supportedCodecs) { + if (codec.getName().equalsIgnoreCase(info.getName())) { + return new NativeVideoEncoder(info, true); + } + } + + return null; + } + + private static native VideoCodecInfo[] getSupportedCodecsInternal(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java index 1bef6583..280c4d92 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java @@ -19,10 +19,10 @@ import dev.onvoid.webrtc.media.video.VideoFrame; /** - * A built-in encoder, created by a {@link DefaultVideoEncoderFactory}. It is a - * placeholder: returned from a {@link VideoEncoderFactory}, it makes WebRTC - * create the native encoder for its codec, which then runs entirely inside - * WebRTC. Its methods are therefore not to be called from Java, and throw + * A built-in encoder, created by a {@link DefaultVideoEncoderFactory} or a + * {@link HardwareVideoEncoderFactory}. It is a placeholder: returned from a + * {@link VideoEncoderFactory}, it makes WebRTC create the native encoder for + * its codec, which then runs entirely inside WebRTC. Its methods are therefore not to be called from Java, and throw * {@link UnsupportedOperationException}. * * @author Alex Andres @@ -32,9 +32,13 @@ public final class NativeVideoEncoder implements VideoEncoder { /** The codec to create the native encoder for; read by native code. */ private final VideoCodecInfo codecInfo; + /** Whether the encoder may use the GPU; read by native code. */ + private final boolean hardwareAcceleration; - NativeVideoEncoder(VideoCodecInfo codecInfo) { + + NativeVideoEncoder(VideoCodecInfo codecInfo, boolean hardwareAcceleration) { this.codecInfo = codecInfo; + this.hardwareAcceleration = hardwareAcceleration; } /** diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java new file mode 100644 index 00000000..e71f4b0a --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -0,0 +1,151 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; + +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests the hardware encoders of the {@link HardwareVideoEncoderFactory}. + *

+ * A machine without a hardware encoder skips the tests that need one, as CI + * runners do. Set the system property {@code webrtc.test.hardwareEncoder} to + * {@code true} on a machine that has one, to make those tests fail instead. + */ +@Execution(ExecutionMode.SAME_THREAD) +class HardwareVideoEncoderIntegrationTest extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + private static final boolean HARDWARE_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareEncoder"); + + private static final String OS = System.getProperty("os.name").toLowerCase(Locale.ROOT); + + /** + * H.264 Constrained Baseline with packetization mode 1, which hardware + * encoders take over. + */ + private static final Predicate H264 = codec -> + "H264".equalsIgnoreCase(codec.getName()) + && "1".equals(codec.getSDPFmtp().get("packetization-mode")) + && codec.getSDPFmtp().getOrDefault("profile-level-id", "").startsWith("42e0"); + + + @Test + void hardwareKeepsCodecs() { + // Hardware encoding must not change what is negotiated. + assertEquals(new DefaultVideoEncoderFactory().getSupportedCodecs(), + new HardwareVideoEncoderFactory().getSupportedCodecs()); + } + + @Test + void hardwareEncodesH264() throws Exception { + assumeTrue(OS.contains("win"), "hardware encoders are implemented on Windows only"); + + PeerConnectionFactory hardware = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .build(); + + try { + String implementation = encoderImplementation(hardware); + + if (HARDWARE_REQUIRED) { + assertTrue(implementation.contains("MediaFoundation"), implementation); + } + else { + assumeTrue(implementation.contains("MediaFoundation"), "no hardware encoder: " + implementation); + } + } + finally { + hardware.dispose(); + } + } + + @Test + void defaultEncodesH264InSoftware() throws Exception { + assumeFalse(OS.contains("mac"), "macOS encodes H.264 through VideoToolbox by default"); + + // The shared factory uses the default encoders. + String implementation = encoderImplementation(factory); + + assertTrue(implementation.contains("OpenH264"), implementation); + } + + /** + * Sends H.264 through a call until frames arrive, and returns what the + * sender reports its encoder to be. + */ + private static String encoderImplementation(PeerConnectionFactory factory) throws Exception { + CountDownLatch received = new CountDownLatch(10); + + try (TestMediaCall call = new TestMediaCall(factory, true, false, H264)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); + + // The statistics catch up with the encoder shortly after. + Object implementation = null; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (implementation == null && System.nanoTime() < deadline) { + Map outbound = call.getOutboundVideoStats(); + + if (outbound != null) { + implementation = outbound.get("encoderImplementation"); + } + if (implementation == null) { + Thread.sleep(100); + } + } + + track.removeSink(sink); + receiver.dispose(); + + return String.valueOf(implementation); + } + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java index 9db91195..dc9e6af5 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java @@ -21,6 +21,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import dev.onvoid.webrtc.media.MediaStreamTrack; import dev.onvoid.webrtc.media.MediaType; @@ -69,6 +74,16 @@ class TestMediaCall implements AutoCloseable { * @param videoCodec The name of the video codec to prefer. */ TestMediaCall(PeerConnectionFactory factory, boolean video, boolean audio, String videoCodec) { + this(factory, video, audio, codec -> videoCodec.equalsIgnoreCase(codec.getName())); + } + + /** + * Creates a call whose video prefers the codecs the given filter accepts. + * + * @param videoCodec Accepts the video codecs to prefer. + */ + TestMediaCall(PeerConnectionFactory factory, boolean video, boolean audio, + Predicate videoCodec) { caller = new TestPeerConnection(factory); callee = new TestPeerConnection(factory); @@ -129,6 +144,28 @@ void startMedia() { feeder.start(); } + /** + * Returns the caller's statistics of its outbound video stream, or null + * if there are none yet. + */ + Map getOutboundVideoStats() throws InterruptedException { + CountDownLatch done = new CountDownLatch(1); + AtomicReference> outbound = new AtomicReference<>(); + + caller.getPeerConnection().getStats(videoSender, report -> { + for (RTCStats stats : report.getStats().values()) { + if (stats.getType() == RTCStatsType.OUTBOUND_RTP) { + outbound.set(stats.getAttributes()); + } + } + done.countDown(); + }); + + done.await(5, TimeUnit.SECONDS); + + return outbound.get(); + } + RTCRtpSender getVideoSender() { return videoSender; } @@ -208,11 +245,11 @@ public void close() throws InterruptedException { * splits frames at start codes in the payload. */ private static void preferCodec(PeerConnectionFactory factory, RTCPeerConnection connection, - RTCRtpSender sender, String name) { + RTCRtpSender sender, Predicate preferred) { List codecs = new ArrayList<>( factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()); - codecs.sort(Comparator.comparing(codec -> !name.equalsIgnoreCase(codec.getName()))); + codecs.sort(Comparator.comparing(codec -> !preferred.test(codec))); for (RTCRtpTransceiver transceiver : connection.getTransceivers()) { RTCRtpSender transceiverSender = transceiver.getSender();