From 0f33b6dc82b1623cbfbe004182a2ec29ccb98b16 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Wed, 30 Sep 2026 18:58:32 +0200 Subject: [PATCH] feat: map the remaining sender encoding parameters RTCRtpEncodingParameters gains the encoding parameters WebRTC has and the Java API did not: rid for simulcast, scaleResolutionDownTo (with the new RTCResolutionRestriction), scalabilityMode and numTemporalLayers for SVC, bitratePriority and networkPriority, adaptivePtime for audio, and codec to send an encoding with a codec of its own. RTCRtpSendParameters gains degradationPreference, with the new RTCDegradationPreference. RTCRtpCodecCapability gains getScalabilityModes(), the modes a codec supports, which is what scalabilityMode accepts; unsupported modes make setParameters() fail. The priorities apply to the whole sender, and WebRTC accepts them on the first encoding only, which the Javadoc says. The media constraints guide covers the new parameters, with simulcast and scalability modes. --- docs/guide/media/constraints.md | 83 ++++++ .../include/api/RTCResolutionRestriction.h | 47 +++ .../cpp/include/api/RTCRtpCodecCapability.h | 1 + .../include/api/RTCRtpEncodingParameters.h | 8 + .../cpp/include/api/RTCRtpSendParameters.h | 1 + webrtc-jni/src/main/cpp/src/WebRTCContext.cpp | 4 + .../cpp/src/api/RTCResolutionRestriction.cpp | 62 ++++ .../cpp/src/api/RTCRtpCodecCapability.cpp | 29 ++ .../cpp/src/api/RTCRtpEncodingParameters.cpp | 70 +++++ .../main/cpp/src/api/RTCRtpSendParameters.cpp | 12 + .../webrtc/RTCDegradationPreference.java | 51 ++++ .../webrtc/RTCResolutionRestriction.java | 60 ++++ .../onvoid/webrtc/RTCRtpCodecCapability.java | 24 +- .../webrtc/RTCRtpEncodingParameters.java | 78 ++++- .../onvoid/webrtc/RTCRtpSendParameters.java | 12 +- .../webrtc/RTCRtpEncodingParametersTests.java | 273 ++++++++++++++++++ 16 files changed, 807 insertions(+), 8 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/api/RTCResolutionRestriction.h create mode 100644 webrtc-jni/src/main/cpp/src/api/RTCResolutionRestriction.cpp create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/RTCDegradationPreference.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/RTCResolutionRestriction.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpEncodingParametersTests.java diff --git a/docs/guide/media/constraints.md b/docs/guide/media/constraints.md index fb3b32e8a..532be7a1d 100644 --- a/docs/guide/media/constraints.md +++ b/docs/guide/media/constraints.md @@ -113,6 +113,89 @@ The `scaleResolutionDownBy` parameter specifies how much to scale down the video Note that these constraints are applied without requiring SDP renegotiation, making them suitable for dynamic adaptation to changing network conditions. ::: +To cap the resolution at an absolute size instead, set `scaleResolutionDownTo`. The video is scaled down to fit, whatever size the source delivers; it takes precedence over `scaleResolutionDownBy` if both are set. + +```java +parameters.encodings.get(0).scaleResolutionDownTo = new RTCResolutionRestriction(1280, 720); +sender.setParameters(parameters); +``` + +## Degradation Preference + +When the network or the CPU cannot keep up, a video sender lowers either its frame rate or its resolution. `degradationPreference` on the send parameters decides which: + +```java +RTCRtpSendParameters parameters = sender.getParameters(); + +// Keep text sharp when sharing a screen: lower the frame rate instead. +parameters.degradationPreference = RTCDegradationPreference.MAINTAIN_RESOLUTION; + +sender.setParameters(parameters); +``` + +| Value | Gives up | +|---|---| +| `MAINTAIN_FRAMERATE` | Resolution; suits motion, such as camera video | +| `MAINTAIN_RESOLUTION` | Frame rate; suits detail, such as screens with text | +| `BALANCED` | Both, in turns | +| `MAINTAIN_FRAMERATE_AND_RESOLUTION` | Neither; frames are dropped instead | + +If unset, WebRTC keeps the resolution of screen shares and of tracks hinted as detailed or text, and the frame rate of other video. + +## Simulcast + +With simulcast, a video sender encodes the same video several times, at different resolutions, so that a server can forward to each receiver the one its connection can take. The encodings are given when the transceiver is created, each with an RTP stream ID (`rid`): + +```java +RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); +init.direction = RTCRtpTransceiverDirection.SEND_ONLY; + +RTCRtpEncodingParameters quarter = new RTCRtpEncodingParameters(); +quarter.rid = "q"; +quarter.scaleResolutionDownBy = 4.0; +quarter.maxBitrate = 150_000; + +RTCRtpEncodingParameters half = new RTCRtpEncodingParameters(); +half.rid = "h"; +half.scaleResolutionDownBy = 2.0; +half.maxBitrate = 500_000; + +RTCRtpEncodingParameters full = new RTCRtpEncodingParameters(); +full.rid = "f"; +full.maxBitrate = 1_500_000; + +init.sendEncodings = Arrays.asList(quarter, half, full); + +RTCRtpTransceiver transceiver = peerConnection.addTransceiver(videoTrack, init); +``` + +The `rid`s cannot be changed afterwards; the other parameters of each encoding can, through `setParameters()`, e.g. `active = false` to pause one of them. An encoding may also send a different codec than the others, by setting its `codec` to one of the negotiated codecs. + +## Scalability Modes (SVC) + +Instead of separate encodings, one encoding can carry layers, which a server can drop to lower the rate: temporal layers (fewer frames per second) and, with VP9 and AV1, spatial layers (lower resolutions). `scalabilityMode` takes the mode names of the [WebRTC SVC specification](https://www.w3.org/TR/webrtc-svc/), e.g. `L1T3` for three temporal layers, or `L3T3_KEY` for three spatial layers with three temporal layers each: + +```java +RTCRtpSendParameters parameters = sender.getParameters(); +parameters.encodings.get(0).scalabilityMode = "L1T3"; +sender.setParameters(parameters); +``` + +The codec has to support the mode; `setParameters()` throws otherwise. `getScalabilityModes()` of a codec's capability, from `factory.getRtpSenderCapabilities(MediaType.VIDEO)`, lists the modes it supports. VP8 supports temporal layers only. + +## Priorities + +`bitratePriority` sets the share of the available bitrate a sender gets relative to the other senders of the peer connection (default 1.0), and `networkPriority` how its packets are marked on the network (DSCP), where the network honors it. Both apply to the whole sender, so only the first encoding may set them: + +```java +RTCRtpSendParameters parameters = sender.getParameters(); +parameters.encodings.get(0).bitratePriority = 2.0; +parameters.encodings.get(0).networkPriority = RTCPriorityType.HIGH; +sender.setParameters(parameters); +``` + +For audio, `adaptivePtime` lets the encoder send longer packets when the bitrate is low, which saves the overhead of many small packets. + ## Conclusion In this guide, we've explored several important techniques for controlling media quality and bandwidth usage in WebRTC applications. diff --git a/webrtc-jni/src/main/cpp/include/api/RTCResolutionRestriction.h b/webrtc-jni/src/main/cpp/include/api/RTCResolutionRestriction.h new file mode 100644 index 000000000..d13598012 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/RTCResolutionRestriction.h @@ -0,0 +1,47 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_RTC_RESOLUTION_RESTRICTION_H_ +#define JNI_WEBRTC_API_RTC_RESOLUTION_RESTRICTION_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/resolution.h" + +#include + +namespace jni +{ + namespace RTCResolutionRestriction + { + class JavaRTCResolutionRestrictionClass : public JavaClass + { + public: + explicit JavaRTCResolutionRestrictionClass(JNIEnv * env); + + jclass cls; + jmethodID ctor; + jfieldID maxWidth; + jfieldID maxHeight; + }; + + JavaLocalRef toJava(JNIEnv * env, const webrtc::Resolution & resolution); + webrtc::Resolution toNative(JNIEnv * env, const JavaRef & restriction); + } +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpCodecCapability.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpCodecCapability.h index 7a8dc6264..94b4c5dba 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCRtpCodecCapability.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpCodecCapability.h @@ -40,6 +40,7 @@ namespace jni jfieldID clockRate; jfieldID channels; jfieldID sdpFmtp; + jfieldID scalabilityModes; }; JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpCodecCapability & capability); diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpEncodingParameters.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpEncodingParameters.h index 6e2c4d47e..16da7121d 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCRtpEncodingParameters.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpEncodingParameters.h @@ -41,6 +41,14 @@ namespace jni jfieldID maxBitrate; jfieldID maxFramerate; jfieldID scaleResolution; + jfieldID rid; + jfieldID scaleResolutionDownTo; + jfieldID scalabilityMode; + jfieldID numTemporalLayers; + jfieldID bitratePriority; + jfieldID networkPriority; + jfieldID adaptivePtime; + jfieldID codec; }; JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpEncodingParameters & parameters); diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpSendParameters.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpSendParameters.h index 61ab494e9..ed6fbe086 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCRtpSendParameters.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpSendParameters.h @@ -37,6 +37,7 @@ namespace jni jmethodID ctor; jfieldID transactionId; jfieldID encodings; + jfieldID degradationPreference; }; JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpParameters & parameters); diff --git a/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp b/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp index 165411e03..699cf584c 100644 --- a/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp +++ b/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp @@ -27,6 +27,8 @@ #include "api/environment/environment_factory.h" #include "api/peer_connection_interface.h" +#include "api/priority.h" +#include "api/rtp_parameters.h" #include "modules/desktop_capture/desktop_capturer.h" #include "rtc_base/ssl_adapter.h" @@ -83,6 +85,8 @@ namespace jni JavaEnums::add(env, PKG"RTCSignalingState"); JavaEnums::add(env, PKG"TlsCertPolicy"); JavaEnums::add(env, PKG"RTCRtpTransceiverDirection"); + JavaEnums::add(env, PKG"RTCPriorityType"); + JavaEnums::add(env, PKG"RTCDegradationPreference"); JavaEnums::add(env, PKG"RTCSdpType"); JavaEnums::add(env, PKG_AUDIO"AudioLayer"); JavaEnums::add(env, PKG_AUDIO"AudioProcessingConfig$GainController$Mode"); diff --git a/webrtc-jni/src/main/cpp/src/api/RTCResolutionRestriction.cpp b/webrtc-jni/src/main/cpp/src/api/RTCResolutionRestriction.cpp new file mode 100644 index 000000000..7c45ecb69 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/RTCResolutionRestriction.cpp @@ -0,0 +1,62 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/RTCResolutionRestriction.h" +#include "JavaClasses.h" +#include "JavaObject.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +namespace jni +{ + namespace RTCResolutionRestriction + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::Resolution & resolution) + { + const auto javaClass = JavaClasses::get(env); + + jobject object = env->NewObject(javaClass->cls, javaClass->ctor, + static_cast(resolution.width), static_cast(resolution.height)); + + ExceptionCheck(env); + + return JavaLocalRef(env, object); + } + + webrtc::Resolution toNative(JNIEnv * env, const JavaRef & restriction) + { + const auto javaClass = JavaClasses::get(env); + + JavaObject obj(env, restriction); + + webrtc::Resolution resolution; + resolution.width = obj.getInt(javaClass->maxWidth); + resolution.height = obj.getInt(javaClass->maxHeight); + + return resolution; + } + + JavaRTCResolutionRestrictionClass::JavaRTCResolutionRestrictionClass(JNIEnv * env) + { + cls = FindClass(env, PKG"RTCResolutionRestriction"); + + ctor = GetMethod(env, cls, "", "(II)V"); + + maxWidth = GetFieldID(env, cls, "maxWidth", "I"); + maxHeight = GetFieldID(env, cls, "maxHeight", "I"); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpCodecCapability.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpCodecCapability.cpp index b9b386e63..c1ee4ba4f 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCRtpCodecCapability.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpCodecCapability.cpp @@ -23,6 +23,12 @@ #include "JavaUtils.h" #include "JNI_WebRTC.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalability_mode_util.h" + +#include +#include + namespace jni { namespace RTCRtpCodecCapability @@ -59,6 +65,16 @@ namespace jni ExceptionCheck(env); + if (!capability.scalability_modes.empty()) { + std::vector modes; + + for (webrtc::ScalabilityMode mode : capability.scalability_modes) { + modes.emplace_back(webrtc::ScalabilityModeToString(mode)); + } + + env->SetObjectField(object, javaClass->scalabilityModes, JavaString::createArray(env, modes).get()); + } + return JavaLocalRef(env, object); } @@ -90,6 +106,18 @@ namespace jni codecCapability.num_channels = Integer::getValue(env, channels); } + JavaLocalRef modes = obj.getObjectArray(javaClass->scalabilityModes); + const jsize count = modes.get() != nullptr ? env->GetArrayLength(modes.get()) : 0; + + for (jsize i = 0; i < count; i++) { + JavaLocalRef mode(env, static_cast(env->GetObjectArrayElement(modes.get(), i))); + auto scalabilityMode = webrtc::ScalabilityModeFromString(JavaString::toNative(env, mode)); + + if (scalabilityMode.has_value()) { + codecCapability.scalability_modes.push_back(*scalabilityMode); + } + } + return codecCapability; } @@ -104,6 +132,7 @@ namespace jni clockRate = GetFieldID(env, cls, "clockRate", INTEGER_SIG); channels = GetFieldID(env, cls, "channels", INTEGER_SIG); sdpFmtp = GetFieldID(env, cls, "sdpFmtp", MAP_SIG); + scalabilityModes = GetFieldID(env, cls, "scalabilityModes", "[" STRING_SIG); } } } diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpEncodingParameters.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpEncodingParameters.cpp index 4efca1260..55ca011fb 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCRtpEncodingParameters.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpEncodingParameters.cpp @@ -15,11 +15,14 @@ */ #include "api/RTCRtpEncodingParameters.h" +#include "api/RTCResolutionRestriction.h" +#include "api/RTCRtpCodecCapability.h" #include "JavaClasses.h" #include "JavaEnums.h" #include "JavaObject.h" #include "JavaPrimitive.h" #include "JavaRef.h" +#include "JavaString.h" #include "JNI_WebRTC.h" namespace jni @@ -48,6 +51,33 @@ namespace jni if (parameters.scale_resolution_down_by.has_value()) { env->SetObjectField(object, javaClass->scaleResolution, Double::create(env, parameters.scale_resolution_down_by.value())); } + if (!parameters.rid.empty()) { + env->SetObjectField(object, javaClass->rid, JavaString::toJava(env, parameters.rid).get()); + } + if (parameters.scale_resolution_down_to.has_value()) { + env->SetObjectField(object, javaClass->scaleResolutionDownTo, + RTCResolutionRestriction::toJava(env, *parameters.scale_resolution_down_to).get()); + } + if (parameters.scalability_mode.has_value()) { + env->SetObjectField(object, javaClass->scalabilityMode, + JavaString::toJava(env, *parameters.scalability_mode).get()); + } + if (parameters.num_temporal_layers.has_value()) { + env->SetObjectField(object, javaClass->numTemporalLayers, + Integer::create(env, *parameters.num_temporal_layers).get()); + } + if (parameters.codec.has_value()) { + // The Java type of a codec is the capability, which adds to + // the codec what the encoding does not carry. + webrtc::RtpCodecCapability capability; + static_cast(capability) = *parameters.codec; + + env->SetObjectField(object, javaClass->codec, RTCRtpCodecCapability::toJava(env, capability).get()); + } + + env->SetObjectField(object, javaClass->bitratePriority, Double::create(env, parameters.bitrate_priority).get()); + env->SetObjectField(object, javaClass->networkPriority, JavaEnums::toJava(env, parameters.network_priority).get()); + env->SetObjectField(object, javaClass->adaptivePtime, Boolean::create(env, parameters.adaptive_ptime).get()); return JavaLocalRef(env, object); } @@ -64,6 +94,14 @@ namespace jni auto maxBitrate = obj.getObject(javaClass->maxBitrate); auto maxFramerate = obj.getObject(javaClass->maxFramerate); auto scaleResolution = obj.getObject(javaClass->scaleResolution); + auto rid = obj.getString(javaClass->rid); + auto scaleResolutionDownTo = obj.getObject(javaClass->scaleResolutionDownTo); + auto scalabilityMode = obj.getString(javaClass->scalabilityMode); + auto numTemporalLayers = obj.getObject(javaClass->numTemporalLayers); + auto bitratePriority = obj.getObject(javaClass->bitratePriority); + auto networkPriority = obj.getObject(javaClass->networkPriority); + auto adaptivePtime = obj.getObject(javaClass->adaptivePtime); + auto codec = obj.getObject(javaClass->codec); webrtc::RtpEncodingParameters params; @@ -85,6 +123,30 @@ namespace jni if (scaleResolution.get()) { params.scale_resolution_down_by.emplace(Double::getValue(env, scaleResolution)); } + if (rid.get()) { + params.rid = JavaString::toNative(env, rid); + } + if (scaleResolutionDownTo.get()) { + params.scale_resolution_down_to = RTCResolutionRestriction::toNative(env, scaleResolutionDownTo); + } + if (scalabilityMode.get()) { + params.scalability_mode = JavaString::toNative(env, scalabilityMode); + } + if (numTemporalLayers.get()) { + params.num_temporal_layers = Integer::getValue(env, numTemporalLayers); + } + if (bitratePriority.get()) { + params.bitrate_priority = Double::getValue(env, bitratePriority); + } + if (networkPriority.get()) { + params.network_priority = JavaEnums::toNative(env, networkPriority); + } + if (adaptivePtime.get()) { + params.adaptive_ptime = Boolean::getValue(env, adaptivePtime); + } + if (codec.get()) { + params.codec = static_cast(RTCRtpCodecCapability::toNative(env, codec)); + } return params; } @@ -101,6 +163,14 @@ namespace jni maxBitrate = GetFieldID(env, cls, "maxBitrate", INTEGER_SIG); maxFramerate = GetFieldID(env, cls, "maxFramerate", DOUBLE_SIG); scaleResolution = GetFieldID(env, cls, "scaleResolutionDownBy", DOUBLE_SIG); + rid = GetFieldID(env, cls, "rid", STRING_SIG); + scaleResolutionDownTo = GetFieldID(env, cls, "scaleResolutionDownTo", "L" PKG "RTCResolutionRestriction;"); + scalabilityMode = GetFieldID(env, cls, "scalabilityMode", STRING_SIG); + numTemporalLayers = GetFieldID(env, cls, "numTemporalLayers", INTEGER_SIG); + bitratePriority = GetFieldID(env, cls, "bitratePriority", DOUBLE_SIG); + networkPriority = GetFieldID(env, cls, "networkPriority", "L" PKG "RTCPriorityType;"); + adaptivePtime = GetFieldID(env, cls, "adaptivePtime", BOOLEAN_SIG); + codec = GetFieldID(env, cls, "codec", "L" PKG "RTCRtpCodecCapability;"); } } } diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpSendParameters.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpSendParameters.cpp index 1aadd9d76..cbc784afd 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCRtpSendParameters.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpSendParameters.cpp @@ -22,6 +22,7 @@ #include "api/RTCRtcpParameters.h" #include "JavaArrayList.h" #include "JavaClasses.h" +#include "JavaEnums.h" #include "JavaIterable.h" #include "JavaList.h" #include "JavaString.h" @@ -50,6 +51,11 @@ namespace jni env->SetObjectField(object, javaParentClass->rtcp, rtcp.get()); env->SetObjectField(object, javaParentClass->codecs, codecs.get()); + if (parameters.degradation_preference.has_value()) { + env->SetObjectField(object, javaClass->degradationPreference, + JavaEnums::toJava(env, *parameters.degradation_preference).get()); + } + return JavaLocalRef(env, object); } @@ -65,6 +71,7 @@ namespace jni JavaLocalRef headerExtensions = obj.getObject(javaParentClass->headerExtensions); JavaLocalRef rtcp = obj.getObject(javaParentClass->rtcp); JavaLocalRef codecs = obj.getObject(javaParentClass->codecs); + JavaLocalRef degradationPreference = obj.getObject(javaClass->degradationPreference); webrtc::RtpParameters params; params.transaction_id = JavaString::toNative(env, transactionId); @@ -81,6 +88,10 @@ namespace jni if (codecs) { params.codecs = JavaList::toVector(env, codecs, &RTCRtpCodecParameters::toNative); } + if (degradationPreference) { + params.degradation_preference = + JavaEnums::toNative(env, degradationPreference); + } return params; } @@ -93,6 +104,7 @@ namespace jni transactionId = GetFieldID(env, cls, "transactionId", STRING_SIG); encodings = GetFieldID(env, cls, "encodings", LIST_SIG); + degradationPreference = GetFieldID(env, cls, "degradationPreference", "L" PKG "RTCDegradationPreference;"); } } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCDegradationPreference.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCDegradationPreference.java new file mode 100644 index 000000000..60c80172a --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCDegradationPreference.java @@ -0,0 +1,51 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +/** + * What a video sender gives up first when the network or the CPU cannot keep + * up: frame rate, resolution, or a balance of both. Set on the {@link + * RTCRtpSendParameters#degradationPreference send parameters}. + * + * @author Alex Andres + */ +public enum RTCDegradationPreference { + + /** + * Keep both frame rate and resolution, and drop frames instead where + * necessary. Quality adaptation is off. + */ + MAINTAIN_FRAMERATE_AND_RESOLUTION, + + /** + * Keep the frame rate, and lower the resolution. Suits motion, such as + * camera video of people. + */ + MAINTAIN_FRAMERATE, + + /** + * Keep the resolution, and lower the frame rate. Suits detail, such as + * shared screens with text. + */ + MAINTAIN_RESOLUTION, + + /** + * Lower both frame rate and resolution in turns. + */ + BALANCED + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCResolutionRestriction.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCResolutionRestriction.java new file mode 100644 index 000000000..0f6f5692d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCResolutionRestriction.java @@ -0,0 +1,60 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +/** + * The largest resolution an encoding may be sent in, as {@link + * RTCRtpEncodingParameters#scaleResolutionDownTo} takes it: an absolute size + * the video is scaled down to fit in, rather than a factor relative to the + * size of the frames. + * + * @author Alex Andres + */ +public class RTCResolutionRestriction { + + /** The largest width in pixels. */ + public int maxWidth; + + /** The largest height in pixels. */ + public int maxHeight; + + + /** + * Creates an empty restriction, whose fields are to be set. + */ + public RTCResolutionRestriction() { + } + + /** + * Creates a restriction to the given size. + * + * @param maxWidth The largest width in pixels. + * @param maxHeight The largest height in pixels. + */ + public RTCResolutionRestriction(int maxWidth, int maxHeight) { + this.maxWidth = maxWidth; + this.maxHeight = maxHeight; + } + + @Override + public String toString() { + return String.format("%s [maxWidth=%d, maxHeight=%d]", + RTCResolutionRestriction.class.getSimpleName(), maxWidth, + maxHeight); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpCodecCapability.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpCodecCapability.java index 545df5b93..800ab6480 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpCodecCapability.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpCodecCapability.java @@ -18,6 +18,9 @@ import dev.onvoid.webrtc.media.MediaType; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; /** @@ -57,6 +60,11 @@ public class RTCRtpCodecCapability { */ private final Map sdpFmtp; + /** + * The scalability modes the codec supports; set by native code. + */ + private String[] scalabilityModes = new String[0]; + /** * Creates an instance of RTCRtpCodecCapability with the specified @@ -127,6 +135,18 @@ public Map getSDPFmtp() { return sdpFmtp; } + /** + * Returns the scalability modes the codec supports, as named in the WebRTC + * SVC specification, e.g. "L1T3". These are what {@link + * RTCRtpEncodingParameters#scalabilityMode} accepts for the codec. Empty + * for audio codecs, and for video codecs without layers. + * + * @return The supported scalability modes, unmodifiable. + */ + public List getScalabilityModes() { + return Collections.unmodifiableList(Arrays.asList(scalabilityModes)); + } + /** * Returns the MIME type composed of the {@code mediaType} and the {@code * name}. @@ -139,8 +159,8 @@ public String getMimeType() { @Override public String toString() { - return String.format("%s [mediaType=%s, name=%s, clockRate=%s, channels=%s, sdpFmtp=%s]", + return String.format("%s [mediaType=%s, name=%s, clockRate=%s, channels=%s, sdpFmtp=%s, scalabilityModes=%s]", RTCRtpCodecCapability.class.getSimpleName(), mediaType, name, - clockRate, channels, sdpFmtp); + clockRate, channels, sdpFmtp, Arrays.toString(scalabilityModes)); } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpEncodingParameters.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpEncodingParameters.java index 43097258f..f5f803542 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpEncodingParameters.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpEncodingParameters.java @@ -23,6 +23,14 @@ */ public class RTCRtpEncodingParameters { + /** + * The RTP stream ID of the encoding, which tells simulcast encodings apart, + * e.g. "f", "h" and "q" for full, half and quarter resolution. Set only + * when the encodings are given to {@link RTCRtpTransceiverInit}; it cannot + * be changed afterwards. Unset, or empty, without simulcast. + */ + public String rid; + /** * If unset, a value is chosen by the implementation. *
@@ -69,6 +77,61 @@ public class RTCRtpEncodingParameters { */ public Double scaleResolutionDownBy; + /** + * Only for video. The largest resolution to send this encoding in: the + * video is scaled down to fit. Takes precedence over {@link + * #scaleResolutionDownBy} if both are set. If unset, the resolution is not + * restricted this way. + */ + public RTCResolutionRestriction scaleResolutionDownTo; + + /** + * Only for video. The scalability mode of the encoding, as named in the + * WebRTC SVC specification, e.g. "L1T3" for three temporal layers or + * "L3T3_KEY" for three spatial layers with three temporal layers each. The + * codec has to support the mode, which {@link RTCRtpCodecCapability} + * lists; setting one it does not support fails. If unset, the encoder + * chooses its layers, or follows {@link #numTemporalLayers}. + */ + public String scalabilityMode; + + /** + * Only for video. The number of temporal layers to encode, if the codec + * supports temporal layers. An older way to ask for them than {@link + * #scalabilityMode}; set one of the two. If unset, the encoder chooses. + */ + public Integer numTemporalLayers; + + /** + * The share of the available bitrate the sender gets relative to the other + * senders of the peer connection. It applies to the whole sender, so only + * the first encoding may set it; setting it on another fails. If unset, + * the default of 1.0. + */ + public Double bitratePriority; + + /** + * The priority the packets of the sender are marked with on the network + * (DSCP), where the network honors it. It applies to the whole sender, so + * only the first encoding may set it; setting it on another fails. If + * unset, {@link RTCPriorityType#LOW}. + */ + public RTCPriorityType networkPriority; + + /** + * Only for audio. Whether the encoder may send longer audio packets when + * the bitrate is low, which saves the overhead of many small packets. If + * unset, it does not. + */ + public Boolean adaptivePtime; + + /** + * The codec to send this encoding with, one of those negotiated, which + * lets simulcast encodings use different codecs. If unset, the encoding + * uses the codec the negotiation settled on. + */ + public RTCRtpCodecCapability codec; + /** * Creates an instance of RTCRtpEncodingParameters. @@ -79,9 +142,16 @@ public RTCRtpEncodingParameters() { @Override public String toString() { - return "RTCRtpEncodingParameters{" + "ssrc=" + ssrc + ", active=" - + active + ", maxBitrate=" + maxBitrate + ", minBitrate=" - + minBitrate + ", maxFramerate=" + maxFramerate - + ", scaleResolutionDownBy=" + scaleResolutionDownBy + '}'; + return "RTCRtpEncodingParameters{" + "rid=" + rid + ", ssrc=" + ssrc + + ", active=" + active + ", maxBitrate=" + maxBitrate + + ", minBitrate=" + minBitrate + ", maxFramerate=" + maxFramerate + + ", scaleResolutionDownBy=" + scaleResolutionDownBy + + ", scaleResolutionDownTo=" + scaleResolutionDownTo + + ", scalabilityMode=" + scalabilityMode + + ", numTemporalLayers=" + numTemporalLayers + + ", bitratePriority=" + bitratePriority + + ", networkPriority=" + networkPriority + + ", adaptivePtime=" + adaptivePtime + + ", codec=" + codec + '}'; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSendParameters.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSendParameters.java index babe19b53..86873c0d0 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSendParameters.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSendParameters.java @@ -37,11 +37,19 @@ public class RTCRtpSendParameters extends RTCRtpParameters { */ public List encodings; + /** + * Only for video. What the sender gives up first when the network or the + * CPU cannot keep up. If unset, WebRTC chooses by the content: it keeps + * the resolution of screen shares and of tracks hinted as detailed or + * text, and the frame rate of other video. + */ + public RTCDegradationPreference degradationPreference; + @Override public String toString() { - return String.format("%s [transactionId=%s, encodings=%s, headerExtensions=%s, rtcp=%s, codecs=%s]", + return String.format("%s [transactionId=%s, encodings=%s, degradationPreference=%s, headerExtensions=%s, rtcp=%s, codecs=%s]", RTCRtpSendParameters.class.getSimpleName(), transactionId, - encodings, headerExtensions, rtcp, codecs); + encodings, degradationPreference, headerExtensions, rtcp, codecs); } } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpEncodingParametersTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpEncodingParametersTests.java new file mode 100644 index 000000000..8d5ae55d4 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpEncodingParametersTests.java @@ -0,0 +1,273 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.onvoid.webrtc.media.MediaType; +import dev.onvoid.webrtc.media.audio.AudioOptions; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.AudioTrackSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests the encoding parameters of senders: that they reach WebRTC and come + * back, and that WebRTC acts on those it checks. + */ +@Execution(ExecutionMode.SAME_THREAD) +class RTCRtpEncodingParametersTests extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + private TestPeerConnection connection; + + private CustomVideoSource videoSource; + + private VideoTrack videoTrack; + + private AudioTrackSource audioSource; + + private AudioTrack audioTrack; + + + @BeforeEach + void init() { + connection = new TestPeerConnection(factory); + videoSource = new CustomVideoSource(); + videoTrack = factory.createVideoTrack("video", videoSource); + } + + @AfterEach + void dispose() { + // The tracks go after the connection, whose senders hold them. + connection.close(); + videoTrack.dispose(); + videoSource.dispose(); + + if (audioTrack != null) { + audioTrack.dispose(); + audioSource.dispose(); + } + } + + @Test + void simulcastEncodingsRoundTrip() { + RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); + init.sendEncodings = Arrays.asList( + encoding("q", 4.0), + encoding("h", 2.0), + encoding("f", 1.0)); + + // The priorities apply to the whole sender, set on the first encoding. + init.sendEncodings.get(0).networkPriority = RTCPriorityType.HIGH; + init.sendEncodings.get(0).bitratePriority = 2.0; + init.sendEncodings.get(2).maxBitrate = 1_500_000; + + RTCRtpTransceiver transceiver = connection.getPeerConnection().addTransceiver(videoTrack, init); + RTCRtpSender sender = transceiver.getSender(); + + List encodings = sender.getParameters().encodings; + + assertEquals(Arrays.asList("q", "h", "f"), + encodings.stream().map(encoding -> encoding.rid).collect(Collectors.toList())); + assertEquals(4.0, encodings.get(0).scaleResolutionDownBy); + assertEquals(RTCPriorityType.HIGH, encodings.get(0).networkPriority); + assertEquals(2.0, encodings.get(0).bitratePriority); + assertEquals(1_500_000, encodings.get(2).maxBitrate); + + sender.dispose(); + transceiver.dispose(); + } + + @Test + void priorityOnlyOnFirstEncoding() { + RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); + init.sendEncodings = Arrays.asList(encoding("h", 2.0), encoding("f", 1.0)); + init.sendEncodings.get(1).networkPriority = RTCPriorityType.HIGH; + + assertThrows(RuntimeException.class, () -> connection.getPeerConnection().addTransceiver(videoTrack, init)); + } + + @Test + void ridCannotChange() { + RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); + init.sendEncodings = Arrays.asList(encoding("h", 2.0), encoding("f", 1.0)); + + RTCRtpTransceiver transceiver = connection.getPeerConnection().addTransceiver(videoTrack, init); + RTCRtpSender sender = transceiver.getSender(); + + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.encodings.get(0).rid = "x"; + + assertThrows(RuntimeException.class, () -> sender.setParameters(parameters)); + + sender.dispose(); + transceiver.dispose(); + } + + @Test + void resolutionAndDegradationRoundTrip() { + RTCRtpSender sender = connection.getPeerConnection().addTrack(videoTrack, Collections.singletonList("stream")); + + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.degradationPreference = RTCDegradationPreference.MAINTAIN_RESOLUTION; + parameters.encodings.get(0).scaleResolutionDownTo = new RTCResolutionRestriction(640, 360); + + sender.setParameters(parameters); + + RTCRtpSendParameters applied = sender.getParameters(); + RTCResolutionRestriction restriction = applied.encodings.get(0).scaleResolutionDownTo; + + assertEquals(RTCDegradationPreference.MAINTAIN_RESOLUTION, applied.degradationPreference); + assertNotNull(restriction); + assertEquals(640, restriction.maxWidth); + assertEquals(360, restriction.maxHeight); + + sender.dispose(); + } + + @Test + void adaptivePtimeRoundTrips() { + audioSource = factory.createAudioSource(new AudioOptions()); + audioTrack = factory.createAudioTrack("audio", audioSource); + + RTCRtpSender sender = connection.getPeerConnection().addTrack(audioTrack, Collections.singletonList("stream")); + + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.encodings.get(0).adaptivePtime = true; + + sender.setParameters(parameters); + + assertEquals(Boolean.TRUE, sender.getParameters().encodings.get(0).adaptivePtime); + + sender.dispose(); + } + + @Test + void scalabilityModeIsApplied() throws Exception { + CountDownLatch received = new CountDownLatch(10); + + // The call prefers VP8, which encodes up to three temporal layers. + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + + RTCRtpSender sender = call.getVideoSender(); + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.encodings.get(0).scalabilityMode = "L1T3"; + + sender.setParameters(parameters); + + assertEquals("L1T3", sender.getParameters().encodings.get(0).scalabilityMode); + + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); + + track.removeSink(sink); + receiver.dispose(); + } + } + + @Test + void capabilitiesListScalabilityModes() { + RTCRtpCodecCapability vp8 = factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs().stream() + .filter(codec -> "VP8".equalsIgnoreCase(codec.getName())) + .findFirst() + .orElseThrow(IllegalStateException::new); + + assertTrue(vp8.getScalabilityModes().contains("L1T3"), vp8.getScalabilityModes().toString()); + assertFalse(vp8.getScalabilityModes().contains("L3T3"), vp8.getScalabilityModes().toString()); + } + + @Test + void unsupportedScalabilityModeFails() throws Exception { + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.negotiate(); + call.awaitConnected(); + + RTCRtpSender sender = call.getVideoSender(); + RTCRtpSendParameters parameters = sender.getParameters(); + + // Spatial layers, which VP8 does not have. + parameters.encodings.get(0).scalabilityMode = "L3T3"; + + assertThrows(RuntimeException.class, () -> sender.setParameters(parameters)); + } + } + + @Test + void encodingCodecRoundTrips() throws Exception { + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.negotiate(); + call.awaitConnected(); + + RTCRtpCodecCapability vp8 = factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs().stream() + .filter(codec -> "VP8".equalsIgnoreCase(codec.getName())) + .findFirst() + .orElseThrow(IllegalStateException::new); + + RTCRtpSender sender = call.getVideoSender(); + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.encodings.get(0).codec = vp8; + + sender.setParameters(parameters); + + RTCRtpCodecCapability codec = sender.getParameters().encodings.get(0).codec; + + assertNotNull(codec); + assertEquals("VP8", codec.getName()); + } + } + + private static RTCRtpEncodingParameters encoding(String rid, double scale) { + RTCRtpEncodingParameters encoding = new RTCRtpEncodingParameters(); + encoding.rid = rid; + encoding.scaleResolutionDownBy = scale; + + return encoding; + } + +}