diff --git a/docs/guide/media/constraints.md b/docs/guide/media/constraints.md index fb3b32e8..532be7a1 100644 --- a/docs/guide/media/constraints.md +++ b/docs/guide/media/constraints.md @@ -113,6 +113,89 @@ The `scaleResolutionDownBy` parameter specifies how much to scale down the video Note that these constraints are applied without requiring SDP renegotiation, making them suitable for dynamic adaptation to changing network conditions. ::: +To cap the resolution at an absolute size instead, set `scaleResolutionDownTo`. The video is scaled down to fit, whatever size the source delivers; it takes precedence over `scaleResolutionDownBy` if both are set. + +```java +parameters.encodings.get(0).scaleResolutionDownTo = new RTCResolutionRestriction(1280, 720); +sender.setParameters(parameters); +``` + +## Degradation Preference + +When the network or the CPU cannot keep up, a video sender lowers either its frame rate or its resolution. `degradationPreference` on the send parameters decides which: + +```java +RTCRtpSendParameters parameters = sender.getParameters(); + +// Keep text sharp when sharing a screen: lower the frame rate instead. +parameters.degradationPreference = RTCDegradationPreference.MAINTAIN_RESOLUTION; + +sender.setParameters(parameters); +``` + +| Value | Gives up | +|---|---| +| `MAINTAIN_FRAMERATE` | Resolution; suits motion, such as camera video | +| `MAINTAIN_RESOLUTION` | Frame rate; suits detail, such as screens with text | +| `BALANCED` | Both, in turns | +| `MAINTAIN_FRAMERATE_AND_RESOLUTION` | Neither; frames are dropped instead | + +If unset, WebRTC keeps the resolution of screen shares and of tracks hinted as detailed or text, and the frame rate of other video. + +## Simulcast + +With simulcast, a video sender encodes the same video several times, at different resolutions, so that a server can forward to each receiver the one its connection can take. The encodings are given when the transceiver is created, each with an RTP stream ID (`rid`): + +```java +RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); +init.direction = RTCRtpTransceiverDirection.SEND_ONLY; + +RTCRtpEncodingParameters quarter = new RTCRtpEncodingParameters(); +quarter.rid = "q"; +quarter.scaleResolutionDownBy = 4.0; +quarter.maxBitrate = 150_000; + +RTCRtpEncodingParameters half = new RTCRtpEncodingParameters(); +half.rid = "h"; +half.scaleResolutionDownBy = 2.0; +half.maxBitrate = 500_000; + +RTCRtpEncodingParameters full = new RTCRtpEncodingParameters(); +full.rid = "f"; +full.maxBitrate = 1_500_000; + +init.sendEncodings = Arrays.asList(quarter, half, full); + +RTCRtpTransceiver transceiver = peerConnection.addTransceiver(videoTrack, init); +``` + +The `rid`s cannot be changed afterwards; the other parameters of each encoding can, through `setParameters()`, e.g. `active = false` to pause one of them. An encoding may also send a different codec than the others, by setting its `codec` to one of the negotiated codecs. + +## Scalability Modes (SVC) + +Instead of separate encodings, one encoding can carry layers, which a server can drop to lower the rate: temporal layers (fewer frames per second) and, with VP9 and AV1, spatial layers (lower resolutions). `scalabilityMode` takes the mode names of the [WebRTC SVC specification](https://www.w3.org/TR/webrtc-svc/), e.g. `L1T3` for three temporal layers, or `L3T3_KEY` for three spatial layers with three temporal layers each: + +```java +RTCRtpSendParameters parameters = sender.getParameters(); +parameters.encodings.get(0).scalabilityMode = "L1T3"; +sender.setParameters(parameters); +``` + +The codec has to support the mode; `setParameters()` throws otherwise. `getScalabilityModes()` of a codec's capability, from `factory.getRtpSenderCapabilities(MediaType.VIDEO)`, lists the modes it supports. VP8 supports temporal layers only. + +## Priorities + +`bitratePriority` sets the share of the available bitrate a sender gets relative to the other senders of the peer connection (default 1.0), and `networkPriority` how its packets are marked on the network (DSCP), where the network honors it. Both apply to the whole sender, so only the first encoding may set them: + +```java +RTCRtpSendParameters parameters = sender.getParameters(); +parameters.encodings.get(0).bitratePriority = 2.0; +parameters.encodings.get(0).networkPriority = RTCPriorityType.HIGH; +sender.setParameters(parameters); +``` + +For audio, `adaptivePtime` lets the encoder send longer packets when the bitrate is low, which saves the overhead of many small packets. + ## Conclusion In this guide, we've explored several important techniques for controlling media quality and bandwidth usage in WebRTC applications. diff --git a/webrtc-jni/src/main/cpp/include/api/RTCResolutionRestriction.h b/webrtc-jni/src/main/cpp/include/api/RTCResolutionRestriction.h new file mode 100644 index 00000000..d1359801 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/RTCResolutionRestriction.h @@ -0,0 +1,47 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_RTC_RESOLUTION_RESTRICTION_H_ +#define JNI_WEBRTC_API_RTC_RESOLUTION_RESTRICTION_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/resolution.h" + +#include + +namespace jni +{ + namespace RTCResolutionRestriction + { + class JavaRTCResolutionRestrictionClass : public JavaClass + { + public: + explicit JavaRTCResolutionRestrictionClass(JNIEnv * env); + + jclass cls; + jmethodID ctor; + jfieldID maxWidth; + jfieldID maxHeight; + }; + + JavaLocalRef toJava(JNIEnv * env, const webrtc::Resolution & resolution); + webrtc::Resolution toNative(JNIEnv * env, const JavaRef & restriction); + } +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpCodecCapability.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpCodecCapability.h index 7a8dc626..94b4c5db 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCRtpCodecCapability.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpCodecCapability.h @@ -40,6 +40,7 @@ namespace jni jfieldID clockRate; jfieldID channels; jfieldID sdpFmtp; + jfieldID scalabilityModes; }; JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpCodecCapability & capability); diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpEncodingParameters.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpEncodingParameters.h index 6e2c4d47..16da7121 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCRtpEncodingParameters.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpEncodingParameters.h @@ -41,6 +41,14 @@ namespace jni jfieldID maxBitrate; jfieldID maxFramerate; jfieldID scaleResolution; + jfieldID rid; + jfieldID scaleResolutionDownTo; + jfieldID scalabilityMode; + jfieldID numTemporalLayers; + jfieldID bitratePriority; + jfieldID networkPriority; + jfieldID adaptivePtime; + jfieldID codec; }; JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpEncodingParameters & parameters); diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpSendParameters.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpSendParameters.h index 61ab494e..ed6fbe08 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCRtpSendParameters.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpSendParameters.h @@ -37,6 +37,7 @@ namespace jni jmethodID ctor; jfieldID transactionId; jfieldID encodings; + jfieldID degradationPreference; }; JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpParameters & parameters); diff --git a/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp b/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp index 165411e0..699cf584 100644 --- a/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp +++ b/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp @@ -27,6 +27,8 @@ #include "api/environment/environment_factory.h" #include "api/peer_connection_interface.h" +#include "api/priority.h" +#include "api/rtp_parameters.h" #include "modules/desktop_capture/desktop_capturer.h" #include "rtc_base/ssl_adapter.h" @@ -83,6 +85,8 @@ namespace jni JavaEnums::add(env, PKG"RTCSignalingState"); JavaEnums::add(env, PKG"TlsCertPolicy"); JavaEnums::add(env, PKG"RTCRtpTransceiverDirection"); + JavaEnums::add(env, PKG"RTCPriorityType"); + JavaEnums::add(env, PKG"RTCDegradationPreference"); JavaEnums::add(env, PKG"RTCSdpType"); JavaEnums::add(env, PKG_AUDIO"AudioLayer"); JavaEnums::add(env, PKG_AUDIO"AudioProcessingConfig$GainController$Mode"); diff --git a/webrtc-jni/src/main/cpp/src/api/RTCResolutionRestriction.cpp b/webrtc-jni/src/main/cpp/src/api/RTCResolutionRestriction.cpp new file mode 100644 index 00000000..7c45ecb6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/RTCResolutionRestriction.cpp @@ -0,0 +1,62 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/RTCResolutionRestriction.h" +#include "JavaClasses.h" +#include "JavaObject.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +namespace jni +{ + namespace RTCResolutionRestriction + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::Resolution & resolution) + { + const auto javaClass = JavaClasses::get(env); + + jobject object = env->NewObject(javaClass->cls, javaClass->ctor, + static_cast(resolution.width), static_cast(resolution.height)); + + ExceptionCheck(env); + + return JavaLocalRef(env, object); + } + + webrtc::Resolution toNative(JNIEnv * env, const JavaRef & restriction) + { + const auto javaClass = JavaClasses::get(env); + + JavaObject obj(env, restriction); + + webrtc::Resolution resolution; + resolution.width = obj.getInt(javaClass->maxWidth); + resolution.height = obj.getInt(javaClass->maxHeight); + + return resolution; + } + + JavaRTCResolutionRestrictionClass::JavaRTCResolutionRestrictionClass(JNIEnv * env) + { + cls = FindClass(env, PKG"RTCResolutionRestriction"); + + ctor = GetMethod(env, cls, "", "(II)V"); + + maxWidth = GetFieldID(env, cls, "maxWidth", "I"); + maxHeight = GetFieldID(env, cls, "maxHeight", "I"); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpCodecCapability.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpCodecCapability.cpp index b9b386e6..c1ee4ba4 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCRtpCodecCapability.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpCodecCapability.cpp @@ -23,6 +23,12 @@ #include "JavaUtils.h" #include "JNI_WebRTC.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalability_mode_util.h" + +#include +#include + namespace jni { namespace RTCRtpCodecCapability @@ -59,6 +65,16 @@ namespace jni ExceptionCheck(env); + if (!capability.scalability_modes.empty()) { + std::vector modes; + + for (webrtc::ScalabilityMode mode : capability.scalability_modes) { + modes.emplace_back(webrtc::ScalabilityModeToString(mode)); + } + + env->SetObjectField(object, javaClass->scalabilityModes, JavaString::createArray(env, modes).get()); + } + return JavaLocalRef(env, object); } @@ -90,6 +106,18 @@ namespace jni codecCapability.num_channels = Integer::getValue(env, channels); } + JavaLocalRef modes = obj.getObjectArray(javaClass->scalabilityModes); + const jsize count = modes.get() != nullptr ? env->GetArrayLength(modes.get()) : 0; + + for (jsize i = 0; i < count; i++) { + JavaLocalRef mode(env, static_cast(env->GetObjectArrayElement(modes.get(), i))); + auto scalabilityMode = webrtc::ScalabilityModeFromString(JavaString::toNative(env, mode)); + + if (scalabilityMode.has_value()) { + codecCapability.scalability_modes.push_back(*scalabilityMode); + } + } + return codecCapability; } @@ -104,6 +132,7 @@ namespace jni clockRate = GetFieldID(env, cls, "clockRate", INTEGER_SIG); channels = GetFieldID(env, cls, "channels", INTEGER_SIG); sdpFmtp = GetFieldID(env, cls, "sdpFmtp", MAP_SIG); + scalabilityModes = GetFieldID(env, cls, "scalabilityModes", "[" STRING_SIG); } } } diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpEncodingParameters.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpEncodingParameters.cpp index 4efca126..55ca011f 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCRtpEncodingParameters.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpEncodingParameters.cpp @@ -15,11 +15,14 @@ */ #include "api/RTCRtpEncodingParameters.h" +#include "api/RTCResolutionRestriction.h" +#include "api/RTCRtpCodecCapability.h" #include "JavaClasses.h" #include "JavaEnums.h" #include "JavaObject.h" #include "JavaPrimitive.h" #include "JavaRef.h" +#include "JavaString.h" #include "JNI_WebRTC.h" namespace jni @@ -48,6 +51,33 @@ namespace jni if (parameters.scale_resolution_down_by.has_value()) { env->SetObjectField(object, javaClass->scaleResolution, Double::create(env, parameters.scale_resolution_down_by.value())); } + if (!parameters.rid.empty()) { + env->SetObjectField(object, javaClass->rid, JavaString::toJava(env, parameters.rid).get()); + } + if (parameters.scale_resolution_down_to.has_value()) { + env->SetObjectField(object, javaClass->scaleResolutionDownTo, + RTCResolutionRestriction::toJava(env, *parameters.scale_resolution_down_to).get()); + } + if (parameters.scalability_mode.has_value()) { + env->SetObjectField(object, javaClass->scalabilityMode, + JavaString::toJava(env, *parameters.scalability_mode).get()); + } + if (parameters.num_temporal_layers.has_value()) { + env->SetObjectField(object, javaClass->numTemporalLayers, + Integer::create(env, *parameters.num_temporal_layers).get()); + } + if (parameters.codec.has_value()) { + // The Java type of a codec is the capability, which adds to + // the codec what the encoding does not carry. + webrtc::RtpCodecCapability capability; + static_cast(capability) = *parameters.codec; + + env->SetObjectField(object, javaClass->codec, RTCRtpCodecCapability::toJava(env, capability).get()); + } + + env->SetObjectField(object, javaClass->bitratePriority, Double::create(env, parameters.bitrate_priority).get()); + env->SetObjectField(object, javaClass->networkPriority, JavaEnums::toJava(env, parameters.network_priority).get()); + env->SetObjectField(object, javaClass->adaptivePtime, Boolean::create(env, parameters.adaptive_ptime).get()); return JavaLocalRef(env, object); } @@ -64,6 +94,14 @@ namespace jni auto maxBitrate = obj.getObject(javaClass->maxBitrate); auto maxFramerate = obj.getObject(javaClass->maxFramerate); auto scaleResolution = obj.getObject(javaClass->scaleResolution); + auto rid = obj.getString(javaClass->rid); + auto scaleResolutionDownTo = obj.getObject(javaClass->scaleResolutionDownTo); + auto scalabilityMode = obj.getString(javaClass->scalabilityMode); + auto numTemporalLayers = obj.getObject(javaClass->numTemporalLayers); + auto bitratePriority = obj.getObject(javaClass->bitratePriority); + auto networkPriority = obj.getObject(javaClass->networkPriority); + auto adaptivePtime = obj.getObject(javaClass->adaptivePtime); + auto codec = obj.getObject(javaClass->codec); webrtc::RtpEncodingParameters params; @@ -85,6 +123,30 @@ namespace jni if (scaleResolution.get()) { params.scale_resolution_down_by.emplace(Double::getValue(env, scaleResolution)); } + if (rid.get()) { + params.rid = JavaString::toNative(env, rid); + } + if (scaleResolutionDownTo.get()) { + params.scale_resolution_down_to = RTCResolutionRestriction::toNative(env, scaleResolutionDownTo); + } + if (scalabilityMode.get()) { + params.scalability_mode = JavaString::toNative(env, scalabilityMode); + } + if (numTemporalLayers.get()) { + params.num_temporal_layers = Integer::getValue(env, numTemporalLayers); + } + if (bitratePriority.get()) { + params.bitrate_priority = Double::getValue(env, bitratePriority); + } + if (networkPriority.get()) { + params.network_priority = JavaEnums::toNative(env, networkPriority); + } + if (adaptivePtime.get()) { + params.adaptive_ptime = Boolean::getValue(env, adaptivePtime); + } + if (codec.get()) { + params.codec = static_cast(RTCRtpCodecCapability::toNative(env, codec)); + } return params; } @@ -101,6 +163,14 @@ namespace jni maxBitrate = GetFieldID(env, cls, "maxBitrate", INTEGER_SIG); maxFramerate = GetFieldID(env, cls, "maxFramerate", DOUBLE_SIG); scaleResolution = GetFieldID(env, cls, "scaleResolutionDownBy", DOUBLE_SIG); + rid = GetFieldID(env, cls, "rid", STRING_SIG); + scaleResolutionDownTo = GetFieldID(env, cls, "scaleResolutionDownTo", "L" PKG "RTCResolutionRestriction;"); + scalabilityMode = GetFieldID(env, cls, "scalabilityMode", STRING_SIG); + numTemporalLayers = GetFieldID(env, cls, "numTemporalLayers", INTEGER_SIG); + bitratePriority = GetFieldID(env, cls, "bitratePriority", DOUBLE_SIG); + networkPriority = GetFieldID(env, cls, "networkPriority", "L" PKG "RTCPriorityType;"); + adaptivePtime = GetFieldID(env, cls, "adaptivePtime", BOOLEAN_SIG); + codec = GetFieldID(env, cls, "codec", "L" PKG "RTCRtpCodecCapability;"); } } } diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpSendParameters.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpSendParameters.cpp index 1aadd9d7..cbc784af 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCRtpSendParameters.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpSendParameters.cpp @@ -22,6 +22,7 @@ #include "api/RTCRtcpParameters.h" #include "JavaArrayList.h" #include "JavaClasses.h" +#include "JavaEnums.h" #include "JavaIterable.h" #include "JavaList.h" #include "JavaString.h" @@ -50,6 +51,11 @@ namespace jni env->SetObjectField(object, javaParentClass->rtcp, rtcp.get()); env->SetObjectField(object, javaParentClass->codecs, codecs.get()); + if (parameters.degradation_preference.has_value()) { + env->SetObjectField(object, javaClass->degradationPreference, + JavaEnums::toJava(env, *parameters.degradation_preference).get()); + } + return JavaLocalRef(env, object); } @@ -65,6 +71,7 @@ namespace jni JavaLocalRef headerExtensions = obj.getObject(javaParentClass->headerExtensions); JavaLocalRef rtcp = obj.getObject(javaParentClass->rtcp); JavaLocalRef codecs = obj.getObject(javaParentClass->codecs); + JavaLocalRef degradationPreference = obj.getObject(javaClass->degradationPreference); webrtc::RtpParameters params; params.transaction_id = JavaString::toNative(env, transactionId); @@ -81,6 +88,10 @@ namespace jni if (codecs) { params.codecs = JavaList::toVector(env, codecs, &RTCRtpCodecParameters::toNative); } + if (degradationPreference) { + params.degradation_preference = + JavaEnums::toNative(env, degradationPreference); + } return params; } @@ -93,6 +104,7 @@ namespace jni transactionId = GetFieldID(env, cls, "transactionId", STRING_SIG); encodings = GetFieldID(env, cls, "encodings", LIST_SIG); + degradationPreference = GetFieldID(env, cls, "degradationPreference", "L" PKG "RTCDegradationPreference;"); } } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCDegradationPreference.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCDegradationPreference.java new file mode 100644 index 00000000..60c80172 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCDegradationPreference.java @@ -0,0 +1,51 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +/** + * What a video sender gives up first when the network or the CPU cannot keep + * up: frame rate, resolution, or a balance of both. Set on the {@link + * RTCRtpSendParameters#degradationPreference send parameters}. + * + * @author Alex Andres + */ +public enum RTCDegradationPreference { + + /** + * Keep both frame rate and resolution, and drop frames instead where + * necessary. Quality adaptation is off. + */ + MAINTAIN_FRAMERATE_AND_RESOLUTION, + + /** + * Keep the frame rate, and lower the resolution. Suits motion, such as + * camera video of people. + */ + MAINTAIN_FRAMERATE, + + /** + * Keep the resolution, and lower the frame rate. Suits detail, such as + * shared screens with text. + */ + MAINTAIN_RESOLUTION, + + /** + * Lower both frame rate and resolution in turns. + */ + BALANCED + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCResolutionRestriction.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCResolutionRestriction.java new file mode 100644 index 00000000..0f6f5692 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCResolutionRestriction.java @@ -0,0 +1,60 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +/** + * The largest resolution an encoding may be sent in, as {@link + * RTCRtpEncodingParameters#scaleResolutionDownTo} takes it: an absolute size + * the video is scaled down to fit in, rather than a factor relative to the + * size of the frames. + * + * @author Alex Andres + */ +public class RTCResolutionRestriction { + + /** The largest width in pixels. */ + public int maxWidth; + + /** The largest height in pixels. */ + public int maxHeight; + + + /** + * Creates an empty restriction, whose fields are to be set. + */ + public RTCResolutionRestriction() { + } + + /** + * Creates a restriction to the given size. + * + * @param maxWidth The largest width in pixels. + * @param maxHeight The largest height in pixels. + */ + public RTCResolutionRestriction(int maxWidth, int maxHeight) { + this.maxWidth = maxWidth; + this.maxHeight = maxHeight; + } + + @Override + public String toString() { + return String.format("%s [maxWidth=%d, maxHeight=%d]", + RTCResolutionRestriction.class.getSimpleName(), maxWidth, + maxHeight); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpCodecCapability.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpCodecCapability.java index 545df5b9..800ab648 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpCodecCapability.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpCodecCapability.java @@ -18,6 +18,9 @@ import dev.onvoid.webrtc.media.MediaType; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; /** @@ -57,6 +60,11 @@ public class RTCRtpCodecCapability { */ private final Map sdpFmtp; + /** + * The scalability modes the codec supports; set by native code. + */ + private String[] scalabilityModes = new String[0]; + /** * Creates an instance of RTCRtpCodecCapability with the specified @@ -127,6 +135,18 @@ public Map getSDPFmtp() { return sdpFmtp; } + /** + * Returns the scalability modes the codec supports, as named in the WebRTC + * SVC specification, e.g. "L1T3". These are what {@link + * RTCRtpEncodingParameters#scalabilityMode} accepts for the codec. Empty + * for audio codecs, and for video codecs without layers. + * + * @return The supported scalability modes, unmodifiable. + */ + public List getScalabilityModes() { + return Collections.unmodifiableList(Arrays.asList(scalabilityModes)); + } + /** * Returns the MIME type composed of the {@code mediaType} and the {@code * name}. @@ -139,8 +159,8 @@ public String getMimeType() { @Override public String toString() { - return String.format("%s [mediaType=%s, name=%s, clockRate=%s, channels=%s, sdpFmtp=%s]", + return String.format("%s [mediaType=%s, name=%s, clockRate=%s, channels=%s, sdpFmtp=%s, scalabilityModes=%s]", RTCRtpCodecCapability.class.getSimpleName(), mediaType, name, - clockRate, channels, sdpFmtp); + clockRate, channels, sdpFmtp, Arrays.toString(scalabilityModes)); } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpEncodingParameters.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpEncodingParameters.java index 43097258..f5f80354 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpEncodingParameters.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpEncodingParameters.java @@ -23,6 +23,14 @@ */ public class RTCRtpEncodingParameters { + /** + * The RTP stream ID of the encoding, which tells simulcast encodings apart, + * e.g. "f", "h" and "q" for full, half and quarter resolution. Set only + * when the encodings are given to {@link RTCRtpTransceiverInit}; it cannot + * be changed afterwards. Unset, or empty, without simulcast. + */ + public String rid; + /** * If unset, a value is chosen by the implementation. *
@@ -69,6 +77,61 @@ public class RTCRtpEncodingParameters { */ public Double scaleResolutionDownBy; + /** + * Only for video. The largest resolution to send this encoding in: the + * video is scaled down to fit. Takes precedence over {@link + * #scaleResolutionDownBy} if both are set. If unset, the resolution is not + * restricted this way. + */ + public RTCResolutionRestriction scaleResolutionDownTo; + + /** + * Only for video. The scalability mode of the encoding, as named in the + * WebRTC SVC specification, e.g. "L1T3" for three temporal layers or + * "L3T3_KEY" for three spatial layers with three temporal layers each. The + * codec has to support the mode, which {@link RTCRtpCodecCapability} + * lists; setting one it does not support fails. If unset, the encoder + * chooses its layers, or follows {@link #numTemporalLayers}. + */ + public String scalabilityMode; + + /** + * Only for video. The number of temporal layers to encode, if the codec + * supports temporal layers. An older way to ask for them than {@link + * #scalabilityMode}; set one of the two. If unset, the encoder chooses. + */ + public Integer numTemporalLayers; + + /** + * The share of the available bitrate the sender gets relative to the other + * senders of the peer connection. It applies to the whole sender, so only + * the first encoding may set it; setting it on another fails. If unset, + * the default of 1.0. + */ + public Double bitratePriority; + + /** + * The priority the packets of the sender are marked with on the network + * (DSCP), where the network honors it. It applies to the whole sender, so + * only the first encoding may set it; setting it on another fails. If + * unset, {@link RTCPriorityType#LOW}. + */ + public RTCPriorityType networkPriority; + + /** + * Only for audio. Whether the encoder may send longer audio packets when + * the bitrate is low, which saves the overhead of many small packets. If + * unset, it does not. + */ + public Boolean adaptivePtime; + + /** + * The codec to send this encoding with, one of those negotiated, which + * lets simulcast encodings use different codecs. If unset, the encoding + * uses the codec the negotiation settled on. + */ + public RTCRtpCodecCapability codec; + /** * Creates an instance of RTCRtpEncodingParameters. @@ -79,9 +142,16 @@ public RTCRtpEncodingParameters() { @Override public String toString() { - return "RTCRtpEncodingParameters{" + "ssrc=" + ssrc + ", active=" - + active + ", maxBitrate=" + maxBitrate + ", minBitrate=" - + minBitrate + ", maxFramerate=" + maxFramerate - + ", scaleResolutionDownBy=" + scaleResolutionDownBy + '}'; + return "RTCRtpEncodingParameters{" + "rid=" + rid + ", ssrc=" + ssrc + + ", active=" + active + ", maxBitrate=" + maxBitrate + + ", minBitrate=" + minBitrate + ", maxFramerate=" + maxFramerate + + ", scaleResolutionDownBy=" + scaleResolutionDownBy + + ", scaleResolutionDownTo=" + scaleResolutionDownTo + + ", scalabilityMode=" + scalabilityMode + + ", numTemporalLayers=" + numTemporalLayers + + ", bitratePriority=" + bitratePriority + + ", networkPriority=" + networkPriority + + ", adaptivePtime=" + adaptivePtime + + ", codec=" + codec + '}'; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSendParameters.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSendParameters.java index babe19b5..86873c0d 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSendParameters.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSendParameters.java @@ -37,11 +37,19 @@ public class RTCRtpSendParameters extends RTCRtpParameters { */ public List encodings; + /** + * Only for video. What the sender gives up first when the network or the + * CPU cannot keep up. If unset, WebRTC chooses by the content: it keeps + * the resolution of screen shares and of tracks hinted as detailed or + * text, and the frame rate of other video. + */ + public RTCDegradationPreference degradationPreference; + @Override public String toString() { - return String.format("%s [transactionId=%s, encodings=%s, headerExtensions=%s, rtcp=%s, codecs=%s]", + return String.format("%s [transactionId=%s, encodings=%s, degradationPreference=%s, headerExtensions=%s, rtcp=%s, codecs=%s]", RTCRtpSendParameters.class.getSimpleName(), transactionId, - encodings, headerExtensions, rtcp, codecs); + encodings, degradationPreference, headerExtensions, rtcp, codecs); } } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpEncodingParametersTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpEncodingParametersTests.java new file mode 100644 index 00000000..8d5ae55d --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpEncodingParametersTests.java @@ -0,0 +1,273 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.onvoid.webrtc.media.MediaType; +import dev.onvoid.webrtc.media.audio.AudioOptions; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.AudioTrackSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests the encoding parameters of senders: that they reach WebRTC and come + * back, and that WebRTC acts on those it checks. + */ +@Execution(ExecutionMode.SAME_THREAD) +class RTCRtpEncodingParametersTests extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + private TestPeerConnection connection; + + private CustomVideoSource videoSource; + + private VideoTrack videoTrack; + + private AudioTrackSource audioSource; + + private AudioTrack audioTrack; + + + @BeforeEach + void init() { + connection = new TestPeerConnection(factory); + videoSource = new CustomVideoSource(); + videoTrack = factory.createVideoTrack("video", videoSource); + } + + @AfterEach + void dispose() { + // The tracks go after the connection, whose senders hold them. + connection.close(); + videoTrack.dispose(); + videoSource.dispose(); + + if (audioTrack != null) { + audioTrack.dispose(); + audioSource.dispose(); + } + } + + @Test + void simulcastEncodingsRoundTrip() { + RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); + init.sendEncodings = Arrays.asList( + encoding("q", 4.0), + encoding("h", 2.0), + encoding("f", 1.0)); + + // The priorities apply to the whole sender, set on the first encoding. + init.sendEncodings.get(0).networkPriority = RTCPriorityType.HIGH; + init.sendEncodings.get(0).bitratePriority = 2.0; + init.sendEncodings.get(2).maxBitrate = 1_500_000; + + RTCRtpTransceiver transceiver = connection.getPeerConnection().addTransceiver(videoTrack, init); + RTCRtpSender sender = transceiver.getSender(); + + List encodings = sender.getParameters().encodings; + + assertEquals(Arrays.asList("q", "h", "f"), + encodings.stream().map(encoding -> encoding.rid).collect(Collectors.toList())); + assertEquals(4.0, encodings.get(0).scaleResolutionDownBy); + assertEquals(RTCPriorityType.HIGH, encodings.get(0).networkPriority); + assertEquals(2.0, encodings.get(0).bitratePriority); + assertEquals(1_500_000, encodings.get(2).maxBitrate); + + sender.dispose(); + transceiver.dispose(); + } + + @Test + void priorityOnlyOnFirstEncoding() { + RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); + init.sendEncodings = Arrays.asList(encoding("h", 2.0), encoding("f", 1.0)); + init.sendEncodings.get(1).networkPriority = RTCPriorityType.HIGH; + + assertThrows(RuntimeException.class, () -> connection.getPeerConnection().addTransceiver(videoTrack, init)); + } + + @Test + void ridCannotChange() { + RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); + init.sendEncodings = Arrays.asList(encoding("h", 2.0), encoding("f", 1.0)); + + RTCRtpTransceiver transceiver = connection.getPeerConnection().addTransceiver(videoTrack, init); + RTCRtpSender sender = transceiver.getSender(); + + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.encodings.get(0).rid = "x"; + + assertThrows(RuntimeException.class, () -> sender.setParameters(parameters)); + + sender.dispose(); + transceiver.dispose(); + } + + @Test + void resolutionAndDegradationRoundTrip() { + RTCRtpSender sender = connection.getPeerConnection().addTrack(videoTrack, Collections.singletonList("stream")); + + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.degradationPreference = RTCDegradationPreference.MAINTAIN_RESOLUTION; + parameters.encodings.get(0).scaleResolutionDownTo = new RTCResolutionRestriction(640, 360); + + sender.setParameters(parameters); + + RTCRtpSendParameters applied = sender.getParameters(); + RTCResolutionRestriction restriction = applied.encodings.get(0).scaleResolutionDownTo; + + assertEquals(RTCDegradationPreference.MAINTAIN_RESOLUTION, applied.degradationPreference); + assertNotNull(restriction); + assertEquals(640, restriction.maxWidth); + assertEquals(360, restriction.maxHeight); + + sender.dispose(); + } + + @Test + void adaptivePtimeRoundTrips() { + audioSource = factory.createAudioSource(new AudioOptions()); + audioTrack = factory.createAudioTrack("audio", audioSource); + + RTCRtpSender sender = connection.getPeerConnection().addTrack(audioTrack, Collections.singletonList("stream")); + + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.encodings.get(0).adaptivePtime = true; + + sender.setParameters(parameters); + + assertEquals(Boolean.TRUE, sender.getParameters().encodings.get(0).adaptivePtime); + + sender.dispose(); + } + + @Test + void scalabilityModeIsApplied() throws Exception { + CountDownLatch received = new CountDownLatch(10); + + // The call prefers VP8, which encodes up to three temporal layers. + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + + RTCRtpSender sender = call.getVideoSender(); + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.encodings.get(0).scalabilityMode = "L1T3"; + + sender.setParameters(parameters); + + assertEquals("L1T3", sender.getParameters().encodings.get(0).scalabilityMode); + + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); + + track.removeSink(sink); + receiver.dispose(); + } + } + + @Test + void capabilitiesListScalabilityModes() { + RTCRtpCodecCapability vp8 = factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs().stream() + .filter(codec -> "VP8".equalsIgnoreCase(codec.getName())) + .findFirst() + .orElseThrow(IllegalStateException::new); + + assertTrue(vp8.getScalabilityModes().contains("L1T3"), vp8.getScalabilityModes().toString()); + assertFalse(vp8.getScalabilityModes().contains("L3T3"), vp8.getScalabilityModes().toString()); + } + + @Test + void unsupportedScalabilityModeFails() throws Exception { + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.negotiate(); + call.awaitConnected(); + + RTCRtpSender sender = call.getVideoSender(); + RTCRtpSendParameters parameters = sender.getParameters(); + + // Spatial layers, which VP8 does not have. + parameters.encodings.get(0).scalabilityMode = "L3T3"; + + assertThrows(RuntimeException.class, () -> sender.setParameters(parameters)); + } + } + + @Test + void encodingCodecRoundTrips() throws Exception { + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.negotiate(); + call.awaitConnected(); + + RTCRtpCodecCapability vp8 = factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs().stream() + .filter(codec -> "VP8".equalsIgnoreCase(codec.getName())) + .findFirst() + .orElseThrow(IllegalStateException::new); + + RTCRtpSender sender = call.getVideoSender(); + RTCRtpSendParameters parameters = sender.getParameters(); + parameters.encodings.get(0).codec = vp8; + + sender.setParameters(parameters); + + RTCRtpCodecCapability codec = sender.getParameters().encodings.get(0).codec; + + assertNotNull(codec); + assertEquals("VP8", codec.getName()); + } + } + + private static RTCRtpEncodingParameters encoding(String rid, double scale) { + RTCRtpEncodingParameters encoding = new RTCRtpEncodingParameters(); + encoding.rid = rid; + encoding.scaleResolutionDownBy = scale; + + return encoding; + } + +}