Start published video at 90% of its target bitrate (x-google-start-bitrate)

.changes/video-publish-start-bitrate

@@ -0,0 +1 @@
+minor type="changed" "Published video starts at 90% of its target bitrate (capped at 1 Mbps unless it is a screen share) instead of libwebrtc's ~300 kbps default, by declaring `x-google-start-bitrate` on the video codecs' fmtp in the publisher's offer. The hint is one connection-level value — the largest among the sending video tracks — written to every video section, and only on the first offer that carries local video, since libwebrtc applies it to the whole peer connection and retains it across network route changes. Same behaviour as client-sdk-js, rust-sdks and client-sdk-android"

Sources/LiveKit/Core/Transport.swift

@@ -66,6 +66,17 @@ final class Transport: NSObject, Loggable {
     private var _isRestartingIce: Bool = false
     private var _latestOfferId: UInt32 = 0
     private var _isClosed: Bool = false
+    /// `x-google-start-bitrate` (kbps) hinted for each video sender id, recorded by
+    /// ``addTransceiver(with:transceiverInit:startBitrateKbps:)``. The largest hint among the
+    /// senders in an offer is the connection-level value; see
+    /// ``mungeVideoStartBitrate(_:kbpsBySenderId:)``.
+    private var _startBitrateKbpsBySenderId: [String: Int] = [:]
+    /// Whether an offer carrying `x-google-start-bitrate` has been accepted locally. The hint
+    /// is written once per peer connection: libwebrtc retains `start_bitrate_bps` in
+    /// `RtpBitrateConfigurator` and re-applies it on network route changes, so a later rewrite
+    /// is at best a no-op and at worst restarts a converged bandwidth estimator. A full
+    /// reconnect builds a new `Transport`, whose new estimator is seeded again.
+    private var _hasAppliedVideoStartBitrate = false
 
     // forbid direct access to PeerConnection; the box parks its blocking release on deinit
     private let _pcBox: RTCBox<LKRTCPeerConnection>
@@ -145,6 +156,12 @@ final class Transport: NSObject, Loggable {
     ///   ``set(localDescription:munging:)`` a munge libwebrtc rejects cannot be dropped
     ///   and retried — the peer has already been told what we offered. Both munges on
     ///   this path are the ones every single PC mode offer already carries.
+    /// - Note: The video start bitrate is deliberately not written here. It derives from
+    ///   senders added by ``addTransceiver(with:transceiverInit:startBitrateKbps:)``, and
+    ///   none exist before the JOIN, so there is never a value to write; and the one-shot
+    ///   latch could not be set safely, since this offer may be dropped unapplied
+    ///   (``clearPendingInitialOffer()``). ``createAndSendOffer(iceRestart:)`` owns the hint
+    ///   and latches only once the offer carrying it has been accepted, as in rust-sdks.
     func createInitialOffer() async throws -> (offer: LKRTCSessionDescription, offerId: UInt32)? {
         guard singlePCMode else { return nil }
 
@@ -270,10 +287,23 @@ final class Transport: NSObject, Loggable {
             _latestOfferId += 1
             var offer = try await createOffer(for: constraints)
             // The direction rewrite is required to receive media in single PC mode; the
-            // stereo preference is optional and must be the one dropped on rejection.
+            // stereo preference and the video start bitrate are optional and are dropped
+            // from the right on rejection. The start bitrate is written on the first offer
+            // that carries local video only: offers before any video is published (data
+            // channel or audio only) find no target and leave the latch unset.
+            let kbpsBySenderId = _startBitrateKbpsBySenderId
+            let hasAppliedStartBitrate = _hasAppliedVideoStartBitrate
+            let startBitrate: (String) -> String = {
+                hasAppliedStartBitrate ? $0 : Self.mungeVideoStartBitrate($0, kbpsBySenderId: kbpsBySenderId)
+            }
             offer = try await set(localDescription: offer, munging: singlePCMode
-                ? [Self.mungeInactiveToRecvOnlyForMedia, Self.mungeOpusStereoForAllAudio]
-                : [])
+                ? [Self.mungeInactiveToRecvOnlyForMedia, Self.mungeOpusStereoForAllAudio, startBitrate]
+                : [startBitrate])
+            // A rejected munge is dropped and the offer retried without it, so the one-shot
+            // hint is consumed only once the description that carries it has been accepted.
+            if !_hasAppliedVideoStartBitrate, Self.carriesVideoStartBitrate(offer.sdp) {
+                _hasAppliedVideoStartBitrate = true
+            }
             try await _onOffer(offer, _latestOfferId)
         }
 
@@ -405,6 +435,116 @@ extension Transport {
         return document.write()
     }
 
+    /// Video codecs libwebrtc reads `x-google-start-bitrate` from.
+    nonisolated static let startBitrateCodecs: Set<String> = ["VP8", "VP9", "AV1", "H264", "H265"]
+
+    /// Largest start bitrate hinted for a non-screen-share track, in kbps. Stops the
+    /// bandwidth estimator from opening too aggressively on high-bitrate (e.g. 4K) tracks.
+    nonisolated static let maxStartBitrateKbps = 1000
+
+    /// Start bitrate hinted to libwebrtc's bandwidth estimator for a video sender whose
+    /// encodings total `targetBps`, or `nil` to leave libwebrtc's default in place.
+    ///
+    /// libwebrtc starts every new video send stream at roughly 300 kbps and ramps up from
+    /// there, regardless of the encodings' `maxBitrate`, so the first seconds of a published
+    /// track are visibly blurry. The hint is 90% of the target bitrate: high enough to skip
+    /// most of the ramp, with headroom for the estimator to settle. The same multiplier is
+    /// used for every codec because the target already reflects the codec's efficiency.
+    /// Camera and other sources are capped at ``maxStartBitrateKbps``; screen share is not,
+    /// because its content needs the bitrate immediately to be legible. Targets under
+    /// 300 kbps get no hint: below that, seeding above the real capacity costs more than the
+    /// ramp it saves.
+    ///
+    /// Same formula as client-sdk-js (`computeTrackStartBitrate`), rust-sdks
+    /// (`compute_start_bitrate_kbps`) and client-sdk-android (`computeTrackStartBitrate`).
+    nonisolated static func startBitrateKbps(targetBps: Int, isScreenShare: Bool) -> Int? {
+        let targetKbps = targetBps / 1000
+        guard targetKbps >= 300 else { return nil }
+        let startKbps = Int((Double(targetKbps) * 0.9).rounded())
+        return isScreenShare ? startKbps : min(startKbps, maxStartBitrateKbps)
+    }
+
+    /// ``startBitrateKbps(targetBps:isScreenShare:)`` for a sender's encodings: the target is
+    /// the sum of the active encodings' `maxBitrate` — every simulcast layer, since they are
+    /// independent streams the estimator has to fund together; for a single (or SVC) encoding
+    /// that is simply its own `maxBitrate`. Encodings without a `maxBitrate` contribute
+    /// nothing, so a track that declares none gets no hint.
+    nonisolated static func startBitrateKbps(for encodings: [LKRTCRtpEncodingParameters], isScreenShare: Bool) -> Int? {
+        let targetBps = encodings.filter(\.isActive).compactMap { $0.maxBitrateBps?.intValue }.reduce(0, +)
+        return startBitrateKbps(targetBps: targetBps, isScreenShare: isScreenShare)
+    }
+
+    /// Whether `section` is a video section that can carry local media. `recvonly` and
+    /// `inactive` are the only directions that cannot, and are exactly where a subscribed,
+    /// pre-populated or unpublished section lands; `sendonly`, `sendrecv` and an omitted
+    /// direction (which defaults to `sendrecv`, RFC 8866 §6.7) all send.
+    nonisolated static func isSendingVideoSection(_ section: SDPMediaSection) -> Bool {
+        section.mediaType == "video" && section.direction != .recvonly && section.direction != .inactive
+    }
+
+    /// The single start bitrate for the peer connection described by `document`: the largest
+    /// hint among its sending video sections whose `a=msid` track id — which libwebrtc sets
+    /// to the sender id — is a key of `kbpsBySenderId`.
+    ///
+    /// libwebrtc reads `x-google-start-bitrate` per m-section but applies it to the shared
+    /// `Call` (`WebRtcVideoSendChannel::ApplyChangedParams` → `SetSdpBitrateParameters`),
+    /// where `RtpBitrateConfigurator` holds one config for the whole peer connection.
+    /// Differing per-section values are therefore last-writer-wins in m-section order — a
+    /// camera capped at 1 Mbps and an uncapped screen share in one offer would seed the
+    /// estimator from whichever section libwebrtc applies last — so every video section is
+    /// given the same number instead.
+    ///
+    /// Only sending sections count: a section that stays in the SDP after its track was
+    /// removed keeps its `a=msid` but sends nothing, and must neither contribute a stale
+    /// target nor consume the one-shot hint.
+    nonisolated static func connectionStartBitrateKbps(for document: SDP, kbpsBySenderId: [String: Int]) -> Int? {
+        document.mediaSections.lazy
+            .filter(isSendingVideoSection)
+            .compactMap { $0.msidTrackId.flatMap { kbpsBySenderId[$0] } }
+            .max()
+    }
+
+    /// Munge the offer to declare `x-google-start-bitrate=<kbps>` — the connection-level value
+    /// from ``connectionStartBitrateKbps(for:kbpsBySenderId:)`` — on every video codec's fmtp
+    /// of every sending video section, inserting an fmtp line for payloads (typically VP8)
+    /// that have none and replacing an existing value. libwebrtc only reads the selected send
+    /// codec's fmtp, so the other codecs' entries are harmless and keep the value uniform
+    /// should the send codec change. An SDP with no sending video, no mapped sender or no
+    /// hint is returned unmodified.
+    ///
+    /// `x-google-max-bitrate` is deliberately never written: the same `Call`-level promotion
+    /// would turn a per-track cap into a ceiling on the connection's total send bandwidth,
+    /// starving concurrent tracks. Per-track and per-layer caps belong in the encodings'
+    /// `maxBitrateBps`, which is scoped per encoding.
+    ///
+    /// Same behaviour as client-sdk-js (`PCTransport.computeConnectionStartBitrate`),
+    /// rust-sdks (`PeerTransport::create_and_send_offer`) and client-sdk-android
+    /// (`PeerConnectionTransport.ensureCodecBitrates`).
+    nonisolated static func mungeVideoStartBitrate(_ sdp: String, kbpsBySenderId: [String: Int]) -> String {
+        guard !kbpsBySenderId.isEmpty else { return sdp }
+        var document = SDP(parsing: sdp)
+        guard let kbps = connectionStartBitrateKbps(for: document, kbpsBySenderId: kbpsBySenderId) else { return sdp }
+        var modified = false
+        for index in document.mediaSections.indices where isSendingVideoSection(document.mediaSections[index]) {
+            for rtpmap in document.mediaSections[index].rtpmaps where startBitrateCodecs.contains(rtpmap.codec.uppercased()) {
+                if document.mediaSections[index].setFmtpParameter("x-google-start-bitrate", value: "\(kbps)", forPayload: rtpmap.payload) {
+                    modified = true
+                }
+            }
+        }
+        return modified ? document.write() : sdp
+    }
+
+    /// Whether any video section of `sdp` declares `x-google-start-bitrate`. Checked on the
+    /// description libwebrtc accepted, since a rejected munge is dropped rather than applied.
+    nonisolated static func carriesVideoStartBitrate(_ sdp: String) -> Bool {
+        SDP(parsing: sdp).mediaSections.contains { section in
+            section.mediaType == "video" && section.fmtps.contains { fmtp in
+                fmtp.parameters.contains { $0.hasPrefix("x-google-start-bitrate=") }
+            }
+        }
+    }
+
     /// Applies `munges` composed left-to-right and sets the result as the local
     /// description. libwebrtc validates munged SDP and rejects some munging types
     /// outright (`IsSdpMungingAllowed`, expanding via field-trial kill switches);
@@ -567,8 +707,13 @@ extension Transport {
         }
     }
 
+    /// `startBitrateKbps` (see ``startBitrateKbps(targetBps:isScreenShare:)``) is recorded
+    /// against the new sender here, before returning, so the offer libwebrtc asks for in
+    /// response to this call cannot be created without it. It only matters until the
+    /// connection-level hint has been written once (``mungeVideoStartBitrate(_:kbpsBySenderId:)``).
     func addTransceiver(with track: LKRTCMediaStreamTrack,
-                        transceiverInit: LKRTCRtpTransceiverInit) throws -> LKRTCRtpTransceiver
+                        transceiverInit: LKRTCRtpTransceiverInit,
+                        startBitrateKbps: Int? = nil) throws -> LKRTCRtpTransceiver
     {
         guard !_isClosed else {
             throw LiveKitError(.invalidState, message: "Transport is closed")
@@ -578,6 +723,10 @@ extension Transport {
             throw LiveKitError(.webRTC, message: "Failed to add transceiver")
         }
 
+        if let startBitrateKbps {
+            _startBitrateKbpsBySenderId[transceiver.sender.senderId] = startBitrateKbps
+        }
+
         return transceiver
     }
 
@@ -600,6 +749,7 @@ extension Transport {
         guard _pc.removeTrack(raw) else {
             throw LiveKitError(.webRTC, message: "Failed to remove track")
         }
+        _startBitrateKbpsBySenderId.removeValue(forKey: sender.senderId)
 
         releaseTransceiver(sender: raw)
     }

Sources/LiveKit/Participant/LocalParticipant.swift

@@ -628,9 +628,8 @@ extension LocalParticipant {
         // Should be already resolved...
         let dimensions = try await track.capturer.dimensionsCompleter.wait()
 
-        let encodings = Utils.computeVideoEncodings(dimensions: dimensions,
-                                                    publishOptions: publishOptions,
-                                                    overrideVideoCodec: videoCodec)
+        let isScreenShare = track.source == .screenShareVideo
+        let encodings = Utils.computeVideoEncodings(dimensions: dimensions, publishOptions: publishOptions, isScreenShare: isScreenShare, overrideVideoCodec: videoCodec)
 
         log("[Publish/Backup] Using encodings: \(encodings.map { $0.toDebugString() }.joined(separator: ", "))")
 
@@ -648,8 +647,10 @@ extension LocalParticipant {
         let degradationPreference = publishOptions.degradationPreference.resolve(for: track.source)
         log("[Publish/Backup] set degradationPreference to \(degradationPreference)")
 
+        let startBitrateKbps = Transport.startBitrateKbps(for: encodings, isScreenShare: isScreenShare)
+
         let sender = try await RTC.run {
-            let transceiver = try publisher.addTransceiver(with: track.mediaTrack.raw, transceiverInit: transInit)
+            let transceiver = try publisher.addTransceiver(with: track.mediaTrack.raw, transceiverInit: transInit, startBitrateKbps: startBitrateKbps)
             try transceiver.set(preferredVideoCodec: videoCodec)
             transceiver.sender.set(degradationPreference: degradationPreference)
             return RTCSender(transceiver.sender)
@@ -875,9 +876,15 @@ extension LocalParticipant {
                                                          populatorFunc)
             }
 
+            let startBitrateKbps = (track is LocalVideoTrack)
+                ? Transport.startBitrateKbps(for: sendEncodings, isScreenShare: track.source == .screenShareVideo)
+                : nil
+
             let negotiateFunc: @Sendable () async throws -> Void = {
                 let sender = try await RTC.run {
-                    let transceiver = try publisher.addTransceiver(with: track.mediaTrack.raw, transceiverInit: transInit)
+                    let transceiver = try publisher.addTransceiver(with: track.mediaTrack.raw,
+                                                                   transceiverInit: transInit,
+                                                                   startBitrateKbps: startBitrateKbps)
 
                     if track is LocalVideoTrack {
                         let publishOptions = (options as? VideoPublishOptions) ?? room._state.roomOptions.defaultVideoPublishOptions

Sources/LiveKit/Support/SDP/SDP.swift

@@ -73,6 +73,11 @@ struct SDPMediaSection {
 
     var mid: String? { attributeValue("mid") }
 
+    /// Track id of the section's `a=msid:<stream id> <track id>` line (RFC 8830 §2), if any.
+    var msidTrackId: String? {
+        attributeValue("msid")?.split(separator: " ").dropFirst().first.map(String.init)
+    }
+
     /// The section's first direction attribute, if any.
     var direction: SDPDirection? {
         lines.dropFirst().lazy.compactMap(SDPDirection.init(line:)).first
@@ -122,6 +127,31 @@ struct SDPMediaSection {
         return false
     }
 
+    /// Sets `name=value` in the fmtp config of `payload`: replaces the parameter's value
+    /// if present, appends it to an existing fmtp line otherwise, and inserts a new
+    /// `a=fmtp:<payload> <name>=<value>` line at the end of the section when the payload
+    /// has none (e.g. VP8, which libwebrtc offers without an fmtp line). Returns `true` if
+    /// the section was modified; `false` when the parameter already carries `value`.
+    @discardableResult
+    mutating func setFmtpParameter(_ name: String, value: String, forPayload payload: String) -> Bool {
+        let parameter = "\(name)=\(value)"
+        for (index, line) in lines.enumerated() {
+            guard let fmtp = Self.fmtp(fromLine: line), fmtp.payload == payload else { continue }
+            // Split without trimming so every other parameter is written back verbatim.
+            var pieces = fmtp.config.split(separator: ";", omittingEmptySubsequences: false).map(String.init)
+            if let existing = pieces.firstIndex(where: { $0.trimmingCharacters(in: .whitespaces).hasPrefix("\(name)=") }) {
+                guard pieces[existing].trimmingCharacters(in: .whitespaces) != parameter else { return false }
+                pieces[existing] = parameter
+            } else {
+                pieces.append(parameter)
+            }
+            lines[index] = "a=fmtp:\(payload) \(pieces.joined(separator: ";"))"
+            return true
+        }
+        lines.append("a=fmtp:\(payload) \(parameter)")
+        return true
+    }
+
     /// Whether an `a=rtcp-fb:<payload> <value>` line with exactly `value` exists for
     /// `payload`. Exact-value match: `nack` does not match `nack pli` (RFC 4585 §4.2
     /// — they are distinct feedback parameters).