From 16d835719b301f3d6659f35356c4e11a09b00abc Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 27 Sep 2026 15:31:28 +0200 Subject: [PATCH 1/3] feat: add encoded frame transforms and a media recorder Encoded frame transforms (insertable streams): RTCRtpSender and RTCRtpReceiver get setTransform(RTCEncodedFrameTransformer), which sees every encoded frame between encoder and packetizer, or depacketizer and decoder, and may read, replace or drop its payload. The transform runs on a thread of its own per sender or receiver, never on a media thread, and frames reach Java without a copy unless the payload is read. Senders gain generateKeyFrame(), receivers requestKeyFrame(). One native transformer is installed per sender or receiver, found through a weak registry, so that a Java transform and native observers share it and a video sender is not restarted on every change. The native extension API gains encoded frame observers, through which the media module's new MediaRecorder writes the frames of senders and receivers into Matroska, WebM or MP4 files without re-encoding. FFmpeg is built with the matroska, webm and mp4 muxers, extract_extradata and the AV1 parser for it. --- README.md | 2 + docs/.vitepress/sidebar.ts | 2 + docs/guide/advanced/encoded-transforms.md | 106 ++++ docs/guide/examples.md | 18 + docs/guide/index.md | 6 + docs/guide/media/media-recording.md | 87 +++ .../examples/EncryptedRecordingExample.java | 500 +++++++++++++++ webrtc-java-media/README.md | 15 +- .../cpp/dependencies/ffmpeg/CMakeLists.txt | 17 +- .../src/main/cpp/include/JNI_MediaRecorder.h | 68 ++ .../src/main/cpp/include/media/ApiCheck.h | 33 + .../cpp/include/media/JavaRecorderObserver.h | 75 +++ .../main/cpp/include/media/MediaRecorder.h | 176 ++++++ .../cpp/include/media/MediaRecorderObserver.h | 47 ++ .../main/cpp/include/media/RecorderTrack.h | 142 +++++ .../src/main/cpp/src/JNI_MediaPlayer.cpp | 21 +- .../src/main/cpp/src/JNI_MediaRecorder.cpp | 118 ++++ .../src/main/cpp/src/media/ApiCheck.cpp | 41 ++ .../cpp/src/media/JavaRecorderObserver.cpp | 182 ++++++ .../src/main/cpp/src/media/MediaRecorder.cpp | 591 ++++++++++++++++++ .../src/main/cpp/src/media/RecorderTrack.cpp | 525 ++++++++++++++++ .../webrtc/media/recorder/MediaRecorder.java | 379 +++++++++++ .../media/recorder/MediaRecorderListener.java | 58 ++ .../media/recorder/MediaRecorderState.java | 35 ++ .../src/main/java/module-info.java | 4 +- .../media/recorder/MediaRecorderTest.java | 294 +++++++++ .../webrtc/media/recorder/TestCall.java | 272 ++++++++ .../src/main/cpp/include/JNI_NativeApi.h | 16 + .../main/cpp/include/JNI_RTCEncodedFrame.h | 52 ++ .../src/main/cpp/include/JNI_RTCRtpReceiver.h | 16 + .../src/main/cpp/include/JNI_RTCRtpSender.h | 16 + .../main/cpp/include/api/EncodedFrameRouter.h | 113 ++++ .../cpp/include/api/EncodedFrameTransformer.h | 117 ++++ .../main/cpp/include/api/EncodedFrameWorker.h | 72 +++ .../include/api/RTCEncodedFrameTransformer.h | 110 ++++ .../src/main/cpp/include/webrtc_java_api.h | 84 +++ webrtc-jni/src/main/cpp/src/JNI_NativeApi.cpp | 27 + .../src/main/cpp/src/JNI_RTCEncodedFrame.cpp | 89 +++ .../src/main/cpp/src/JNI_RTCRtpReceiver.cpp | 51 ++ .../src/main/cpp/src/JNI_RTCRtpSender.cpp | 40 ++ .../main/cpp/src/api/EncodedFrameRouter.cpp | 213 +++++++ .../cpp/src/api/EncodedFrameTransformer.cpp | 264 ++++++++ .../main/cpp/src/api/EncodedFrameWorker.cpp | 156 +++++ .../src/main/cpp/src/api/ExtensionApi.cpp | 49 +- .../src/api/RTCEncodedFrameTransformer.cpp | 220 +++++++ .../onvoid/webrtc/RTCEncodedAudioFrame.java | 92 +++ .../dev/onvoid/webrtc/RTCEncodedFrame.java | 339 ++++++++++ .../webrtc/RTCEncodedFrameTransformer.java | 69 ++ .../onvoid/webrtc/RTCEncodedVideoFrame.java | 119 ++++ .../dev/onvoid/webrtc/RTCRtpReceiver.java | 24 + .../java/dev/onvoid/webrtc/RTCRtpSender.java | 28 + .../dev/onvoid/webrtc/internal/NativeApi.java | 44 ++ .../webrtc/RTCEncodedFrameTransformTests.java | 372 +++++++++++ .../java/dev/onvoid/webrtc/TestMediaCall.java | 234 +++++++ 54 files changed, 6818 insertions(+), 22 deletions(-) create mode 100644 docs/guide/advanced/encoded-transforms.md create mode 100644 docs/guide/media/media-recording.md create mode 100644 webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java create mode 100644 webrtc-java-media/src/main/cpp/include/JNI_MediaRecorder.h create mode 100644 webrtc-java-media/src/main/cpp/include/media/ApiCheck.h create mode 100644 webrtc-java-media/src/main/cpp/include/media/JavaRecorderObserver.h create mode 100644 webrtc-java-media/src/main/cpp/include/media/MediaRecorder.h create mode 100644 webrtc-java-media/src/main/cpp/include/media/MediaRecorderObserver.h create mode 100644 webrtc-java-media/src/main/cpp/include/media/RecorderTrack.h create mode 100644 webrtc-java-media/src/main/cpp/src/JNI_MediaRecorder.cpp create mode 100644 webrtc-java-media/src/main/cpp/src/media/ApiCheck.cpp create mode 100644 webrtc-java-media/src/main/cpp/src/media/JavaRecorderObserver.cpp create mode 100644 webrtc-java-media/src/main/cpp/src/media/MediaRecorder.cpp create mode 100644 webrtc-java-media/src/main/cpp/src/media/RecorderTrack.cpp create mode 100644 webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorder.java create mode 100644 webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorderListener.java create mode 100644 webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorderState.java create mode 100644 webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/MediaRecorderTest.java create mode 100644 webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java create mode 100644 webrtc-jni/src/main/cpp/include/JNI_RTCEncodedFrame.h create mode 100644 webrtc-jni/src/main/cpp/include/api/EncodedFrameRouter.h create mode 100644 webrtc-jni/src/main/cpp/include/api/EncodedFrameTransformer.h create mode 100644 webrtc-jni/src/main/cpp/include/api/EncodedFrameWorker.h create mode 100644 webrtc-jni/src/main/cpp/include/api/RTCEncodedFrameTransformer.h create mode 100644 webrtc-jni/src/main/cpp/src/JNI_RTCEncodedFrame.cpp create mode 100644 webrtc-jni/src/main/cpp/src/api/EncodedFrameRouter.cpp create mode 100644 webrtc-jni/src/main/cpp/src/api/EncodedFrameTransformer.cpp create mode 100644 webrtc-jni/src/main/cpp/src/api/EncodedFrameWorker.cpp create mode 100644 webrtc-jni/src/main/cpp/src/api/RTCEncodedFrameTransformer.cpp create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedAudioFrame.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedFrame.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedFrameTransformer.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedVideoFrame.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/RTCEncodedFrameTransformTests.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java diff --git a/README.md b/README.md index 8a3a0e1e..43dc8f77 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ For more detailed information, check out the documentation: - [Guides](https://jrtc.dev/guide/) - Comprehensive documentation on using the library - [Examples](https://jrtc.dev/guide/examples) - Sample code demonstrating various features - [Media Files](https://jrtc.dev/guide/media/media-files) - Sending video and audio files with the media module +- [Media Recording](https://jrtc.dev/guide/media/media-recording) - Recording calls into media files with the media module +- [Encoded Transforms](https://jrtc.dev/guide/advanced/encoded-transforms) - Reading and changing encoded frames, e.g. for end-to-end encryption - [Build Notes](https://jrtc.dev/guide/build) - Instructions for building the library from source ## License diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index 07228b5c..55a77a74 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -23,6 +23,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { { text: 'Media Constraints', link: '/media/constraints' }, { text: 'Media Directionality', link: '/media/directionality' }, { text: 'Media Files', link: '/media/media-files' }, + { text: 'Media Recording', link: '/media/media-recording' }, ], }, { @@ -72,6 +73,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { collapsed: false, items: [ { text: 'Field Trials', link: '/advanced/field-trials' }, + { text: 'Encoded Transforms', link: '/advanced/encoded-transforms' }, ], }, ] diff --git a/docs/guide/advanced/encoded-transforms.md b/docs/guide/advanced/encoded-transforms.md new file mode 100644 index 00000000..507cae98 --- /dev/null +++ b/docs/guide/advanced/encoded-transforms.md @@ -0,0 +1,106 @@ +# Encoded Transforms + +This guide explains how to read and change the encoded frames of a call as they pass through an `RTCRtpSender` or an `RTCRtpReceiver`, the way [WebRTC Encoded Transforms](https://www.w3.org/TR/webrtc-encoded-transform/) (also known as insertable streams) do in a browser. It covers: + +- Setting a transform on a sender or a receiver +- Reading and replacing a frame's payload +- Dropping frames +- What each frame tells about itself +- Asking for key frames +- End-to-end encryption as a worked example +- Threading and performance + +A sender's transform sees every frame after the encoder and before the packetizer; a receiver's sees every frame after the depacketizer and before the decoder. Everything between the two, whether that is the network, a TURN server or an SFU, only ever carries what the transform produced. That is what makes end-to-end encryption possible, and also frame metadata, watermarks, or analysis of the encoded stream. + +## Setting a Transform + +An `RTCEncodedFrameTransformer` is a functional interface with a single method, `transform(RTCEncodedFrame frame)`. It changes the frame in place, and the frame is sent on when the method returns: + +```java +RTCRtpSender sender = peerConnection.addTrack(videoTrack, List.of("stream")); + +sender.setTransform(frame -> { + System.out.println(frame); // mime type, size, key frame, timestamp, SSRC +}); +``` + +On the receiving side, the receivers are there once the remote description is set: + +```java +for (RTCRtpReceiver receiver : peerConnection.getReceivers()) { + receiver.setTransform(frame -> inspect(frame)); +} +``` + +`setTransform(null)` removes the transform again, after which frames pass unchanged. + +::: tip +Set the transform of a video sender before negotiating when you can. The first transform set on a running video sender restarts its send stream, which costs a key frame. Later changes, including removing it, cost nothing. +::: + +The transform belongs to the native sender or receiver, not to the Java object: every `RTCRtpSender` instance for the same sender shares it, setting it through one replaces what was set through another, and disposing of an instance leaves it in place. + +## Reading and Changing the Payload + +`getData()` returns the payload in a `ByteBuffer`, from position zero to its size. The buffer holds a copy, which may be changed freely; `setData(...)` then writes it back: + +```java +receiver.setTransform(frame -> { + ByteBuffer data = frame.getData(); + + for (int i = 0; i < data.limit(); i++) { + data.put(i, (byte) (data.get(i) ^ 0x5A)); + } + + frame.setData(data); +}); +``` + +`setData` takes a `ByteBuffer` (its remaining bytes) or a `byte[]`, and the new payload may be larger or smaller than the old one. The payload is only copied into Java when `getData()` is called, so a transform that looks at the metadata alone costs no copy at all. + +A frame, and the buffer `getData()` returned, is valid only while `transform` runs and only on its thread. The buffer's memory is reused for the next frame, so copy what you need to keep. Reading or changing a frame after the transform returned throws an `IllegalStateException`; its metadata stays readable. + +## Dropping Frames + +`frame.drop()` releases the frame instead of sending it on. A transform that throws drops the frame as well, and the exception goes to the thread's uncaught exception handler. It is dropped rather than sent on unchanged because an encryption transform that failed must never send the frame in the clear. + +A receiver cannot decode a video frame that depends on one it never got, so dropping video frames usually means waiting for the next key frame. + +## Frame Metadata + +Every `RTCEncodedFrame` tells its `getMimeType()` (e.g. `video/VP8`, `audio/opus`), `getSize()`, RTP `getTimestamp()`, `getSsrc()`, `getPayloadType()` and `getCaptureTimeUs()`. The subclasses add what is particular to their kind: + +| `RTCEncodedVideoFrame` | `RTCEncodedAudioFrame` | +| --- | --- | +| `isKeyFrame()` | `getSequenceNumber()` (received frames) | +| `getWidth()`, `getHeight()` (key frames) | `getAudioLevel()` in -dBov | +| `getRid()` (simulcast layer) | `getContributingSources()` | +| `getFrameId()`, `getSpatialIndex()`, `getTemporalIndex()` | | + +## Key Frames + +A transform that needs a key frame, e.g. to start over after a key change, can ask for one: + +```java +videoSender.generateKeyFrame(); // the local encoder makes the next frame a key frame +videoReceiver.requestKeyFrame(); // asks the remote sender for one +``` + +Both do nothing for audio. + +## End-to-End Encryption + +A complete example with AES-GCM is in [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java). Two points matter for any encrypting transform: + +- **Keep the codec header in the clear.** The packetizer and depacketizer read the start of each frame: a VP8 receiver finds key frames and the frame size in the first bytes. Browsers leave 10 bytes of a VP8 key frame, 3 of a VP8 delta frame and 1 of an Opus frame unencrypted, and authenticate them as associated data instead. H.264 is split at its NAL unit start codes before it is sent, so encrypting it whole breaks the packetizer; prefer VP8, VP9 or AV1 for encrypted calls, or leave the NAL unit headers in the clear. +- **Make each frame self-contained.** Frames are lost, so each needs what it takes to be decrypted, such as its IV, and a receiver must drop a frame that fails to authenticate rather than pass it on. + +## Threading and Performance + +The transform never runs on a thread that carries media. Each sender and receiver with a transform gets a thread of its own. Frames are queued to that thread and transformed there one at a time, in order, and are handed back to WebRTC from it. This means that: + +- a slow transform delays only its own frames, never the connection; +- a transform may call back into the peer connection, even close it, without deadlocking; +- a transform that falls several seconds behind has frames dropped until it catches up, rather than having them queue up in memory. + +Per frame, the cost is one small Java object and, only if the payload is read, one copy into a buffer that is reused from frame to frame. Senders and receivers without a transform are not touched at all. diff --git a/docs/guide/examples.md b/docs/guide/examples.md index a6636a84..5e9a4e9a 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -95,6 +95,24 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.MediaFilePlayerExampl ``` ::: +## Encrypted Recording + +The [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java) encrypts a call end to end with encoded frame transforms, and records what the receiving side decrypted into a media file. See the [Encoded Transforms](/guide/advanced/encoded-transforms) and [Media Recording](/guide/media/media-recording) guides for the APIs it uses. + +**Key features demonstrated:** +- Encrypting every encoded audio and video frame with AES-GCM in an `RTCEncodedFrameTransformer` on the senders, and decrypting it on the receivers +- Keeping the codec header of each frame in the clear for the packetizer, and authenticating it instead +- Dropping frames that fail to encrypt or to authenticate, rather than passing them on +- Recording the receivers with a `MediaRecorder`, without decoding or re-encoding + +::: info +Like the [Media File](#media-file) example, this one needs the `webrtc-java-media` module. The output file is optional; its extension picks the container. + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.EncryptedRecordingExample" -D"exec.args=call.mkv" +``` +::: + ## Web Client The [`WebClientExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java) demonstrates how to combine WebSocket signaling with WebRTC peer connections for real-time communication between web and Java clients. diff --git a/docs/guide/index.md b/docs/guide/index.md index b851497e..c478919a 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -8,6 +8,7 @@ This section provides detailed guides for various features of the webrtc-java li - [Bitrate and Framerate Constraints](/guide/media/constraints) - Controlling media quality - [Send-only and Receive-only](/guide/media/directionality) - Configure transceiver directions (send-only, receive-only or inactive) - [Media Files](/guide/media/media-files) - Sending video and audio files instead of a camera and microphone +- [Media Recording](/guide/media/media-recording) - Recording what a call sends or receives into a media file, without re-encoding ## Audio @@ -36,6 +37,11 @@ This section provides detailed guides for various features of the webrtc-java li - [RTC Stats](/guide/monitoring/rtc-stats) - Monitoring connection quality and performance - [Logging](/guide/monitoring/logging) - Configuring and using the logging system +## Advanced + +- [Field Trials](/guide/advanced/field-trials) - Enabling experimental features and tuning WebRTC internals +- [Encoded Transforms](/guide/advanced/encoded-transforms) - Reading and changing encoded frames, e.g. for end-to-end encryption + ## Additional Resources For a complete API reference, check the [JavaDoc](https://javadoc.io/doc/dev.onvoid.webrtc/webrtc-java/latest/index.html). \ No newline at end of file diff --git a/docs/guide/media/media-recording.md b/docs/guide/media/media-recording.md new file mode 100644 index 00000000..db3c23c9 --- /dev/null +++ b/docs/guide/media/media-recording.md @@ -0,0 +1,87 @@ +# Media Recording + +This guide explains how to record what a peer connection sends or receives into a media file with `MediaRecorder`, from the `webrtc-java-media` module. It covers: + +- Recording senders and receivers +- Choosing the container +- Following a recording with a listener +- How tracks are started, timed and left out +- Recording an end-to-end encrypted call + +`MediaRecorder` writes the encoded frames of each sender or receiver into the file as they are, without decoding or re-encoding them. A recording therefore costs next to no CPU, keeps exactly the quality that went over the network, and needs no codec at all, only the container. The frames are taken in native code, straight from WebRTC, and written by a thread of the recorder's own, so neither Java nor a slow disk ever holds up the call. + +See [Media Files](/guide/media/media-files) for how to add the module to a project. + +## Recording a Call + +Add the senders and receivers to record, then start: + +```java +try (MediaRecorder recorder = new MediaRecorder(Paths.get("call.mkv"))) { + recorder.addTrack(videoReceiver); + recorder.addTrack(audioReceiver); + recorder.start(); + + // ... the call goes on ... + + recorder.stop(); +} +``` + +A sender's frames are recorded as they leave the encoder, a receiver's as they go to the decoder. Senders and receivers of any number of peer connections can go into one file, e.g. both sides of a call. + +`stop()` waits until everything received so far is written and the file is finished, and returns whether the file holds a recording. A recorder that never got any media to write deletes its file rather than leave an unplayable one behind. `close()` stops a recording that still runs. + +::: warning +Keep the senders and receivers undisposed while recording. The recorder asks them for key frames, and a disposed one cannot be asked. +::: + +## Containers + +The container follows the file name: + +| Extension | Video | Audio | Notes | +| --- | --- | --- | --- | +| `.mkv` | VP8, VP9, AV1, H.264, H.265 | Opus, G.711 | Holds every codec WebRTC sends; the safe choice. | +| `.webm` | VP8, VP9, AV1 | Opus | Plays in browsers. | +| `.mp4` | VP9, AV1, H.264, H.265 | Opus | Written fragmented, so it plays up to where a recording was cut short. | + +A name FFmpeg does not know gets Matroska. A track whose codec the file cannot hold, VP8 in MP4 for example, is left out and reported as a warning, and the rest is recorded. + +## Following a Recording + +A `MediaRecorderListener` hears when the file begins, what was left out, and whether writing failed: + +```java +recorder.setListener(new MediaRecorderListener() { + + @Override + public void onStarted() { + System.out.println("Recording"); + } + + @Override + public void onWarning(String message) { + System.out.println("Warning: " + message); + } + + @Override + public void onError(String message) { + System.out.println("Failed: " + message); + } +}); +``` + +Calls arrive in order on a thread the recorder keeps for them, so a listener may take its time and may even stop the recorder. + +## How Tracks Are Recorded + +- **Video starts at a key frame.** A decoder can do nothing with the frames before one, so they are skipped. The recorder asks the sender or receiver for a key frame, so recording starts within moments instead of waiting until WebRTC sends one of its own accord, which it rarely does. +- **The file begins once the tracks are known.** The file header describes every stream, and a stream is only known from its first frames. The file begins once every track has sent some, or once the tracks that have waited three seconds for the rest. A track that sends nothing by then is left out, with a warning. +- **Tracks share one timeline.** Each track is placed by when its first frame arrived, and follows its own RTP timestamps from there, so its timing is exact. The tracks of one sender line up within the jitter of the network. +- **One simulcast layer.** A sender with simulcast is recorded at the layer whose frames reach the recorder first. +- **Nothing is lost to a slow disk, up to a point.** Frames queue up in memory for the writer, up to 64 MB. Beyond that, frames are dropped, and video starts over at the next key frame. + +## Recording an Encrypted Call + +A sender's frames are recorded before an [encoded transform](/guide/advanced/encoded-transforms) runs on them, and a receiver's after it. An end-to-end encrypted call is therefore recorded in the clear, on either side. [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java) shows both together. diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java new file mode 100644 index 00000000..ec894559 --- /dev/null +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java @@ -0,0 +1,500 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.examples; + +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import dev.onvoid.webrtc.CreateSessionDescriptionObserver; +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.PeerConnectionObserver; +import dev.onvoid.webrtc.RTCAnswerOptions; +import dev.onvoid.webrtc.RTCConfiguration; +import dev.onvoid.webrtc.RTCEncodedFrame; +import dev.onvoid.webrtc.RTCEncodedFrameTransformer; +import dev.onvoid.webrtc.RTCEncodedVideoFrame; +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCOfferOptions; +import dev.onvoid.webrtc.RTCPeerConnection; +import dev.onvoid.webrtc.RTCPeerConnectionState; +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.RTCRtpSender; +import dev.onvoid.webrtc.RTCRtpTransceiver; +import dev.onvoid.webrtc.RTCSessionDescription; +import dev.onvoid.webrtc.SetSessionDescriptionObserver; +import dev.onvoid.webrtc.media.audio.AudioDeviceModule; +import dev.onvoid.webrtc.media.audio.AudioLayer; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.recorder.MediaRecorder; +import dev.onvoid.webrtc.media.recorder.MediaRecorderListener; +import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoTrack; + +/** + * Encrypts a call end to end with encoded frame transforms, and records what + * the receiving side decrypted into a media file. + *

+ * This example shows how to: + *

+ *

+ * Both peers live in this application and the key is simply shared between + * them; a real application exchanges it over its signaling channel, or better + * derives it with a key agreement such as MLS. + *

+ * Run it with an optional output file, whose extension picks the container: + * 

+ * java dev.onvoid.webrtc.examples.EncryptedRecordingExample call.mkv
+ *
+ * + * @author Alex Andres + */ +public class EncryptedRecordingExample { + + private static final Logger LOG = Logger.getLogger(EncryptedRecordingExample.class.getName()); + + private static final int RECORD_SECONDS = 10; + + private static final int WIDTH = 640; + private static final int HEIGHT = 480; + + + public static void main(String[] args) throws Exception { + Path file = Paths.get(args.length > 0 ? args[0] : "encrypted-call.mkv"); + + // Pushed audio needs a factory whose audio layer does not capture. + AudioDeviceModule audioModule = new AudioDeviceModule(AudioLayer.kDummyAudio); + PeerConnectionFactory factory = new PeerConnectionFactory(audioModule); + + CustomVideoSource videoSource = new CustomVideoSource(); + CustomAudioSource audioSource = new CustomAudioSource(); + VideoTrack videoTrack = factory.createVideoTrack("video", videoSource); + AudioTrack audioTrack = factory.createAudioTrack("audio", audioSource); + + Peer caller = new Peer(factory); + Peer callee = new Peer(factory); + caller.remote = callee; + callee.remote = caller; + + FrameCipher cipher = new FrameCipher(newKey()); + + RTCRtpSender videoSender = caller.connection.addTrack(videoTrack, List.of("stream")); + RTCRtpSender audioSender = caller.connection.addTrack(audioTrack, List.of("stream")); + + // Set before negotiating, so the encoder needs no restart. + videoSender.setTransform(cipher::encrypt); + audioSender.setTransform(cipher::encrypt); + + callee.setRemoteDescription(caller.createOffer()); + caller.setRemoteDescription(callee.createAnswer()); + + // The callee's transceivers follow the order of the offer. + RTCRtpTransceiver[] transceivers = callee.connection.getTransceivers(); + RTCRtpReceiver videoReceiver = transceivers[0].getReceiver(); + RTCRtpReceiver audioReceiver = transceivers[1].getReceiver(); + + videoReceiver.setTransform(cipher::decrypt); + audioReceiver.setTransform(cipher::decrypt); + + caller.connected.await(10, TimeUnit.SECONDS); + callee.connected.await(10, TimeUnit.SECONDS); + + MediaGenerator generator = new MediaGenerator(videoSource, audioSource); + generator.start(); + + try (MediaRecorder recorder = new MediaRecorder(file)) { + recorder.setListener(new MediaRecorderListener() { + + @Override + public void onStarted() { + LOG.info("Recording into " + recorder.getFile()); + } + + @Override + public void onWarning(String message) { + LOG.warning(message); + } + + @Override + public void onError(String message) { + LOG.severe(message); + } + }); + + // The receivers hand the recorder what they decrypted. + recorder.addTrack(videoReceiver); + recorder.addTrack(audioReceiver); + recorder.start(); + + Thread.sleep(TimeUnit.SECONDS.toMillis(RECORD_SECONDS)); + + if (recorder.stop()) { + LOG.info("Recorded " + RECORD_SECONDS + " seconds into " + recorder.getFile()); + } + } + finally { + generator.stop(); + + LOG.info(String.format("Encrypted %d frames, decrypted %d, rejected %d", + cipher.encrypted.get(), cipher.decrypted.get(), cipher.rejected.get())); + + videoReceiver.dispose(); + audioReceiver.dispose(); + for (RTCRtpTransceiver transceiver : transceivers) { + transceiver.dispose(); + } + videoSender.dispose(); + audioSender.dispose(); + + caller.connection.close(); + callee.connection.close(); + + videoTrack.dispose(); + audioTrack.dispose(); + videoSource.dispose(); + audioSource.dispose(); + + factory.dispose(); + audioModule.dispose(); + } + } + + private static SecretKey newKey() { + byte[] key = new byte[16]; + new SecureRandom().nextBytes(key); + + return new SecretKeySpec(key, "AES"); + } + + + /** + * AES-GCM over the payload of each frame. An encrypted frame is laid out + * as the clear header, the ciphertext with its tag, then the 12-byte IV. + * Each transform runs on a thread of its own, one per sender or receiver, + * so each call makes its own Cipher rather than share one. + */ + private static class FrameCipher { + + private static final int IV_SIZE = 12; + private static final int TAG_BITS = 128; + + private final SecretKey key; + private final SecureRandom random = new SecureRandom(); + + // The audio and the video transform each run on a thread of their own. + final AtomicLong encrypted = new AtomicLong(); + final AtomicLong decrypted = new AtomicLong(); + final AtomicLong rejected = new AtomicLong(); + + + FrameCipher(SecretKey key) { + this.key = key; + } + + void encrypt(RTCEncodedFrame frame) { + ByteBuffer data = frame.getData(); + int clear = clearBytes(frame, data.remaining()); + + byte[] iv = new byte[IV_SIZE]; + random.nextBytes(iv); + + try { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(TAG_BITS, iv)); + + ByteBuffer header = data.duplicate(); + header.limit(clear); + cipher.updateAAD(header); + + ByteBuffer out = ByteBuffer.allocate(clear + cipher.getOutputSize(data.remaining() - clear) + IV_SIZE); + ByteBuffer headerCopy = data.duplicate(); + headerCopy.limit(clear); + out.put(headerCopy); + + data.position(clear); + cipher.doFinal(data, out); + out.put(iv); + out.flip(); + + frame.setData(out); + encrypted.incrementAndGet(); + } + catch (GeneralSecurityException e) { + // Never send a frame that failed to encrypt. + frame.drop(); + LOG.log(Level.WARNING, "Encrypting a frame failed", e); + } + } + + void decrypt(RTCEncodedFrame frame) { + ByteBuffer data = frame.getData(); + int size = data.remaining(); + int clear = clearBytes(frame, size); + + if (size < clear + TAG_BITS / 8 + IV_SIZE) { + frame.drop(); + rejected.incrementAndGet(); + return; + } + + byte[] iv = new byte[IV_SIZE]; + data.position(size - IV_SIZE); + data.get(iv); + + try { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(TAG_BITS, iv)); + + ByteBuffer header = data.duplicate(); + header.position(0); + header.limit(clear); + cipher.updateAAD(header); + + ByteBuffer out = ByteBuffer.allocate(size); + ByteBuffer headerCopy = data.duplicate(); + headerCopy.position(0); + headerCopy.limit(clear); + out.put(headerCopy); + + ByteBuffer body = data.duplicate(); + body.position(clear); + body.limit(size - IV_SIZE); + cipher.doFinal(body, out); + out.flip(); + + frame.setData(out); + decrypted.incrementAndGet(); + } + catch (GeneralSecurityException e) { + // A frame that does not authenticate was tampered with, or + // encrypted with another key: it must not reach the decoder. + frame.drop(); + rejected.incrementAndGet(); + } + } + + /** + * The bytes left in the clear: the VP8 payload header, which the + * receiver's depacketizer reads to find key frames and the frame + * size (10 bytes on a key frame, 3 otherwise), and the Opus TOC byte. + */ + private static int clearBytes(RTCEncodedFrame frame, int size) { + int clear = 1; + + if (frame instanceof RTCEncodedVideoFrame video) { + clear = video.isKeyFrame() ? 10 : 3; + } + + return Math.min(clear, size); + } + } + + + /** + * Pushes a moving test picture at 30 frames and a tone in 10 ms chunks + * per second, in real time. + */ + private static class MediaGenerator { + + private final CustomVideoSource videoSource; + private final CustomAudioSource audioSource; + + private volatile boolean running; + private Thread thread; + + + MediaGenerator(CustomVideoSource videoSource, CustomAudioSource audioSource) { + this.videoSource = videoSource; + this.audioSource = audioSource; + } + + void start() { + running = true; + thread = new Thread(this::run, "media-generator"); + thread.setDaemon(true); + thread.start(); + } + + void stop() throws InterruptedException { + running = false; + thread.join(); + } + + private void run() { + byte[] audio = new byte[480 * 2]; + long startNs = System.nanoTime(); + long chunks = 0; + long frames = 0; + + while (running) { + long elapsedMs = (System.nanoTime() - startNs) / 1_000_000; + + while (chunks * 10 <= elapsedMs) { + for (int i = 0; i < 480; i++) { + double t = (chunks * 480 + i) / 48000.0; + short sample = (short) (Math.sin(2 * Math.PI * 440 * t) * 6000); + audio[2 * i] = (byte) sample; + audio[2 * i + 1] = (byte) (sample >> 8); + } + + audioSource.pushAudio(audio, 16, 48000, 1, 480); + chunks++; + } + + if (frames * 1000 / 30 <= elapsedMs) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(WIDTH, HEIGHT); + ByteBuffer y = buffer.getDataY(); + int stride = buffer.getStrideY(); + int offset = (int) (frames * 4); + + // Diagonal stripes that move, so every frame differs. + for (int row = 0; row < HEIGHT; row++) { + for (int col = 0; col < WIDTH; col++) { + y.put(row * stride + col, (byte) (((row + col + offset) / 16 % 2) * 180 + 40)); + } + } + + VideoFrame frame = new VideoFrame(buffer, 0); + videoSource.pushFrame(frame); + frame.release(); + frames++; + } + + try { + Thread.sleep(2); + } + catch (InterruptedException e) { + return; + } + } + } + } + + + /** + * One end of the call, exchanging candidates with the other directly. + */ + private static class Peer implements PeerConnectionObserver { + + final RTCPeerConnection connection; + final CountDownLatch connected = new CountDownLatch(1); + + volatile Peer remote; + + + Peer(PeerConnectionFactory factory) { + connection = factory.createPeerConnection(new RTCConfiguration(), this); + } + + @Override + public void onIceCandidate(RTCIceCandidate candidate) { + remote.connection.addIceCandidate(candidate); + } + + @Override + public void onConnectionChange(RTCPeerConnectionState state) { + if (state == RTCPeerConnectionState.CONNECTED) { + connected.countDown(); + } + } + + RTCSessionDescription createOffer() throws Exception { + CompletableFuture created = new CompletableFuture<>(); + connection.createOffer(new RTCOfferOptions(), created(created)); + + return setLocalDescription(created.get(10, TimeUnit.SECONDS)); + } + + RTCSessionDescription createAnswer() throws Exception { + CompletableFuture created = new CompletableFuture<>(); + connection.createAnswer(new RTCAnswerOptions(), created(created)); + + return setLocalDescription(created.get(10, TimeUnit.SECONDS)); + } + + void setRemoteDescription(RTCSessionDescription description) throws Exception { + CompletableFuture set = new CompletableFuture<>(); + connection.setRemoteDescription(description, set(set)); + set.get(10, TimeUnit.SECONDS); + } + + private RTCSessionDescription setLocalDescription(RTCSessionDescription description) throws Exception { + CompletableFuture set = new CompletableFuture<>(); + connection.setLocalDescription(description, set(set)); + set.get(10, TimeUnit.SECONDS); + + return description; + } + + private static CreateSessionDescriptionObserver created(CompletableFuture future) { + return new CreateSessionDescriptionObserver() { + + @Override + public void onSuccess(RTCSessionDescription description) { + future.complete(description); + } + + @Override + public void onFailure(String error) { + future.completeExceptionally(new IllegalStateException(error)); + } + }; + } + + private static SetSessionDescriptionObserver set(CompletableFuture future) { + return new SetSessionDescriptionObserver() { + + @Override + public void onSuccess() { + future.complete(null); + } + + @Override + public void onFailure(String error) { + future.completeExceptionally(new IllegalStateException(error)); + } + }; + } + } +} diff --git a/webrtc-java-media/README.md b/webrtc-java-media/README.md index c496a8f8..8ead425f 100644 --- a/webrtc-java-media/README.md +++ b/webrtc-java-media/README.md @@ -2,7 +2,8 @@ Media extension for [webrtc-java](https://github.com/devopvoid/webrtc-java). It reads media files and network streams with FFmpeg and feeds them into a peer connection, so an application can send -a video file the way it would send a camera. +a video file the way it would send a camera. It also records what a peer connection sends or +receives into media files. ```java MediaFileSource source = new MediaFileSource(Path.of("movie.mp4")); @@ -16,6 +17,18 @@ peerConnection.addTrack(audioTrack, List.of("stream")); source.play(); ``` +Recording writes the encoded frames into the file as they are, without decoding or re-encoding: + +```java +try (MediaRecorder recorder = new MediaRecorder(Path.of("call.mkv"))) { + recorder.addTrack(videoReceiver); + recorder.addTrack(audioReceiver); + recorder.start(); + // ... + recorder.stop(); +} +``` + ## How it fits together Decoding happens entirely in native code. The module does not carry frames through Java: it calls diff --git a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt index 7dbc3384..a4189938 100644 --- a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt +++ b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt @@ -14,8 +14,8 @@ project(ffmpeg) # The configuration is deliberately LGPL only: no --enable-gpl and no # --enable-nonfree, and the libraries are shared so that they can be replaced, # which is what the LGPL asks of us. Only the demuxers, decoders, parsers and -# protocols this module actually plays are enabled; everything else is off, to -# keep the shipped libraries small. +# protocols this module actually plays, and the muxers it records into, are +# enabled; everything else is off, to keep the shipped libraries small. # set(FFMPEG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../third-party/ffmpeg") @@ -98,6 +98,19 @@ set(FFMPEG_COMPONENTS --enable-parser=opus --enable-parser=vorbis --enable-parser=flac + # Recording: the containers MediaRecorder writes the encoded frames of a + # peer connection into, without transcoding. Matroska carries every codec + # WebRTC sends; WebM and MP4 each carry a subset. The muxers pull in the + # bitstream filters they insert on their own (vp9_superframe and the like). + --enable-muxer=matroska + --enable-muxer=webm + --enable-muxer=mp4 + --enable-muxer=mov + # Takes the parameter sets of H.264, H.265 and AV1 out of a key frame, as + # the containers want them in their headers. + --enable-bsf=extract_extradata + # Reads the frame size from an AV1 key frame when WebRTC does not say. + --enable-parser=av1 # Local files, and what RTSP runs over. Protocols that need TLS are left # out until a TLS backend is chosen per platform. --enable-protocol=file diff --git a/webrtc-java-media/src/main/cpp/include/JNI_MediaRecorder.h b/webrtc-java-media/src/main/cpp/include/JNI_MediaRecorder.h new file mode 100644 index 00000000..6b6b56be --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/JNI_MediaRecorder.h @@ -0,0 +1,68 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_recorder_MediaRecorder */ + +#ifndef _Included_dev_onvoid_webrtc_media_recorder_MediaRecorder +#define _Included_dev_onvoid_webrtc_media_recorder_MediaRecorder +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_recorder_MediaRecorder + * Method: create + * Signature: (Ljava/lang/String;J)J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_create + (JNIEnv *, jobject, jstring, jlong); + + /* + * Class: dev_onvoid_webrtc_media_recorder_MediaRecorder + * Method: addTrack + * Signature: (JJ)I + */ + JNIEXPORT jint JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_addTrack + (JNIEnv *, jclass, jlong, jlong); + + /* + * Class: dev_onvoid_webrtc_media_recorder_MediaRecorder + * Method: start + * Signature: (J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_start + (JNIEnv *, jclass, jlong); + + /* + * Class: dev_onvoid_webrtc_media_recorder_MediaRecorder + * Method: stop + * Signature: (J)Z + */ + JNIEXPORT jboolean JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_stop + (JNIEnv *, jclass, jlong); + + /* + * Class: dev_onvoid_webrtc_media_recorder_MediaRecorder + * Method: dispose + * Signature: (J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_dispose + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/ApiCheck.h b/webrtc-java-media/src/main/cpp/include/media/ApiCheck.h new file mode 100644 index 00000000..9e6927d0 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/ApiCheck.h @@ -0,0 +1,33 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_API_CHECK_H_ +#define WEBRTC_JAVA_MEDIA_API_CHECK_H_ + +#include "webrtc_java_api.h" + +#include + +namespace ffmpeg +{ + // Checks that the function table of the loaded webrtc-java library is one + // this module can use: the interface version it was built against, and at + // least every member it knows of. Returns what is wrong, or an empty + // string if nothing is. + std::string CheckApi(const webrtc_java_api * api); +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/JavaRecorderObserver.h b/webrtc-java-media/src/main/cpp/include/media/JavaRecorderObserver.h new file mode 100644 index 00000000..29d4ffee --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/JavaRecorderObserver.h @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_JAVA_RECORDER_OBSERVER_H_ +#define WEBRTC_JAVA_MEDIA_JAVA_RECORDER_OBSERVER_H_ + +#include "media/MediaRecorderObserver.h" + +#include + +namespace ffmpeg +{ + // Passes a recorder's reports on to its Java MediaRecorder. + // + // The Java side does no more than queue each report for a thread of its + // own, so these calls return at once and the writer thread never waits + // for the application, nor for WebRTC when a key frame is asked for. + // + // The writer thread is attached to the JVM for each call and detached + // after it: reports are rare, a handful per recording plus at most one + // key frame request per second and track. + class JavaRecorderObserver : public MediaRecorderObserver + { + public: + // Keeps a global reference to the given Java MediaRecorder. + JavaRecorderObserver(JNIEnv * env, jobject recorder); + ~JavaRecorderObserver() override; + + JavaRecorderObserver(const JavaRecorderObserver &) = delete; + JavaRecorderObserver & operator=(const JavaRecorderObserver &) = delete; + + void OnStarted() override; + void OnWarning(const std::string & message) override; + void OnError(const std::string & message) override; + void OnKeyFrameNeeded(int track) override; + + private: + // Calls a void Java method taking one String. + void CallWithMessage(jmethodID method, const std::string & message); + + // Returns an environment for the calling thread, attaching it if + // it is not known to the JVM. Sets attached when it did, which is + // then the caller's to undo. + JNIEnv * Attach(bool * attached); + void Detach(bool attached); + + // A thread already carrying an exception must not call Java. + static bool CanCallJava(JNIEnv * env); + + // Nothing called here may leave an exception behind. + static void ClearPendingException(JNIEnv * env); + + JavaVM * vm_ = nullptr; + jobject recorder_ = nullptr; + jmethodID on_started_ = nullptr; + jmethodID on_warning_ = nullptr; + jmethodID on_error_ = nullptr; + jmethodID on_key_frame_needed_ = nullptr; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/MediaRecorder.h b/webrtc-java-media/src/main/cpp/include/media/MediaRecorder.h new file mode 100644 index 00000000..b63ec68c --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/MediaRecorder.h @@ -0,0 +1,176 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_MEDIA_RECORDER_H_ +#define WEBRTC_JAVA_MEDIA_MEDIA_RECORDER_H_ + +#include "media/MediaRecorderObserver.h" +#include "media/RecorderTrack.h" +#include "webrtc_java_api.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +namespace ffmpeg +{ + // Records the encoded frames of RTCRtpSenders and RTCRtpReceivers into a + // media file, as they are: nothing is decoded or encoded again, so a + // recording costs a copy of each frame and the writing, nothing more. + // + // Frames arrive through webrtc-java's encoded frame observers, on the + // WebRTC threads that carry media. There they are only checked and copied + // into a queue, and a thread of the recorder's own writes them out, so + // that a slow disk never holds up a call. + // + // The file header describes every stream, and a stream is only known once + // its first frames arrived (for video, its first key frame). So the + // header is written once every track has described itself, or once the + // tracks that did have waited long enough for the rest; frames that come + // before are held back until then. + class MediaRecorder + { + public: + MediaRecorder(const webrtc_java_api * api, std::string path); + + // Stops the recording if it still runs. + ~MediaRecorder(); + + MediaRecorder(const MediaRecorder &) = delete; + MediaRecorder & operator=(const MediaRecorder &) = delete; + + // Takes over the observer. Call before Start(). + void SetObserver(std::unique_ptr observer); + + // Opens the output file, choosing the container from its name. + // Returns 0 or a negative AVERROR. + int Open(); + + // Adds a sender or receiver to record, by the handle that + // NativeApi.encodedFramesOf() returned. The recorder takes over + // that handle's reference. Returns the track index. Only before + // Start(). + int AddTrack(void * frames); + + // Attaches to every track and starts writing. + void Start(); + + // Detaches from every track, writes out what is queued, finishes + // the file and waits for the writer thread. Returns whether any + // media was written. Stopping twice does nothing more. + bool Stop(); + + private: + // What the observer callback knows about a track. Guarded by + // mutex_, since WebRTC calls in on threads of its own. + struct Input + { + MediaRecorder * recorder = nullptr; + int index = 0; + void * frames = nullptr; + void * observer = nullptr; + + // Set by the first frame the track takes, which fixes the + // stream it records: one SSRC (one simulcast layer) and one + // codec. + bool locked = false; + uint32_t ssrc = 0; + std::string mime_type; + int64_t last_time_us = 0; + + // A video track takes nothing until it sees a key frame, at + // the start and after it had to drop frames. + bool waiting_for_key_frame = true; + bool key_frame_wanted = false; + int64_t last_key_frame_request_us = 0; + + bool reported_codec_change = false; + }; + + static void OnEncodedFrame(void * opaque, const wj_encoded_frame * frame); + + void Enqueue(Input * input, const wj_encoded_frame * frame); + void WantKeyFrameLocked(Input * input); + + // Makes a video track wait for its next key frame, and asks for + // one. + void RestartAtKeyFrame(int track); + + void Run(); + + void Handle(RecordedFrame frame); + void Write(RecordedFrame & frame); + + // Writes the header once the tracks are known, or once waiting + // for the rest is no longer worth it. Returns whether the header + // is written. + bool MaybeWriteHeader(bool finishing); + + void Finish(); + void Fail(const std::string & message, int error); + void Warn(const std::string & message); + + // Called with the lock held; the requests are sent once it is + // released. + void CollectKeyFrameRequestsLocked(std::vector * tracks, int64_t now_us); + + const webrtc_java_api * api_; + const std::string path_; + + std::unique_ptr observer_; + + // The writer thread's, once Start() ran. + AVFormatContext * context_ = nullptr; + std::vector> tracks_; + std::deque pending_; + size_t pending_bytes_ = 0; + int64_t first_ready_us_ = 0; + // Also read by Stop(), on the thread that stops. + std::atomic header_written_{ false }; + bool failed_ = false; + + // When recording started, in the clock of the frames. + int64_t start_us_ = 0; + + std::thread thread_; + + std::mutex mutex_; + std::condition_variable wake_; + std::vector> inputs_; + std::deque queue_; + size_t queued_bytes_ = 0; + // What the callbacks have to report, which the writer does. + std::vector pending_warnings_; + // Wakes the writer for something other than frames. + bool poke_ = false; + bool accepting_ = false; + bool stopping_ = false; + bool started_ = false; + bool stopped_ = false; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/MediaRecorderObserver.h b/webrtc-java-media/src/main/cpp/include/media/MediaRecorderObserver.h new file mode 100644 index 00000000..f0a774e6 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/MediaRecorderObserver.h @@ -0,0 +1,47 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_MEDIA_RECORDER_OBSERVER_H_ +#define WEBRTC_JAVA_MEDIA_MEDIA_RECORDER_OBSERVER_H_ + +#include + +namespace ffmpeg +{ + // What a MediaRecorder reports while it runs. Every call arrives on the + // recorder's writer thread, which must not be held up for long. + class MediaRecorderObserver + { + public: + virtual ~MediaRecorderObserver() = default; + + // The file header was written, and media now goes into the file. + virtual void OnStarted() = 0; + + // Something went wrong that the recording carries on without, + // such as a track that cannot be recorded. + virtual void OnWarning(const std::string & message) = 0; + + // Writing failed; nothing more goes into the file. + virtual void OnError(const std::string & message) = 0; + + // The given video track waits for a key frame, and the sender or + // receiver it records should be asked for one. + virtual void OnKeyFrameNeeded(int track) = 0; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/RecorderTrack.h b/webrtc-java-media/src/main/cpp/include/media/RecorderTrack.h new file mode 100644 index 00000000..9687d353 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/RecorderTrack.h @@ -0,0 +1,142 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_RECORDER_TRACK_H_ +#define WEBRTC_JAVA_MEDIA_RECORDER_TRACK_H_ + +#include +#include +#include +#include + +extern "C" { +#include +#include +} + +namespace ffmpeg +{ + struct PacketDeleter + { + void operator()(AVPacket * packet) const + { + av_packet_free(&packet); + } + }; + + using PacketPtr = std::unique_ptr; + + // One encoded frame on its way into the file: the payload, copied out of + // WebRTC, and what the recorder needs to know about it. + struct RecordedFrame + { + int track = 0; + PacketPtr packet; + std::string mime_type; + uint32_t ssrc = 0; + uint32_t rtp_timestamp = 0; + // When the frame passed the sender or receiver, in microseconds of + // the webrtc-java clock. + int64_t time_us = 0; + bool key_frame = false; + int width = 0; + int height = 0; + }; + + // One recorded sender or receiver, as a stream in the output file. + // + // Nothing about the stream is known in advance: the codec, the frame size + // and the parameter sets all come from the first frames. A track is + // therefore pending until a frame tells it enough to describe its stream, + // which for video takes a key frame. + // + // Used on the recorder's writer thread only. + class RecorderTrack + { + public: + enum class Status + { + // Still waiting for a frame that describes the stream. + kPending, + // Knows enough to describe its stream. + kReady, + // Cannot be recorded into this file at all. + kUnsupported + }; + + explicit RecorderTrack(int index); + ~RecorderTrack() = default; + + RecorderTrack(const RecorderTrack &) = delete; + RecorderTrack & operator=(const RecorderTrack &) = delete; + + int GetIndex() const; + Status GetStatus() const; + bool IsVideo() const; + + // Learns what it can about the stream from the frame. When this + // makes the track unsupported, the message says why. + Status Probe(const RecordedFrame & frame, const AVOutputFormat * format, + std::string * message); + + // Adds the stream of a ready track to the output. + int CreateStream(AVFormatContext * context); + + // The stream in the output, or null while there is none. + AVStream * GetStream() const; + + // Turns the frame into a packet of this track's stream, stamped + // with its presentation time. start_us is when recording started, + // in the clock of RecordedFrame::time_us. + AVPacket * Stamp(RecordedFrame & frame, int64_t start_us); + + private: + bool DescribeVideo(const RecordedFrame & frame); + bool DescribeAudio(const RecordedFrame & frame); + + // Takes the parameter sets out of a key frame. + bool ExtractExtradata(const AVPacket * packet); + + // Reads the frame size out of the bitstream of a key frame, for + // when WebRTC did not say. + bool ParseDimensions(const AVPacket * packet); + + const int index_; + Status status_ = Status::kPending; + + AVMediaType media_type_ = AVMEDIA_TYPE_UNKNOWN; + AVCodecID codec_id_ = AV_CODEC_ID_NONE; + int clock_rate_ = 0; + int width_ = 0; + int height_ = 0; + int channels_ = 0; + std::vector extradata_; + + AVStream * stream_ = nullptr; + + // Timing. RTP timestamps are exact within a stream, but start at a + // random value, so a track is anchored on the clock once and + // follows its RTP timestamps from there. + bool anchored_ = false; + uint32_t ssrc_ = 0; + uint32_t last_rtp_ = 0; + int64_t rtp_offset_ = 0; + int64_t base_pts_ = 0; + int64_t last_pts_ = -1; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/src/JNI_MediaPlayer.cpp b/webrtc-java-media/src/main/cpp/src/JNI_MediaPlayer.cpp index 3120128b..eb4030c0 100644 --- a/webrtc-java-media/src/main/cpp/src/JNI_MediaPlayer.cpp +++ b/webrtc-java-media/src/main/cpp/src/JNI_MediaPlayer.cpp @@ -15,6 +15,7 @@ */ #include "JNI_MediaPlayer.h" +#include "media/ApiCheck.h" #include "media/ErrorText.h" #include "media/JavaPlayerObserver.h" #include "media/MediaPlayer.h" @@ -64,24 +65,10 @@ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_create const webrtc_java_api * api = reinterpret_cast(tableAddress); - if (api == nullptr) { - ThrowIOException(env, "The webrtc-java function table is not available"); + std::string apiError = ffmpeg::CheckApi(api); - return 0; - } - if (api->version != WEBRTC_JAVA_API_VERSION) { - ThrowIOException(env, "This module was built against webrtc-java interface version " - + std::to_string(WEBRTC_JAVA_API_VERSION) + ", but the loaded library provides " - + std::to_string(api->version)); - - return 0; - } - if (api->size < sizeof(webrtc_java_api)) { - // The same version, but from before the members this module relies - // on were appended. - ThrowIOException(env, "The loaded webrtc-java library provides " - + std::to_string(api->size) + " bytes of its interface, but this module needs " - + std::to_string(sizeof(webrtc_java_api))); + if (!apiError.empty()) { + ThrowIOException(env, apiError); return 0; } diff --git a/webrtc-java-media/src/main/cpp/src/JNI_MediaRecorder.cpp b/webrtc-java-media/src/main/cpp/src/JNI_MediaRecorder.cpp new file mode 100644 index 00000000..a6983ca5 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/JNI_MediaRecorder.cpp @@ -0,0 +1,118 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_MediaRecorder.h" +#include "media/ApiCheck.h" +#include "media/ErrorText.h" +#include "media/JavaRecorderObserver.h" +#include "media/MediaRecorder.h" +#include "webrtc_java_api.h" + +#include +#include + +namespace +{ + void ThrowIOException(JNIEnv * env, const std::string & message) + { + jclass cls = env->FindClass("java/io/IOException"); + + if (cls != nullptr) { + env->ThrowNew(cls, message.c_str()); + env->DeleteLocalRef(cls); + } + } + + ffmpeg::MediaRecorder * RecorderOf(jlong handle) + { + return reinterpret_cast(handle); + } +} + +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_create +(JNIEnv * env, jobject caller, jstring jPath, jlong tableAddress) +{ + const webrtc_java_api * api = reinterpret_cast(tableAddress); + std::string apiError = ffmpeg::CheckApi(api); + + if (!apiError.empty()) { + ThrowIOException(env, apiError); + + return 0; + } + + const char * chars = env->GetStringUTFChars(jPath, nullptr); + + if (chars == nullptr) { + return 0; + } + + std::string path(chars); + + env->ReleaseStringUTFChars(jPath, chars); + + auto recorder = std::make_unique(api, path); + + int result = recorder->Open(); + + if (result < 0) { + recorder.reset(); + + ThrowIOException(env, "Opening " + path + " for recording failed: " + ffmpeg::ErrorText(result)); + + return 0; + } + + recorder->SetObserver(std::make_unique(env, caller)); + + return reinterpret_cast(recorder.release()); +} + +JNIEXPORT jint JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_addTrack +(JNIEnv * env, jclass caller, jlong handle, jlong frames) +{ + if (handle == 0 || frames == 0) { + return -1; + } + + return RecorderOf(handle)->AddTrack(reinterpret_cast(frames)); +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_start +(JNIEnv * env, jclass caller, jlong handle) +{ + if (handle != 0) { + RecorderOf(handle)->Start(); + } +} + +JNIEXPORT jboolean JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_stop +(JNIEnv * env, jclass caller, jlong handle) +{ + if (handle == 0) { + return JNI_FALSE; + } + + return RecorderOf(handle)->Stop() ? JNI_TRUE : JNI_FALSE; +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_recorder_MediaRecorder_dispose +(JNIEnv * env, jclass caller, jlong handle) +{ + if (handle != 0) { + delete RecorderOf(handle); + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/ApiCheck.cpp b/webrtc-java-media/src/main/cpp/src/media/ApiCheck.cpp new file mode 100644 index 00000000..6de1add2 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/ApiCheck.cpp @@ -0,0 +1,41 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/ApiCheck.h" + +namespace ffmpeg +{ + std::string CheckApi(const webrtc_java_api * api) + { + if (api == nullptr) { + return "The webrtc-java function table is not available"; + } + if (api->version != WEBRTC_JAVA_API_VERSION) { + return "This module was built against webrtc-java interface version " + + std::to_string(WEBRTC_JAVA_API_VERSION) + ", but the loaded library provides " + + std::to_string(api->version); + } + if (api->size < sizeof(webrtc_java_api)) { + // The same version, but from before the members this module relies + // on were appended. + return "The loaded webrtc-java library provides " + + std::to_string(api->size) + " bytes of its interface, but this module needs " + + std::to_string(sizeof(webrtc_java_api)); + } + + return std::string(); + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/JavaRecorderObserver.cpp b/webrtc-java-media/src/main/cpp/src/media/JavaRecorderObserver.cpp new file mode 100644 index 00000000..c7addf39 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/JavaRecorderObserver.cpp @@ -0,0 +1,182 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/JavaRecorderObserver.h" + +namespace ffmpeg +{ + JavaRecorderObserver::JavaRecorderObserver(JNIEnv * env, jobject recorder) + { + if (env->GetJavaVM(&vm_) != JNI_OK) { + return; + } + + recorder_ = env->NewGlobalRef(recorder); + + if (recorder_ == nullptr) { + return; + } + + jclass cls = env->GetObjectClass(recorder_); + + if (cls == nullptr) { + return; + } + + on_started_ = env->GetMethodID(cls, "onNativeStarted", "()V"); + on_warning_ = env->GetMethodID(cls, "onNativeWarning", "(Ljava/lang/String;)V"); + on_error_ = env->GetMethodID(cls, "onNativeError", "(Ljava/lang/String;)V"); + on_key_frame_needed_ = env->GetMethodID(cls, "onNativeKeyFrameNeeded", "(I)V"); + + env->DeleteLocalRef(cls); + + ClearPendingException(env); + } + + JavaRecorderObserver::~JavaRecorderObserver() + { + if (recorder_ == nullptr || vm_ == nullptr) { + return; + } + + bool attached = false; + JNIEnv * env = Attach(&attached); + + if (env != nullptr) { + env->DeleteGlobalRef(recorder_); + } + + recorder_ = nullptr; + + Detach(attached); + } + + void JavaRecorderObserver::OnStarted() + { + if (on_started_ == nullptr) { + return; + } + + bool attached = false; + JNIEnv * env = Attach(&attached); + + if (CanCallJava(env)) { + env->CallVoidMethod(recorder_, on_started_); + + ClearPendingException(env); + } + + Detach(attached); + } + + void JavaRecorderObserver::OnWarning(const std::string & message) + { + CallWithMessage(on_warning_, message); + } + + void JavaRecorderObserver::OnError(const std::string & message) + { + CallWithMessage(on_error_, message); + } + + void JavaRecorderObserver::OnKeyFrameNeeded(int track) + { + if (on_key_frame_needed_ == nullptr) { + return; + } + + bool attached = false; + JNIEnv * env = Attach(&attached); + + if (CanCallJava(env)) { + env->CallVoidMethod(recorder_, on_key_frame_needed_, static_cast(track)); + + ClearPendingException(env); + } + + Detach(attached); + } + + void JavaRecorderObserver::CallWithMessage(jmethodID method, const std::string & message) + { + if (method == nullptr) { + return; + } + + bool attached = false; + JNIEnv * env = Attach(&attached); + + if (CanCallJava(env)) { + jstring text = env->NewStringUTF(message.c_str()); + + if (text != nullptr) { + env->CallVoidMethod(recorder_, method, text); + env->DeleteLocalRef(text); + } + + ClearPendingException(env); + } + + Detach(attached); + } + + JNIEnv * JavaRecorderObserver::Attach(bool * attached) + { + *attached = false; + + if (vm_ == nullptr) { + return nullptr; + } + + JNIEnv * env = nullptr; + jint result = vm_->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6); + + if (result == JNI_OK) { + return env; + } + if (result != JNI_EDETACHED) { + return nullptr; + } + + if (vm_->AttachCurrentThreadAsDaemon(reinterpret_cast(&env), nullptr) != JNI_OK) { + return nullptr; + } + + *attached = true; + + return env; + } + + void JavaRecorderObserver::Detach(bool attached) + { + if (attached && vm_ != nullptr) { + vm_->DetachCurrentThread(); + } + } + + bool JavaRecorderObserver::CanCallJava(JNIEnv * env) + { + return env != nullptr && env->ExceptionCheck() == JNI_FALSE; + } + + void JavaRecorderObserver::ClearPendingException(JNIEnv * env) + { + if (env->ExceptionCheck() == JNI_TRUE) { + env->ExceptionDescribe(); + env->ExceptionClear(); + } + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/MediaRecorder.cpp b/webrtc-java-media/src/main/cpp/src/media/MediaRecorder.cpp new file mode 100644 index 00000000..b8b06318 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/MediaRecorder.cpp @@ -0,0 +1,591 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/MediaRecorder.h" +#include "media/ErrorText.h" + +#include +#include +#include + +extern "C" { +#include +} + +namespace ffmpeg +{ + namespace + { + // How much may wait for the writer, and how much may wait for the + // file header, before frames are dropped: far more than a healthy + // recording ever holds, far less than memory lasts. + constexpr size_t kMaxQueuedBytes = 64 * 1024 * 1024; + constexpr size_t kMaxPendingBytes = 64 * 1024 * 1024; + + // How long the tracks that are ready wait for the others before the + // file starts without them. + constexpr int64_t kTrackWaitUs = 3000000; + + // How often a track that waits for a key frame asks for one. + constexpr int64_t kKeyFrameRequestIntervalUs = 1000000; + + // How long the recorded SSRC of a track has to be silent before a + // different one takes its place. + constexpr int64_t kSsrcSwitchUs = 1000000; + + // How often the writer looks at the clock when nothing arrives, for + // the header timeout and for key frame requests it held back. + constexpr std::chrono::milliseconds kWriterTick(250); + + bool IsMp4(const AVOutputFormat * format) + { + return std::strcmp(format->name, "mp4") == 0 || std::strcmp(format->name, "mov") == 0; + } + } + + MediaRecorder::MediaRecorder(const webrtc_java_api * api, std::string path) : + api_(api), + path_(std::move(path)) + { + } + + MediaRecorder::~MediaRecorder() + { + Stop(); + + if (context_ != nullptr) { + if (context_->pb != nullptr && !(context_->oformat->flags & AVFMT_NOFILE)) { + avio_closep(&context_->pb); + } + + avformat_free_context(context_); + } + } + + void MediaRecorder::SetObserver(std::unique_ptr observer) + { + observer_ = std::move(observer); + } + + int MediaRecorder::Open() + { + // The container follows the file name; a name FFmpeg cannot place + // gets Matroska, which holds every codec WebRTC sends. + int result = avformat_alloc_output_context2(&context_, nullptr, nullptr, path_.c_str()); + + if (result < 0 || context_ == nullptr) { + result = avformat_alloc_output_context2(&context_, nullptr, "matroska", path_.c_str()); + } + if (result < 0) { + return result; + } + + // Frames arrive in real time, so the streams never drift far apart, + // and holding back more for interleaving only costs memory. + context_->max_interleave_delta = 2000000; + + if (!(context_->oformat->flags & AVFMT_NOFILE)) { + result = avio_open(&context_->pb, path_.c_str(), AVIO_FLAG_WRITE); + + if (result < 0) { + return result; + } + } + + return 0; + } + + int MediaRecorder::AddTrack(void * frames) + { + std::lock_guard lock(mutex_); + + if (started_ || stopped_) { + api_->encoded_frames_release(frames); + + return -1; + } + + auto input = std::make_unique(); + input->recorder = this; + input->index = static_cast(inputs_.size()); + input->frames = frames; + + tracks_.push_back(std::make_unique(input->index)); + inputs_.push_back(std::move(input)); + + return static_cast(inputs_.size()) - 1; + } + + void MediaRecorder::Start() + { + { + std::lock_guard lock(mutex_); + + if (started_ || stopped_) { + return; + } + + started_ = true; + accepting_ = true; + start_us_ = api_->now_us(); + } + + thread_ = std::thread(&MediaRecorder::Run, this); + + // The inputs do not change once started, so the callbacks may hold on + // to them without the lock. + for (const auto & input : inputs_) { + input->observer = api_->encoded_observer_add(input->frames, &MediaRecorder::OnEncodedFrame, + input.get()); + + // The observer keeps what it observes alive on its own. + api_->encoded_frames_release(input->frames); + input->frames = nullptr; + } + } + + bool MediaRecorder::Stop() + { + { + std::lock_guard lock(mutex_); + + if (stopped_) { + return header_written_; + } + + stopped_ = true; + } + + // No callback runs once its observer is removed, so after this nothing + // comes in any more. + for (const auto & input : inputs_) { + if (input->observer != nullptr) { + api_->encoded_observer_remove(input->observer); + input->observer = nullptr; + } + if (input->frames != nullptr) { + api_->encoded_frames_release(input->frames); + input->frames = nullptr; + } + } + + { + std::lock_guard lock(mutex_); + + accepting_ = false; + stopping_ = true; + } + + wake_.notify_all(); + + if (thread_.joinable()) { + thread_.join(); + } + + // The writer closes the file when it finishes. A recorder that never + // started has no writer, and its file must not stay open either: the + // caller deletes an empty file, which Windows refuses while it is. + if (context_ != nullptr && context_->pb != nullptr && !(context_->oformat->flags & AVFMT_NOFILE)) { + avio_closep(&context_->pb); + } + + return header_written_; + } + + void MediaRecorder::OnEncodedFrame(void * opaque, const wj_encoded_frame * frame) + { + Input * input = static_cast(opaque); + + input->recorder->Enqueue(input, frame); + } + + void MediaRecorder::Enqueue(Input * input, const wj_encoded_frame * frame) + { + if (frame->data == nullptr || frame->size == 0 || frame->size > INT32_MAX) { + return; + } + + const bool video = frame->media_type == WEBRTC_JAVA_MEDIA_VIDEO; + + std::lock_guard lock(mutex_); + + if (!accepting_) { + return; + } + + if (!input->locked) { + // A video track starts with a key frame; a decoder could make + // nothing of what comes before. + if (video && !frame->key_frame) { + WantKeyFrameLocked(input); + return; + } + + input->locked = true; + input->ssrc = frame->ssrc; + input->mime_type = frame->mime_type != nullptr ? frame->mime_type : ""; + } + else if (frame->ssrc != input->ssrc) { + // Another simulcast layer, which is not recorded; or the stream + // was restarted under a new SSRC, which takes over once the old + // one has gone quiet. + if (frame->time_us - input->last_time_us < kSsrcSwitchUs) { + return; + } + if (video && !frame->key_frame) { + WantKeyFrameLocked(input); + return; + } + + input->ssrc = frame->ssrc; + } + + if (frame->mime_type == nullptr || input->mime_type != frame->mime_type) { + // A stream in a file cannot change its codec halfway. + if (!input->reported_codec_change) { + input->reported_codec_change = true; + + pending_warnings_.push_back("Track " + std::to_string(input->index) + + " changed its codec and is no longer recorded"); + poke_ = true; + wake_.notify_one(); + } + + return; + } + + if (video && input->waiting_for_key_frame) { + if (!frame->key_frame) { + WantKeyFrameLocked(input); + return; + } + + input->waiting_for_key_frame = false; + } + + if (queued_bytes_ + frame->size > kMaxQueuedBytes) { + // The writer does not keep up. Video goes on from the next key + // frame, since everything up to it could not be decoded anyway. + if (video) { + input->waiting_for_key_frame = true; + WantKeyFrameLocked(input); + } + + return; + } + + RecordedFrame recorded; + recorded.packet = PacketPtr(av_packet_alloc()); + + if (recorded.packet == nullptr + || av_new_packet(recorded.packet.get(), static_cast(frame->size)) < 0) { + return; + } + + std::memcpy(recorded.packet->data, frame->data, frame->size); + + recorded.track = input->index; + recorded.mime_type = input->mime_type; + recorded.ssrc = frame->ssrc; + recorded.rtp_timestamp = frame->rtp_timestamp; + recorded.time_us = frame->time_us; + recorded.key_frame = frame->key_frame != 0; + recorded.width = frame->width; + recorded.height = frame->height; + + input->last_time_us = frame->time_us; + + queued_bytes_ += frame->size; + queue_.push_back(std::move(recorded)); + + wake_.notify_one(); + } + + void MediaRecorder::WantKeyFrameLocked(Input * input) + { + if (!input->key_frame_wanted) { + input->key_frame_wanted = true; + poke_ = true; + + wake_.notify_one(); + } + } + + void MediaRecorder::RestartAtKeyFrame(int track) + { + std::lock_guard lock(mutex_); + + Input * input = inputs_[track].get(); + input->waiting_for_key_frame = true; + + WantKeyFrameLocked(input); + } + + void MediaRecorder::CollectKeyFrameRequestsLocked(std::vector * tracks, int64_t now_us) + { + for (const auto & input : inputs_) { + if (input->key_frame_wanted + && now_us - input->last_key_frame_request_us >= kKeyFrameRequestIntervalUs) { + input->key_frame_wanted = false; + input->last_key_frame_request_us = now_us; + + tracks->push_back(input->index); + } + } + } + + void MediaRecorder::Run() + { + std::deque batch; + std::vector warnings; + std::vector key_frame_requests; + + std::unique_lock lock(mutex_); + + while (true) { + wake_.wait_for(lock, kWriterTick, [this] { + return !queue_.empty() || stopping_ || poke_; + }); + + poke_ = false; + + batch.swap(queue_); + queued_bytes_ = 0; + warnings.swap(pending_warnings_); + + CollectKeyFrameRequestsLocked(&key_frame_requests, api_->now_us()); + + const bool stopping = stopping_; + + lock.unlock(); + + for (const std::string & warning : warnings) { + Warn(warning); + } + for (int track : key_frame_requests) { + if (observer_ != nullptr) { + observer_->OnKeyFrameNeeded(track); + } + } + + warnings.clear(); + key_frame_requests.clear(); + + while (!batch.empty()) { + RecordedFrame frame = std::move(batch.front()); + batch.pop_front(); + + Handle(std::move(frame)); + } + + // Nothing may arrive for a track that is waited for, so the + // clock is looked at here as well. + MaybeWriteHeader(false); + + lock.lock(); + + if (stopping && queue_.empty()) { + break; + } + } + + lock.unlock(); + + Finish(); + } + + void MediaRecorder::Handle(RecordedFrame frame) + { + if (failed_) { + return; + } + + RecorderTrack & track = *tracks_[frame.track]; + + if (track.GetStatus() == RecorderTrack::Status::kPending) { + std::string message; + RecorderTrack::Status status = track.Probe(frame, context_->oformat, &message); + + if (status == RecorderTrack::Status::kUnsupported) { + Warn(message); + return; + } + if (status == RecorderTrack::Status::kPending) { + // A key frame that did not describe the stream, e.g. one + // without parameter sets; the next one may. + if (track.IsVideo()) { + RestartAtKeyFrame(frame.track); + } + return; + } + if (first_ready_us_ == 0) { + first_ready_us_ = api_->now_us(); + } + } + + if (track.GetStatus() != RecorderTrack::Status::kReady) { + return; + } + + if (!header_written_) { + pending_bytes_ += static_cast(frame.packet->size); + pending_.push_back(std::move(frame)); + + MaybeWriteHeader(false); + return; + } + + // A track that got ready only after the header has no stream. + if (track.GetStream() != nullptr) { + Write(frame); + } + } + + void MediaRecorder::Write(RecordedFrame & frame) + { + AVPacket * packet = tracks_[frame.track]->Stamp(frame, start_us_); + + // Takes the packet's reference and leaves the packet blank. + int result = av_interleaved_write_frame(context_, packet); + + if (result < 0) { + Fail("Writing to " + path_ + " failed", result); + } + } + + bool MediaRecorder::MaybeWriteHeader(bool finishing) + { + if (header_written_) { + return true; + } + if (failed_ || first_ready_us_ == 0) { + return false; + } + + bool all_known = true; + + for (const auto & track : tracks_) { + if (track->GetStatus() == RecorderTrack::Status::kPending) { + all_known = false; + } + } + + if (!all_known && !finishing + && api_->now_us() - first_ready_us_ < kTrackWaitUs + && pending_bytes_ < kMaxPendingBytes) { + return false; + } + + for (const auto & track : tracks_) { + if (track->GetStatus() == RecorderTrack::Status::kReady) { + int result = track->CreateStream(context_); + + if (result < 0) { + Fail("Adding track " + std::to_string(track->GetIndex()) + " to the file failed", result); + pending_.clear(); + return false; + } + } + else if (track->GetStatus() == RecorderTrack::Status::kPending) { + Warn("Track " + std::to_string(track->GetIndex()) + + " had no media to record when the recording began, and is left out"); + } + } + + AVDictionary * options = nullptr; + + if (IsMp4(context_->oformat)) { + // Fragmented, so that a recording cut short, by a crash or a full + // disk, still plays up to where it stopped; and in fragments of a + // second rather than per key frame, which WebRTC sends rarely and + // which would hold whole minutes in memory. + av_dict_set(&options, "movflags", "+empty_moov+default_base_moof", 0); + av_dict_set(&options, "frag_duration", "1000000", 0); + } + + int result = avformat_write_header(context_, &options); + + av_dict_free(&options); + + if (result < 0) { + Fail("Writing the header of " + path_ + " failed", result); + pending_.clear(); + return false; + } + + header_written_ = true; + + if (observer_ != nullptr) { + observer_->OnStarted(); + } + + while (!pending_.empty() && !failed_) { + RecordedFrame frame = std::move(pending_.front()); + pending_.pop_front(); + + Write(frame); + } + + pending_.clear(); + pending_bytes_ = 0; + + return true; + } + + void MediaRecorder::Finish() + { + MaybeWriteHeader(true); + + if (header_written_) { + // Even after a failed write: whatever made it into the file + // is only playable with a trailer. + int result = av_write_trailer(context_); + + if (result < 0 && !failed_) { + Fail("Finishing " + path_ + " failed", result); + } + } + + pending_.clear(); + + if (context_->pb != nullptr && !(context_->oformat->flags & AVFMT_NOFILE)) { + avio_closep(&context_->pb); + } + } + + void MediaRecorder::Fail(const std::string & message, int error) + { + failed_ = true; + + { + std::lock_guard lock(mutex_); + + // Nothing more goes into the file, so nothing needs copying. + accepting_ = false; + } + + if (observer_ != nullptr) { + observer_->OnError(message + ": " + ErrorText(error)); + } + } + + void MediaRecorder::Warn(const std::string & message) + { + if (observer_ != nullptr) { + observer_->OnWarning(message); + } + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/RecorderTrack.cpp b/webrtc-java-media/src/main/cpp/src/media/RecorderTrack.cpp new file mode 100644 index 00000000..bbb3a4a7 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/RecorderTrack.cpp @@ -0,0 +1,525 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/RecorderTrack.h" + +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +} + +namespace ffmpeg +{ + namespace + { + // A jump this large between where the RTP timestamps and the clock + // put a frame means the sender restarted its RTP clock. + constexpr int64_t kMaxDriftSeconds = 10; + + struct CodecInfo + { + const char * mime_type; + AVMediaType media_type; + AVCodecID codec_id; + int clock_rate; + }; + + // The codecs WebRTC sends that a container can hold as they are. + constexpr CodecInfo kCodecs[] = { + { "video/vp8", AVMEDIA_TYPE_VIDEO, AV_CODEC_ID_VP8, 90000 }, + { "video/vp9", AVMEDIA_TYPE_VIDEO, AV_CODEC_ID_VP9, 90000 }, + { "video/av1", AVMEDIA_TYPE_VIDEO, AV_CODEC_ID_AV1, 90000 }, + { "video/h264", AVMEDIA_TYPE_VIDEO, AV_CODEC_ID_H264, 90000 }, + { "video/h265", AVMEDIA_TYPE_VIDEO, AV_CODEC_ID_HEVC, 90000 }, + { "audio/opus", AVMEDIA_TYPE_AUDIO, AV_CODEC_ID_OPUS, 48000 }, + { "audio/pcmu", AVMEDIA_TYPE_AUDIO, AV_CODEC_ID_PCM_MULAW, 8000 }, + { "audio/pcma", AVMEDIA_TYPE_AUDIO, AV_CODEC_ID_PCM_ALAW, 8000 }, + }; + + const CodecInfo * FindCodec(const std::string & mime_type) + { + std::string lower(mime_type); + + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + + for (const CodecInfo & codec : kCodecs) { + if (lower == codec.mime_type) { + return &codec; + } + } + + return nullptr; + } + + // Whether the container holds the codec. The codec tags say most of + // it, but the MP4 family refuses some codecs only when the header is + // written, which would fail the whole file rather than leave out one + // track. + bool CanHold(const AVOutputFormat * format, AVCodecID codec_id) + { + const bool mp4 = std::strcmp(format->name, "mp4") == 0; + const bool mov = std::strcmp(format->name, "mov") == 0; + + if ((mp4 || mov) && codec_id == AV_CODEC_ID_VP8) { + return false; + } + if (mov && (codec_id == AV_CODEC_ID_VP9 || codec_id == AV_CODEC_ID_AV1)) { + return false; + } + + return avformat_query_codec(format, codec_id, FF_COMPLIANCE_NORMAL) == 1; + } + + bool NeedsExtradata(AVCodecID codec_id) + { + return codec_id == AV_CODEC_ID_H264 || codec_id == AV_CODEC_ID_HEVC + || codec_id == AV_CODEC_ID_AV1; + } + + // Reads bits most significant first, as the VP9 headers are written. + class BitReader + { + public: + BitReader(const uint8_t * data, size_t size) : + data_(data), size_(size) + { + } + + bool Read(int count, uint32_t * value) + { + uint32_t result = 0; + + for (int i = 0; i < count; i++) { + if (position_ >= size_ * 8) { + return false; + } + + int bit = (data_[position_ / 8] >> (7 - position_ % 8)) & 1; + result = (result << 1) | static_cast(bit); + position_++; + } + + *value = result; + + return true; + } + + private: + const uint8_t * data_; + size_t size_; + size_t position_ = 0; + }; + + // The frame size of a VP8 key frame, from its frame header (RFC 6386, + // section 9.1). + bool ParseVp8Dimensions(const uint8_t * data, size_t size, int * width, int * height) + { + if (size < 10 || (data[0] & 0x01) != 0) { + return false; + } + if (data[3] != 0x9D || data[4] != 0x01 || data[5] != 0x2A) { + return false; + } + + *width = AV_RL16(data + 6) & 0x3FFF; + *height = AV_RL16(data + 8) & 0x3FFF; + + return *width > 0 && *height > 0; + } + + // The frame size of a VP9 key frame, from its uncompressed header + // (VP9 bitstream specification, section 6.2). + bool ParseVp9Dimensions(const uint8_t * data, size_t size, int * width, int * height) + { + BitReader reader(data, size); + uint32_t value = 0; + uint32_t profile_low = 0; + uint32_t profile_high = 0; + + if (!reader.Read(2, &value) || value != 2) { + return false; + } + if (!reader.Read(1, &profile_low) || !reader.Read(1, &profile_high)) { + return false; + } + + uint32_t profile = (profile_high << 1) | profile_low; + + if (profile == 3 && !reader.Read(1, &value)) { + return false; + } + + // show_existing_frame, then frame_type, which is 0 for a key frame. + if (!reader.Read(1, &value) || value != 0) { + return false; + } + if (!reader.Read(1, &value) || value != 0) { + return false; + } + + // show_frame, error_resilient_mode, then the sync code. + if (!reader.Read(2, &value) || !reader.Read(24, &value) || value != 0x498342) { + return false; + } + + // color_config() + if (profile >= 2 && !reader.Read(1, &value)) { + return false; + } + + uint32_t color_space = 0; + + if (!reader.Read(3, &color_space)) { + return false; + } + + if (color_space != 7) { + // color_range, and for profiles 1 and 3 the subsampling and a + // reserved bit. + if (!reader.Read(1, &value)) { + return false; + } + if ((profile == 1 || profile == 3) && !reader.Read(3, &value)) { + return false; + } + } + else if ((profile == 1 || profile == 3) && !reader.Read(1, &value)) { + return false; + } + + uint32_t width_minus_one = 0; + uint32_t height_minus_one = 0; + + if (!reader.Read(16, &width_minus_one) || !reader.Read(16, &height_minus_one)) { + return false; + } + + *width = static_cast(width_minus_one) + 1; + *height = static_cast(height_minus_one) + 1; + + return true; + } + } + + RecorderTrack::RecorderTrack(int index) : + index_(index) + { + } + + int RecorderTrack::GetIndex() const + { + return index_; + } + + RecorderTrack::Status RecorderTrack::GetStatus() const + { + return status_; + } + + bool RecorderTrack::IsVideo() const + { + return media_type_ == AVMEDIA_TYPE_VIDEO; + } + + RecorderTrack::Status RecorderTrack::Probe(const RecordedFrame & frame, + const AVOutputFormat * format, std::string * message) + { + if (status_ != Status::kPending) { + return status_; + } + + if (codec_id_ == AV_CODEC_ID_NONE) { + const CodecInfo * codec = FindCodec(frame.mime_type); + + if (codec == nullptr) { + *message = "Track " + std::to_string(index_) + " is " + frame.mime_type + + ", which cannot be recorded"; + status_ = Status::kUnsupported; + + return status_; + } + if (!CanHold(format, codec->codec_id)) { + *message = "Track " + std::to_string(index_) + " is " + frame.mime_type + + ", which a " + format->name + " file cannot hold; Matroska (.mkv) holds every codec WebRTC sends"; + status_ = Status::kUnsupported; + + return status_; + } + + media_type_ = codec->media_type; + codec_id_ = codec->codec_id; + clock_rate_ = codec->clock_rate; + } + + bool described = media_type_ == AVMEDIA_TYPE_VIDEO + ? DescribeVideo(frame) + : DescribeAudio(frame); + + if (described) { + status_ = Status::kReady; + } + + return status_; + } + + int RecorderTrack::CreateStream(AVFormatContext * context) + { + AVStream * stream = avformat_new_stream(context, nullptr); + + if (stream == nullptr) { + return AVERROR(ENOMEM); + } + + AVCodecParameters * par = stream->codecpar; + par->codec_type = media_type_; + par->codec_id = codec_id_; + + if (media_type_ == AVMEDIA_TYPE_VIDEO) { + par->width = width_; + par->height = height_; + } + else { + av_channel_layout_default(&par->ch_layout, channels_); + par->sample_rate = clock_rate_; + } + + if (!extradata_.empty()) { + par->extradata = static_cast( + av_mallocz(extradata_.size() + AV_INPUT_BUFFER_PADDING_SIZE)); + + if (par->extradata == nullptr) { + return AVERROR(ENOMEM); + } + + std::memcpy(par->extradata, extradata_.data(), extradata_.size()); + par->extradata_size = static_cast(extradata_.size()); + } + + // A hint; the muxer picks the time base it stores, which Stamp() + // converts to. + stream->time_base = AVRational{ 1, clock_rate_ }; + stream_ = stream; + + return 0; + } + + AVStream * RecorderTrack::GetStream() const + { + return stream_; + } + + AVPacket * RecorderTrack::Stamp(RecordedFrame & frame, int64_t start_us) + { + AVPacket * packet = frame.packet.get(); + const AVRational track_time_base{ 1, clock_rate_ }; + + // Where the clock puts the frame, which is what a track starts from, + // and what it falls back on if its RTP timestamps stop making sense. + int64_t clock_pts = av_rescale(frame.time_us - start_us, clock_rate_, 1000000); + + if (!anchored_ || frame.ssrc != ssrc_) { + // A new SSRC is a new RTP clock, e.g. after the remote side + // restarted its sender. + anchored_ = true; + ssrc_ = frame.ssrc; + last_rtp_ = frame.rtp_timestamp; + rtp_offset_ = 0; + base_pts_ = clock_pts; + } + else { + // The signed difference unwraps the 32-bit timestamp. + rtp_offset_ += static_cast(frame.rtp_timestamp - last_rtp_); + last_rtp_ = frame.rtp_timestamp; + } + + int64_t pts = base_pts_ + rtp_offset_; + + if (std::llabs(pts - clock_pts) > kMaxDriftSeconds * clock_rate_) { + base_pts_ = clock_pts - rtp_offset_; + pts = clock_pts; + } + + // Every container wants increasing timestamps, and some want them + // strictly increasing even where frames share an RTP timestamp. + int64_t stream_pts = av_rescale_q(std::max(pts, 0), track_time_base, stream_->time_base); + + if (stream_pts <= last_pts_) { + stream_pts = last_pts_ + 1; + } + + last_pts_ = stream_pts; + + packet->pts = stream_pts; + packet->dts = stream_pts; + packet->stream_index = stream_->index; + packet->time_base = stream_->time_base; + + if (frame.key_frame || media_type_ == AVMEDIA_TYPE_AUDIO) { + packet->flags |= AV_PKT_FLAG_KEY; + } + + return packet; + } + + bool RecorderTrack::DescribeVideo(const RecordedFrame & frame) + { + // Only a key frame describes the stream, and the file has to start + // with one anyway. + if (!frame.key_frame) { + return false; + } + + const AVPacket * packet = frame.packet.get(); + + if (NeedsExtradata(codec_id_) && !ExtractExtradata(packet)) { + return false; + } + + width_ = frame.width; + height_ = frame.height; + + if (width_ <= 0 || height_ <= 0) { + return ParseDimensions(packet); + } + + return true; + } + + bool RecorderTrack::DescribeAudio(const RecordedFrame & frame) + { + const AVPacket * packet = frame.packet.get(); + + if (codec_id_ != AV_CODEC_ID_OPUS) { + // G.711 is always mono in WebRTC. + channels_ = 1; + + return true; + } + + if (packet->size < 1) { + return false; + } + + // The stereo flag of the TOC byte (RFC 6716, section 3.1). A decoder + // outputs whatever channel count the header asks for, whatever a + // packet holds, so the first packet is as good as any. + channels_ = (packet->data[0] & 0x04) != 0 ? 2 : 1; + + // The OpusHead that Ogg, Matroska and MP4 all want as the codec's + // private data (RFC 7845, section 5.1): no pre-skip, since WebRTC + // packets are cut at arbitrary points of a running stream anyway. + extradata_.assign(19, 0); + std::memcpy(extradata_.data(), "OpusHead", 8); + extradata_[8] = 1; + extradata_[9] = static_cast(channels_); + AV_WL32(extradata_.data() + 12, 48000); + + return true; + } + + bool RecorderTrack::ExtractExtradata(const AVPacket * packet) + { + const AVBitStreamFilter * filter = av_bsf_get_by_name("extract_extradata"); + AVBSFContext * bsf = nullptr; + + if (filter == nullptr || av_bsf_alloc(filter, &bsf) < 0) { + return false; + } + + bsf->par_in->codec_type = AVMEDIA_TYPE_VIDEO; + bsf->par_in->codec_id = codec_id_; + + bool found = false; + + if (av_bsf_init(bsf) >= 0) { + // The filter takes the packet over, so it gets a reference of its + // own rather than the frame's. + AVPacket * copy = av_packet_clone(packet); + + if (copy != nullptr && av_bsf_send_packet(bsf, copy) >= 0) { + while (av_bsf_receive_packet(bsf, copy) >= 0) { + size_t size = 0; + const uint8_t * data = av_packet_get_side_data(copy, + AV_PKT_DATA_NEW_EXTRADATA, &size); + + if (data != nullptr && size > 0) { + extradata_.assign(data, data + size); + found = true; + } + + av_packet_unref(copy); + } + } + + av_packet_free(©); + } + + av_bsf_free(&bsf); + + return found; + } + + bool RecorderTrack::ParseDimensions(const AVPacket * packet) + { + const size_t size = static_cast(packet->size); + + if (codec_id_ == AV_CODEC_ID_VP8) { + return ParseVp8Dimensions(packet->data, size, &width_, &height_); + } + if (codec_id_ == AV_CODEC_ID_VP9) { + return ParseVp9Dimensions(packet->data, size, &width_, &height_); + } + + AVCodecParserContext * parser = av_parser_init(codec_id_); + AVCodecContext * context = avcodec_alloc_context3(nullptr); + + if (parser != nullptr && context != nullptr) { + uint8_t * output = nullptr; + int output_size = 0; + + av_parser_parse2(parser, context, &output, &output_size, packet->data, packet->size, + AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0); + + if (parser->width <= 0) { + // A parser that waits for the start of the next frame before + // it looks at this one gets it by being flushed. + av_parser_parse2(parser, context, &output, &output_size, nullptr, 0, + AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0); + } + + width_ = parser->width > 0 ? parser->width : context->width; + height_ = parser->height > 0 ? parser->height : context->height; + } + + if (parser != nullptr) { + av_parser_close(parser); + } + + avcodec_free_context(&context); + + return width_ > 0 && height_ > 0; + } +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorder.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorder.java new file mode 100644 index 00000000..803d9a2f --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorder.java @@ -0,0 +1,379 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.recorder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; + +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.RTCRtpSender; +import dev.onvoid.webrtc.internal.NativeApi; +import dev.onvoid.webrtc.media.player.FFmpeg; + +/** + * Records what a peer connection sends or receives into a media file, without + * decoding or re-encoding it: the encoded frames of each sender or receiver + * go into the file as they are. A recording therefore costs next to no CPU, + * keeps the exact quality that went over the network, and needs no codec + * beyond the container. + *

+ * The container follows the file name: {@code .mkv} (Matroska) holds every + * codec WebRTC sends and is the safe choice; {@code .webm} holds VP8, VP9, + * AV1 and Opus; {@code .mp4} holds H.264, H.265, AV1, VP9 and Opus, and is + * written fragmented, so that it plays up to where a recording was cut short. + * A track whose codec the file cannot hold is left out and reported to the + * {@link MediaRecorderListener#onWarning(String) listener}. + *

+ * Example: + * 

{@code
+ * try (MediaRecorder recorder = new MediaRecorder(Paths.get("call.mkv"))) {
+ *     recorder.addTrack(videoReceiver);
+ *     recorder.addTrack(audioReceiver);
+ *     recorder.start();
+ *
+ *     // ... the call goes on ...
+ *
+ *     recorder.stop();
+ * }
+ * }
+ *

+ * How it behaves: + *

    + *
  • A video track is recorded from its first key frame, and the recorder + * asks the sender or receiver for one so that it does not have to wait. The + * sender or receiver must therefore not be disposed while recording.
    • + *
  • The file begins once every track has sent its first frames, or once + * the tracks that have waited three seconds for the rest; a track that sends + * nothing by then is left out.
    • + *
  • Tracks are placed on a common timeline by when their frames arrived, + * and follow their own RTP timestamps from there.
    • + *
  • A sender with simulcast is recorded at one of its layers.
    • + *
  • A sender's frames are recorded before an {@link + * dev.onvoid.webrtc.RTCEncodedFrameTransformer} runs on them, a receiver's + * after, so an end-to-end encrypted call is recorded in the clear.
    • + *
+ * + * @author Alex Andres + */ +public class MediaRecorder implements AutoCloseable { + + static { + FFmpeg.load(); + } + + private final Object lock = new Object(); + + private final Path file; + + /** + * How to ask each track for a key frame, by track index. Filled before + * the start, read by the writer thread after it. + */ + private final List keyFrameRequests = new CopyOnWriteArrayList<>(); + + /** Read on the event thread, so never a stale value. */ + private volatile MediaRecorderListener listener; + + /** Where listener calls and key frame requests run. */ + private ExecutorService events; + + /** The native recorder, or 0 once this recorder has been closed. */ + private long handle; + + private MediaRecorderState state = MediaRecorderState.IDLE; + + + /** + * Creates a recorder writing to the given file, which is created, or + * emptied if it exists. + * + * @param file Where to record to; its extension picks the container. + * + * @throws IOException If the file cannot be written. + */ + public MediaRecorder(Path file) throws IOException { + Objects.requireNonNull(file, "File is null"); + + this.file = file.toAbsolutePath(); + this.handle = create(this.file.toString(), NativeApi.tableAddress()); + } + + /** + * Sets what to report to, replacing whatever was set before. A listener of + * {@code null} stops reporting. + * + * @param listener The listener, or {@code null}. + */ + public void setListener(MediaRecorderListener listener) { + this.listener = listener; + } + + /** + * Records what the given sender sends, as it comes out of the encoder. + * + * @param sender The sender to record. + * + * @throws IllegalStateException If the recorder was started or closed. + * @throws IllegalArgumentException If the sender was disposed. + */ + public void addTrack(RTCRtpSender sender) { + Objects.requireNonNull(sender, "Sender is null"); + + synchronized (lock) { + checkState(MediaRecorderState.IDLE); + + addTrack(NativeApi.encodedFramesOf(sender), sender::generateKeyFrame); + } + } + + /** + * Records what the given receiver receives, as it goes into the decoder. + * + * @param receiver The receiver to record. + * + * @throws IllegalStateException If the recorder was started or closed. + * @throws IllegalArgumentException If the receiver was disposed. + */ + public void addTrack(RTCRtpReceiver receiver) { + Objects.requireNonNull(receiver, "Receiver is null"); + + synchronized (lock) { + checkState(MediaRecorderState.IDLE); + + addTrack(NativeApi.encodedFramesOf(receiver), receiver::requestKeyFrame); + } + } + + /** + * Starts recording the tracks added so far. + * + * @throws IllegalStateException If the recorder has no tracks, or was + * started or closed before. + */ + public void start() { + synchronized (lock) { + checkState(MediaRecorderState.IDLE); + + if (keyFrameRequests.isEmpty()) { + throw new IllegalStateException("A recorder needs at least one track"); + } + + events = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "MediaRecorder-events"); + thread.setDaemon(true); + return thread; + }); + + state = MediaRecorderState.RECORDING; + + start(handle); + } + } + + /** + * Stops recording and finishes the file, waiting for everything received + * so far to be written. A recorder that never got any media to write + * deletes its file rather than leave an unplayable one behind. + *

+ * Stopping a recorder that is stopped does nothing. Stopping one that was + * never started deletes its file. + * + * @return True if the file holds a recording, false if it was deleted. + */ + public boolean stop() { + boolean recorded; + + synchronized (lock) { + if (handle == 0) { + return false; + } + if (state == MediaRecorderState.STOPPED) { + return Files.exists(file); + } + + recorded = stop(handle); + state = MediaRecorderState.STOPPED; + + if (events != null) { + // Lets the reports already queued run, then ends the thread. + events.shutdown(); + } + } + + if (!recorded) { + try { + Files.deleteIfExists(file); + } + catch (IOException e) { + // The file stays behind, empty; there is nothing else to do. + } + } + + return recorded; + } + + /** + * Returns what the recorder is currently doing. + * + * @return The recorder state. + */ + public MediaRecorderState getState() { + synchronized (lock) { + return state; + } + } + + /** + * @return The file this recorder writes to. + */ + public Path getFile() { + return file; + } + + /** + * Stops the recording if it still runs and releases the native recorder. + * Closing a recorder that is closed does nothing. + */ + @Override + public void close() { + stop(); + + synchronized (lock) { + if (handle != 0) { + dispose(handle); + handle = 0; + } + } + } + + /** Called with the lock held, in the idle state. */ + private void addTrack(long frames, Runnable keyFrameRequest) { + if (frames == 0) { + throw new IllegalArgumentException("The sender or receiver was disposed"); + } + + // The native recorder takes over the reference the handle carries. + addTrack(handle, frames); + + keyFrameRequests.add(keyFrameRequest); + } + + private void checkState(MediaRecorderState expected) { + if (handle == 0) { + throw new IllegalStateException("The recorder is closed"); + } + if (state != expected) { + throw new IllegalStateException("The recorder is " + state.name().toLowerCase() + + ", not " + expected.name().toLowerCase()); + } + } + + private void post(Runnable event) { + ExecutorService executor = events; + + if (executor == null) { + return; + } + + try { + executor.execute(() -> { + try { + event.run(); + } + catch (Throwable e) { + Thread thread = Thread.currentThread(); + thread.getUncaughtExceptionHandler().uncaughtException(thread, e); + } + }); + } + catch (RejectedExecutionException e) { + // Stopped in the meantime; nobody is waiting for this any more. + } + } + + /** Called by native code on the writer thread. */ + private void onNativeStarted() { + post(() -> { + MediaRecorderListener current = listener; + + if (current != null) { + current.onStarted(); + } + }); + } + + /** Called by native code on the writer thread. */ + private void onNativeWarning(String message) { + post(() -> { + MediaRecorderListener current = listener; + + if (current != null) { + current.onWarning(message); + } + }); + } + + /** Called by native code on the writer thread. */ + private void onNativeError(String message) { + post(() -> { + MediaRecorderListener current = listener; + + if (current != null) { + current.onError(message); + } + }); + } + + /** Called by native code on the writer thread. */ + private void onNativeKeyFrameNeeded(int track) { + if (track < 0 || track >= keyFrameRequests.size()) { + return; + } + + Runnable request = keyFrameRequests.get(track); + + // Run away from the writer thread: asking WebRTC waits for its + // signaling thread, which may be busy stopping this very recorder. + post(() -> { + try { + request.run(); + } + catch (RuntimeException e) { + // Not negotiated yet, or disposed of by the application. The + // recording waits for the next key frame instead. + } + }); + } + + private native long create(String path, long tableAddress) throws IOException; + + private static native int addTrack(long handle, long frames); + + private static native void start(long handle); + + private static native boolean stop(long handle); + + private static native void dispose(long handle); + +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorderListener.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorderListener.java new file mode 100644 index 00000000..56ebaba6 --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorderListener.java @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.recorder; + +/** + * What a {@link MediaRecorder} reports while it records. + *

+ * Calls arrive in order on a thread the recorder keeps for them, never on a + * thread that carries media, so an implementation may take its time and may + * call back into the recorder, including to stop it. + * + * @author Alex Andres + */ +public interface MediaRecorderListener { + + /** + * The file header was written and media now goes into the file. This + * happens once every track has sent enough to describe its stream (for + * video, its first key frame), or once the tracks that have waited a few + * seconds for the others. + */ + default void onStarted() { + } + + /** + * Something went wrong that the recording carries on without, e.g. a + * track whose codec the file cannot hold, which is then left out. + * + * @param message What went wrong. + */ + default void onWarning(String message) { + } + + /** + * Writing the file failed, e.g. because the disk is full. Nothing more + * goes into the file; what is in it stays, and {@link MediaRecorder#stop()} + * still finishes it as far as it can. + * + * @param message What went wrong. + */ + default void onError(String message) { + } + +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorderState.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorderState.java new file mode 100644 index 00000000..5d00663c --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorderState.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.recorder; + +/** + * The states a {@link MediaRecorder} goes through, in this order. + * + * @author Alex Andres + */ +public enum MediaRecorderState { + + /** Created, taking tracks, not recording yet. */ + IDLE, + + /** Started; frames go into the file. */ + RECORDING, + + /** Stopped, with the file finished. A recorder cannot start again. */ + STOPPED + +} diff --git a/webrtc-java-media/src/main/java/module-info.java b/webrtc-java-media/src/main/java/module-info.java index 7ea0e974..b55c7029 100644 --- a/webrtc-java-media/src/main/java/module-info.java +++ b/webrtc-java-media/src/main/java/module-info.java @@ -1,11 +1,13 @@ /** * Media extension for webrtc-java: reads media files and network streams with - * FFmpeg and feeds them into a peer connection. + * FFmpeg and feeds them into a peer connection, and records what a peer + * connection sends or receives into media files. */ module webrtc.java.media { requires webrtc.java; exports dev.onvoid.webrtc.media.player; + exports dev.onvoid.webrtc.media.recorder; } diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/MediaRecorderTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/MediaRecorderTest.java new file mode 100644 index 00000000..36fab892 --- /dev/null +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/MediaRecorderTest.java @@ -0,0 +1,294 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.recorder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.media.audio.AudioDeviceModule; +import dev.onvoid.webrtc.media.audio.AudioLayer; +import dev.onvoid.webrtc.media.player.MediaInfo; +import dev.onvoid.webrtc.media.player.MediaReader; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests recording a call between two local peer connections, checked by + * reading the recorded file back. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Execution(ExecutionMode.SAME_THREAD) +class MediaRecorderTest { + + private static final long RECORD_MS = 3000; + + private AudioDeviceModule audioModule; + private PeerConnectionFactory factory; + + @TempDir + Path tempDir; + + + @BeforeAll + void initFactory() { + // A dummy audio layer, because this factory sends pushed audio. + audioModule = new AudioDeviceModule(AudioLayer.kDummyAudio); + factory = new PeerConnectionFactory(audioModule); + } + + @AfterAll + void disposeFactory() { + factory.dispose(); + audioModule.dispose(); + } + + @Test + void recordsReceivedCallIntoMatroska() throws Exception { + Path file = tempDir.resolve("received.mkv"); + Listener listener = new Listener(); + + try (TestCall call = new TestCall(factory)) { + // Well into the call, so that the receiver has had its key frame + // long ago and the recorder has to ask for one. + Thread.sleep(1500); + + RTCRtpReceiver video = call.getReceiver(0); + RTCRtpReceiver audio = call.getReceiver(1); + + try (MediaRecorder recorder = new MediaRecorder(file)) { + recorder.setListener(listener); + recorder.addTrack(video); + recorder.addTrack(audio); + recorder.start(); + + assertEquals(MediaRecorderState.RECORDING, recorder.getState()); + assertTrue(listener.started.await(10, TimeUnit.SECONDS), "not started"); + + Thread.sleep(RECORD_MS); + + assertTrue(recorder.stop()); + assertEquals(MediaRecorderState.STOPPED, recorder.getState()); + } + finally { + video.dispose(); + audio.dispose(); + } + } + + assertNull(listener.error.get()); + assertTrue(listener.warnings.isEmpty(), listener.warnings.toString()); + + MediaInfo info = readInfo(file); + + assertTrue(info.hasVideo()); + assertTrue(info.hasAudio()); + assertEquals("vp8", info.getVideoCodec()); + assertEquals(TestCall.WIDTH, info.getVideoWidth()); + assertEquals(TestCall.HEIGHT, info.getVideoHeight()); + assertEquals("opus", info.getAudioCodec()); + assertEquals(48000, info.getSampleRate()); + assertDuration(info); + } + + @Test + void recordsSentCallIntoWebm() throws Exception { + Path file = tempDir.resolve("sent.webm"); + Listener listener = new Listener(); + + try (TestCall call = new TestCall(factory); + MediaRecorder recorder = new MediaRecorder(file)) { + recorder.setListener(listener); + recorder.addTrack(call.getVideoSender()); + recorder.addTrack(call.getAudioSender()); + recorder.start(); + + assertTrue(listener.started.await(10, TimeUnit.SECONDS), "not started"); + + Thread.sleep(RECORD_MS); + + assertTrue(recorder.stop()); + } + + assertNull(listener.error.get()); + + MediaInfo info = readInfo(file); + + assertEquals("vp8", info.getVideoCodec()); + assertEquals("opus", info.getAudioCodec()); + assertDuration(info); + } + + @Test + void leavesOutCodecFileCannotHold() throws Exception { + // MP4 has no place for VP8, which the call sends. + Path file = tempDir.resolve("audio-only.mp4"); + Listener listener = new Listener(); + + try (TestCall call = new TestCall(factory); + MediaRecorder recorder = new MediaRecorder(file)) { + recorder.setListener(listener); + recorder.addTrack(call.getVideoSender()); + recorder.addTrack(call.getAudioSender()); + recorder.start(); + + assertTrue(listener.started.await(10, TimeUnit.SECONDS), "not started"); + + Thread.sleep(RECORD_MS); + + assertTrue(recorder.stop()); + } + + assertFalse(listener.warnings.isEmpty()); + assertTrue(listener.warnings.get(0).contains("video/VP8"), listener.warnings.get(0)); + + MediaInfo info = readInfo(file); + + assertFalse(info.hasVideo()); + assertEquals("opus", info.getAudioCodec()); + } + + @Test + void deletesFileWithoutMedia() throws Exception { + Path file = tempDir.resolve("nothing.mkv"); + + try (TestCall call = new TestCall(factory); + MediaRecorder recorder = new MediaRecorder(file)) { + assertTrue(Files.exists(file)); + + recorder.addTrack(call.getVideoSender()); + recorder.start(); + + // Too soon for any frame to have made it into the file. + assertFalse(recorder.stop()); + } + + assertFalse(Files.exists(file)); + } + + @Test + void unstartedRecorderDeletesFile() throws Exception { + Path file = tempDir.resolve("unstarted.mkv"); + + try (MediaRecorder recorder = new MediaRecorder(file)) { + assertEquals(MediaRecorderState.IDLE, recorder.getState()); + assertThrows(IllegalStateException.class, recorder::start); + } + + assertFalse(Files.exists(file)); + } + + @Test + void rejectsTracksOnceStarted() throws Exception { + Path file = tempDir.resolve("rejects.mkv"); + + try (TestCall call = new TestCall(factory); + MediaRecorder recorder = new MediaRecorder(file)) { + recorder.addTrack(call.getVideoSender()); + recorder.start(); + + assertThrows(IllegalStateException.class, () -> recorder.addTrack(call.getAudioSender())); + assertThrows(IllegalStateException.class, recorder::start); + } + } + + @Test + void stopFromListener() throws Exception { + Path file = tempDir.resolve("listener-stop.mkv"); + AtomicReference stopped = new AtomicReference<>(); + CountDownLatch done = new CountDownLatch(1); + + try (TestCall call = new TestCall(factory); + MediaRecorder recorder = new MediaRecorder(file)) { + recorder.setListener(new MediaRecorderListener() { + + @Override + public void onStarted() { + stopped.set(recorder.stop()); + done.countDown(); + } + }); + recorder.addTrack(call.getVideoSender()); + recorder.start(); + + assertTrue(done.await(10, TimeUnit.SECONDS), "stopping from the listener hung"); + } + + assertTrue(stopped.get()); + assertTrue(readInfo(file).hasVideo()); + } + + private static MediaInfo readInfo(Path file) throws Exception { + try (MediaReader reader = new MediaReader(file)) { + return reader.getInfo(); + } + } + + private static void assertDuration(MediaInfo info) { + long durationMs = info.getDurationUs() / 1000; + + // The file begins when the first key frame arrives, somewhat after + // the start, and ends with what was queued at the stop. + assertTrue(durationMs > RECORD_MS - 1000 && durationMs < RECORD_MS + 2000, + "duration: " + durationMs + " ms"); + } + + + private static class Listener implements MediaRecorderListener { + + final CountDownLatch started = new CountDownLatch(1); + + final List warnings = new CopyOnWriteArrayList<>(); + + final AtomicReference error = new AtomicReference<>(); + + + @Override + public void onStarted() { + started.countDown(); + } + + @Override + public void onWarning(String message) { + warnings.add(message); + } + + @Override + public void onError(String message) { + error.set(message); + } + } + +} diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java new file mode 100644 index 00000000..d105e4cf --- /dev/null +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java @@ -0,0 +1,272 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.recorder; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import dev.onvoid.webrtc.CreateSessionDescriptionObserver; +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.PeerConnectionObserver; +import dev.onvoid.webrtc.RTCAnswerOptions; +import dev.onvoid.webrtc.RTCConfiguration; +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCOfferOptions; +import dev.onvoid.webrtc.RTCPeerConnection; +import dev.onvoid.webrtc.RTCPeerConnectionState; +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.RTCRtpSender; +import dev.onvoid.webrtc.RTCRtpTransceiver; +import dev.onvoid.webrtc.RTCSessionDescription; +import dev.onvoid.webrtc.SetSessionDescriptionObserver; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoTrack; + +/** + * A call between two local peer connections that sends generated video + * (320x240 at 30 frames a second) and audio (a tone) from the caller to the + * callee, for recording. + */ +class TestCall implements AutoCloseable { + + static final int WIDTH = 320; + static final int HEIGHT = 240; + + private final CustomVideoSource videoSource = new CustomVideoSource(); + private final CustomAudioSource audioSource = new CustomAudioSource(); + private final VideoTrack videoTrack; + private final AudioTrack audioTrack; + + private final Peer caller; + private final Peer callee; + + private final RTCRtpSender videoSender; + private final RTCRtpSender audioSender; + + private volatile boolean feeding; + private Thread feeder; + + + TestCall(PeerConnectionFactory factory) throws Exception { + caller = new Peer(factory); + callee = new Peer(factory); + + caller.remote = callee; + callee.remote = caller; + + videoTrack = factory.createVideoTrack("video", videoSource); + audioTrack = factory.createAudioTrack("audio", audioSource); + + videoSender = caller.connection.addTrack(videoTrack, Collections.singletonList("stream")); + audioSender = caller.connection.addTrack(audioTrack, Collections.singletonList("stream")); + + RTCSessionDescription offer = caller.createOffer(); + callee.setRemoteDescription(offer); + RTCSessionDescription answer = callee.createAnswer(); + caller.setRemoteDescription(answer); + + if (!caller.connected.await(10, TimeUnit.SECONDS) || !callee.connected.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("The call did not connect"); + } + + feeding = true; + feeder = new Thread(this::feed, "TestCall-feeder"); + feeder.setDaemon(true); + feeder.start(); + } + + RTCRtpSender getVideoSender() { + return videoSender; + } + + RTCRtpSender getAudioSender() { + return audioSender; + } + + /** + * Returns the callee's receiver of the video (0) or audio (1) transceiver, + * which follow the order of the offer. The instance is the caller's to + * dispose. + */ + RTCRtpReceiver getReceiver(int index) { + RTCRtpTransceiver[] transceivers = callee.connection.getTransceivers(); + RTCRtpReceiver receiver = transceivers[index].getReceiver(); + + for (RTCRtpTransceiver transceiver : transceivers) { + transceiver.dispose(); + } + + return receiver; + } + + @Override + public void close() throws InterruptedException { + feeding = false; + feeder.join(); + + videoSender.dispose(); + audioSender.dispose(); + + caller.connection.close(); + callee.connection.close(); + + videoTrack.dispose(); + audioTrack.dispose(); + videoSource.dispose(); + audioSource.dispose(); + } + + private void feed() { + byte[] audio = new byte[480 * 2]; + long startNs = System.nanoTime(); + long chunks = 0; + long frames = 0; + int shade = 0; + + while (feeding) { + long elapsedMs = (System.nanoTime() - startNs) / 1_000_000; + + while (chunks * 10 <= elapsedMs) { + for (int i = 0; i < 480; i++) { + short sample = (short) (Math.sin(2 * Math.PI * 440 * (chunks * 480 + i) / 48000) * 8000); + audio[2 * i] = (byte) sample; + audio[2 * i + 1] = (byte) (sample >> 8); + } + + audioSource.pushAudio(audio, 16, 48000, 1, 480); + chunks++; + } + + if (frames * 1000 / 30 <= elapsedMs) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(WIDTH, HEIGHT); + ByteBuffer y = buffer.getDataY(); + + for (int i = 0; i < y.capacity(); i++) { + y.put(i, (byte) (i + shade)); + } + shade += 3; + + VideoFrame frame = new VideoFrame(buffer, 0); + videoSource.pushFrame(frame); + frame.release(); + frames++; + } + + try { + Thread.sleep(2); + } + catch (InterruptedException e) { + return; + } + } + } + + + private static class Peer implements PeerConnectionObserver { + + final RTCPeerConnection connection; + + final CountDownLatch connected = new CountDownLatch(1); + + volatile Peer remote; + + + Peer(PeerConnectionFactory factory) { + connection = factory.createPeerConnection(new RTCConfiguration(), this); + } + + @Override + public void onIceCandidate(RTCIceCandidate candidate) { + remote.connection.addIceCandidate(candidate); + } + + @Override + public void onConnectionChange(RTCPeerConnectionState state) { + if (state == RTCPeerConnectionState.CONNECTED) { + connected.countDown(); + } + } + + RTCSessionDescription createOffer() throws Exception { + CompletableFuture created = new CompletableFuture<>(); + connection.createOffer(new RTCOfferOptions(), created(created)); + + return setLocalDescription(created.get(10, TimeUnit.SECONDS)); + } + + RTCSessionDescription createAnswer() throws Exception { + CompletableFuture created = new CompletableFuture<>(); + connection.createAnswer(new RTCAnswerOptions(), created(created)); + + return setLocalDescription(created.get(10, TimeUnit.SECONDS)); + } + + void setRemoteDescription(RTCSessionDescription description) throws Exception { + CompletableFuture set = new CompletableFuture<>(); + connection.setRemoteDescription(description, set(set)); + set.get(10, TimeUnit.SECONDS); + } + + private RTCSessionDescription setLocalDescription(RTCSessionDescription description) + throws Exception { + CompletableFuture set = new CompletableFuture<>(); + connection.setLocalDescription(description, set(set)); + set.get(10, TimeUnit.SECONDS); + + return description; + } + + private static CreateSessionDescriptionObserver created( + CompletableFuture future) { + return new CreateSessionDescriptionObserver() { + + @Override + public void onSuccess(RTCSessionDescription description) { + future.complete(description); + } + + @Override + public void onFailure(String error) { + future.completeExceptionally(new IllegalStateException(error)); + } + }; + } + + private static SetSessionDescriptionObserver set(CompletableFuture future) { + return new SetSessionDescriptionObserver() { + + @Override + public void onSuccess() { + future.complete(null); + } + + @Override + public void onFailure(String error) { + future.completeExceptionally(new IllegalStateException(error)); + } + }; + } + } + +} diff --git a/webrtc-jni/src/main/cpp/include/JNI_NativeApi.h b/webrtc-jni/src/main/cpp/include/JNI_NativeApi.h index 320b8851..3014d693 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_NativeApi.h +++ b/webrtc-jni/src/main/cpp/include/JNI_NativeApi.h @@ -38,6 +38,22 @@ extern "C" { JNIEXPORT jint JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_version (JNIEnv *, jclass); + /* + * Class: dev_onvoid_webrtc_internal_NativeApi + * Method: senderFrames + * Signature: (J)J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_senderFrames + (JNIEnv *, jclass, jlong); + + /* + * Class: dev_onvoid_webrtc_internal_NativeApi + * Method: receiverFrames + * Signature: (J)J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_receiverFrames + (JNIEnv *, jclass, jlong); + #ifdef __cplusplus } #endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_RTCEncodedFrame.h b/webrtc-jni/src/main/cpp/include/JNI_RTCEncodedFrame.h new file mode 100644 index 00000000..45f193b3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_RTCEncodedFrame.h @@ -0,0 +1,52 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_RTCEncodedFrame */ + +#ifndef _Included_dev_onvoid_webrtc_RTCEncodedFrame +#define _Included_dev_onvoid_webrtc_RTCEncodedFrame +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_RTCEncodedFrame + * Method: copyData + * Signature: (JLjava/nio/ByteBuffer;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCEncodedFrame_copyData + (JNIEnv *, jclass, jlong, jobject); + + /* + * Class: dev_onvoid_webrtc_RTCEncodedFrame + * Method: setDataBuffer + * Signature: (JLjava/nio/ByteBuffer;II)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCEncodedFrame_setDataBuffer + (JNIEnv *, jclass, jlong, jobject, jint, jint); + + /* + * Class: dev_onvoid_webrtc_RTCEncodedFrame + * Method: setDataArray + * Signature: (J[BII)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCEncodedFrame_setDataArray + (JNIEnv *, jclass, jlong, jbyteArray, jint, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_RTCRtpReceiver.h b/webrtc-jni/src/main/cpp/include/JNI_RTCRtpReceiver.h index 2a16ba70..8a9c8b2e 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_RTCRtpReceiver.h +++ b/webrtc-jni/src/main/cpp/include/JNI_RTCRtpReceiver.h @@ -47,6 +47,22 @@ extern "C" { JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_RTCRtpReceiver_getSynchronizationSources (JNIEnv *, jobject); + /* + * Class: dev_onvoid_webrtc_RTCRtpReceiver + * Method: setTransform + * Signature: (Ldev/onvoid/webrtc/RTCEncodedFrameTransformer;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpReceiver_setTransform + (JNIEnv *, jobject, jobject); + + /* + * Class: dev_onvoid_webrtc_RTCRtpReceiver + * Method: requestKeyFrame + * Signature: ()V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpReceiver_requestKeyFrame + (JNIEnv *, jobject); + /* * Class: dev_onvoid_webrtc_RTCRtpReceiver * Method: dispose diff --git a/webrtc-jni/src/main/cpp/include/JNI_RTCRtpSender.h b/webrtc-jni/src/main/cpp/include/JNI_RTCRtpSender.h index eec94f3f..eced0d4d 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_RTCRtpSender.h +++ b/webrtc-jni/src/main/cpp/include/JNI_RTCRtpSender.h @@ -63,6 +63,22 @@ extern "C" { JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_RTCRtpSender_getDtmfSender (JNIEnv *, jobject); + /* + * Class: dev_onvoid_webrtc_RTCRtpSender + * Method: setTransform + * Signature: (Ldev/onvoid/webrtc/RTCEncodedFrameTransformer;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpSender_setTransform + (JNIEnv *, jobject, jobject); + + /* + * Class: dev_onvoid_webrtc_RTCRtpSender + * Method: generateKeyFrame + * Signature: ()V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpSender_generateKeyFrame + (JNIEnv *, jobject); + /* * Class: dev_onvoid_webrtc_RTCRtpSender * Method: dispose diff --git a/webrtc-jni/src/main/cpp/include/api/EncodedFrameRouter.h b/webrtc-jni/src/main/cpp/include/api/EncodedFrameRouter.h new file mode 100644 index 00000000..f7529dc4 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/EncodedFrameRouter.h @@ -0,0 +1,113 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_ENCODED_FRAME_ROUTER_H_ +#define JNI_WEBRTC_API_ENCODED_FRAME_ROUTER_H_ + +#include "webrtc_java_api.h" + +#include "api/frame_transformer_interface.h" +#include "api/media_types.h" +#include "api/scoped_refptr.h" + +#include +#include +#include +#include +#include +#include + +namespace jni +{ + class RTCEncodedFrameTransformer; + + // Everything an EncodedFrameTransformer knows about where its frames go: + // the WebRTC callbacks that take them back, the native observers that + // watch them pass, and the Java transform that may change them. + // + // It lives apart from the transformer so that the transformer's worker + // thread can keep it alive on its own. That thread is never joined, since + // a Java transform may do whatever it likes, including close the peer + // connection and with it release the transformer on a thread that would + // then wait for the transform to return. + // + // All methods are thread-safe. + class EncodedFrameRouter + { + public: + // A native observer, as the extension API hands it out. + struct Observer + { + wj_encoded_frame_fn fn; + void * opaque; + }; + + public: + EncodedFrameRouter(webrtc::MediaType mediaType, bool sender); + ~EncodedFrameRouter() = default; + + EncodedFrameRouter(const EncodedFrameRouter &) = delete; + EncodedFrameRouter & operator=(const EncodedFrameRouter &) = delete; + + webrtc::MediaType GetMediaType() const; + + // A sender's frames are observed before the Java transform, a + // receiver's after it, so that observers see what the codec made + // or will get, not what an encryption transform turns it into. + bool IsSender() const; + + void SetCallback(webrtc::scoped_refptr callback); + void SetSinkCallback(uint32_t ssrc, webrtc::scoped_refptr callback); + void RemoveCallback(); + void RemoveSinkCallback(uint32_t ssrc); + + void SetTransformer(std::shared_ptr transformer); + std::shared_ptr GetTransformer() const; + + Observer * AddObserver(wj_encoded_frame_fn fn, void * opaque); + // When this returns, the observer is not running and never runs + // again. + void RemoveObserver(Observer * observer); + + // Shows the frame to every observer. + void Observe(const webrtc::TransformableFrameInterface & frame); + + // Hands the frame back to WebRTC, to the callback that registered + // for its SSRC. + void Forward(std::unique_ptr frame); + + private: + const webrtc::MediaType mediaType; + const bool sender; + + mutable std::mutex callbackMutex; + webrtc::scoped_refptr callback; + std::map> sinkCallbacks; + + mutable std::mutex transformerMutex; + std::shared_ptr transformer; + + // Held while observers run, which is what lets RemoveObserver() + // promise that its observer is done. + std::mutex observerMutex; + std::vector> observers; + // Read without the lock, to keep the common case of no observers + // free of it. + std::atomic observerCount; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/api/EncodedFrameTransformer.h b/webrtc-jni/src/main/cpp/include/api/EncodedFrameTransformer.h new file mode 100644 index 00000000..43c44371 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/EncodedFrameTransformer.h @@ -0,0 +1,117 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_ENCODED_FRAME_TRANSFORMER_H_ +#define JNI_WEBRTC_API_ENCODED_FRAME_TRANSFORMER_H_ + +#include "api/EncodedFrameRouter.h" +#include "api/EncodedFrameWorker.h" + +#include "api/frame_transformer_interface.h" +#include "api/rtp_receiver_interface.h" +#include "api/rtp_sender_interface.h" +#include "api/scoped_refptr.h" + +#include + +#include +#include +#include +#include +#include + +namespace jni +{ + // The frame transformer this library installs on an RtpSender or + // RtpReceiver, through which both a Java transform and native observers + // (e.g. a recorder) see its encoded frames. + // + // A sender or receiver has room for one transformer, and there are good + // reasons to install it only once: replacing the transformer of a video + // sender recreates its send stream, which costs a key frame. So there is + // exactly one of these per sender or receiver, installed the first time + // anything asks for it and kept for the rest of its life; with nothing to + // do, it hands frames straight back on the thread they came on. + // + // Many Java objects may stand for the same native sender or receiver, so + // the transformer is found through a process-wide registry, not through + // the Java object. The registry holds no reference: a transformer leaves + // it when its sender or receiver, and everything else, lets it go. + class EncodedFrameTransformer : public webrtc::FrameTransformerInterface + { + public: + // Returns the transformer of the sender, installing one first if + // it has none. + static webrtc::scoped_refptr Of(webrtc::RtpSenderInterface * sender); + // Returns the transformer of the receiver, installing one first if + // it has none. + static webrtc::scoped_refptr Of(webrtc::RtpReceiverInterface * receiver); + + // Returns the transformer of the sender or receiver if one was + // installed, and null otherwise. + static webrtc::scoped_refptr Find(webrtc::RtpSenderInterface * sender); + static webrtc::scoped_refptr Find(webrtc::RtpReceiverInterface * receiver); + + // Sets the Java transform, or clears it with null. The first + // transform starts the worker it runs on, and from then on every + // frame passes through that worker, transform or not, so that + // frames never overtake each other when the transform changes. + void SetTransformer(JNIEnv * env, jobject transformer); + + EncodedFrameRouter::Observer * AddObserver(wj_encoded_frame_fn fn, void * opaque); + void RemoveObserver(EncodedFrameRouter::Observer * observer); + + // FrameTransformerInterface implementation. + void Transform(std::unique_ptr frame) override; + void RegisterTransformedFrameCallback(webrtc::scoped_refptr callback) override; + void RegisterTransformedFrameSinkCallback(webrtc::scoped_refptr callback, uint32_t ssrc) override; + void UnregisterTransformedFrameCallback() override; + void UnregisterTransformedFrameSinkCallback(uint32_t ssrc) override; + + // RefCountInterface implementation, done by hand so that the + // registry can tell a live transformer from a dying one. + void AddRef() const override; + webrtc::RefCountReleaseStatus Release() const override; + + protected: + ~EncodedFrameTransformer() override; + + private: + // A sender or receiver is known by its address and its ID, so that + // a new one allocated where an old one was is not mistaken for it + // while the old one's transformer is still being torn down. + using Key = std::pair; + + EncodedFrameTransformer(Key key, webrtc::MediaType mediaType, bool sender); + + template + static webrtc::scoped_refptr Lookup(T * owner, bool install); + + bool TryAddRef() const; + + private: + const Key key; + const std::shared_ptr router; + + mutable std::atomic refCount; + + // Set once, by the first Java transform, and only read after. + std::mutex workerMutex; + std::atomic worker; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/api/EncodedFrameWorker.h b/webrtc-jni/src/main/cpp/include/api/EncodedFrameWorker.h new file mode 100644 index 00000000..f3120c48 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/EncodedFrameWorker.h @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_ENCODED_FRAME_WORKER_H_ +#define JNI_WEBRTC_API_ENCODED_FRAME_WORKER_H_ + +#include "api/EncodedFrameRouter.h" + +#include "api/frame_transformer_interface.h" + +#include + +#include + +namespace jni +{ + // The thread a Java frame transform runs on. + // + // WebRTC hands frames to a transformer on the threads that carry media: a + // receiver's on the network thread, a sender's on the encoder queue. + // Running application code on those would stall the connection whenever + // the transform takes a while, and deadlock it whenever the transform + // calls back into the peer connection, which blocks on those very + // threads. So frames are queued and transformed here instead, in order, + // and handed back to WebRTC from here, which WebRTC allows from any + // thread. + // + // The thread is attached to the JVM once, as a daemon so that it never + // holds up the JVM's exit, and detaches itself when it ends. + class EncodedFrameWorker + { + public: + EncodedFrameWorker(JavaVM * vm, std::shared_ptr router); + + // Stops the thread without waiting for it. Frames still queued + // are released rather than sent; the frame being transformed, if + // any, finishes on its own. + ~EncodedFrameWorker(); + + EncodedFrameWorker(const EncodedFrameWorker &) = delete; + EncodedFrameWorker & operator=(const EncodedFrameWorker &) = delete; + + // Queues the frame for the transform. A transform that falls this + // far behind has stopped keeping up for good, so the frame is + // dropped rather than queued without bound. + void Post(std::unique_ptr frame); + + private: + struct Queue; + + static void Run(JavaVM * vm, std::shared_ptr queue, + std::shared_ptr router); + + private: + std::shared_ptr queue; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/api/RTCEncodedFrameTransformer.h b/webrtc-jni/src/main/cpp/include/api/RTCEncodedFrameTransformer.h new file mode 100644 index 00000000..24bd3e10 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/RTCEncodedFrameTransformer.h @@ -0,0 +1,110 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_RTC_ENCODED_FRAME_TRANSFORMER_H_ +#define JNI_WEBRTC_API_RTC_ENCODED_FRAME_TRANSFORMER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/frame_transformer_interface.h" +#include "api/media_types.h" + +#include + +#include +#include +#include + +namespace jni +{ + // Runs a Java RTCEncodedFrameTransformer on encoded frames. + // + // Each frame is shown to Java as an RTCEncodedAudioFrame or + // RTCEncodedVideoFrame carrying its metadata and the address of the + // native frame. The payload is not copied up front: the Java frame copies + // it only when asked for it, and only for as long as the transform runs. + class RTCEncodedFrameTransformer + { + public: + // Resolves the Java classes on the calling thread, which has to be + // one the application called in on, so that the worker running + // the transform never needs a class loader. + RTCEncodedFrameTransformer(JNIEnv * env, jobject transformer); + ~RTCEncodedFrameTransformer() = default; + + RTCEncodedFrameTransformer(const RTCEncodedFrameTransformer &) = delete; + RTCEncodedFrameTransformer & operator=(const RTCEncodedFrameTransformer &) = delete; + + // Runs the Java transform on the frame, which it may change in + // place. Returns whether the frame is to be sent on: false if the + // transform dropped it, or if it could not be run at all, since + // sending a frame untransformed could send in the clear what an + // encryption transform was meant to protect. + // + // Must be called on a thread attached to the JVM, and always on + // the same one. + bool Transform(JNIEnv * env, webrtc::TransformableFrameInterface & frame, + webrtc::MediaType mediaType); + + private: + jstring MimeType(JNIEnv * env, const std::string & mimeType); + + jobject NewVideoFrame(JNIEnv * env, webrtc::TransformableVideoFrameInterface & frame); + jobject NewAudioFrame(JNIEnv * env, webrtc::TransformableAudioFrameInterface & frame); + + private: + class JavaEncodedFrameClass : public JavaClass + { + public: + explicit JavaEncodedFrameClass(JNIEnv * env); + + jclass cls; + jmethodID dispatch; + }; + + class JavaEncodedVideoFrameClass : public JavaClass + { + public: + explicit JavaEncodedVideoFrameClass(JNIEnv * env); + + jclass cls; + jmethodID ctor; + }; + + class JavaEncodedAudioFrameClass : public JavaClass + { + public: + explicit JavaEncodedAudioFrameClass(JNIEnv * env); + + jclass cls; + jmethodID ctor; + }; + + private: + JavaGlobalRef transformer; + + const std::shared_ptr javaFrameClass; + const std::shared_ptr javaVideoFrameClass; + const std::shared_ptr javaAudioFrameClass; + + // A stream carries one or two codecs, so the MIME type strings are + // made once rather than for every frame. + std::unordered_map>> mimeTypes; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/webrtc_java_api.h b/webrtc-jni/src/main/cpp/include/webrtc_java_api.h index 155c9739..6ceded42 100644 --- a/webrtc-jni/src/main/cpp/include/webrtc_java_api.h +++ b/webrtc-jni/src/main/cpp/include/webrtc_java_api.h @@ -51,6 +51,11 @@ * synchronously on the calling thread, so the caller must not hold locks that * a WebRTC callback could need, and must pace its calls in real time: a frame * is encoded and sent when it arrives, not when its timestamp says. + * + * The same table also lets an extension observe the encoded frames of an + * RTCRtpSender or RTCRtpReceiver, which is how media is recorded without + * decoding it: dev.onvoid.webrtc.internal.NativeApi.encodedFramesOf() returns + * a handle that encoded_observer_add() attaches an observer to. */ #include @@ -147,6 +152,56 @@ struct wj_audio_chunk { int64_t timestamp_us; }; +/** The media kind of an encoded frame. */ +#define WEBRTC_JAVA_MEDIA_AUDIO 0 +#define WEBRTC_JAVA_MEDIA_VIDEO 1 + +/** + * One encoded frame, as an RTCRtpSender hands it to the packetizer or as an + * RTCRtpReceiver hands it to the decoder. + * + * A sender's frames are seen before the application's frame transform runs, + * a receiver's after it, so an observer always sees what the codec produced + * or will consume, never what an end-to-end encryption transform made of it. + * + * Everything the struct points to is borrowed for the duration of the call. + */ +struct wj_encoded_frame { + /** The encoded payload, "size" bytes. */ + const uint8_t * data; + /** The payload size in bytes; zero for frames without payload. */ + size_t size; + + /** WEBRTC_JAVA_MEDIA_AUDIO or WEBRTC_JAVA_MEDIA_VIDEO. */ + int media_type; + /** The codec as a MIME type, e.g. "video/VP8" or "audio/opus". */ + const char * mime_type; + /** The RTP payload type. */ + int payload_type; + /** The SSRC of the RTP stream the frame belongs to. */ + uint32_t ssrc; + /** The RTP timestamp of the frame, in the clock rate of the codec. */ + uint32_t rtp_timestamp; + + /** Non-zero for a video key frame; always zero for audio. */ + int key_frame; + /** Video frame width in pixels, or zero if not known for this frame. */ + int width; + /** Video frame height in pixels, or zero if not known for this frame. */ + int height; + + /** now_us() at the moment the frame passed. */ + int64_t time_us; +}; + +/** + * Called for every encoded frame passing the sender or receiver an observer + * is attached to. It runs on a WebRTC thread that carries media, so it must + * copy what it needs and return at once: it must not block, must not call + * into WebRTC, and must not add or remove observers. + */ +typedef void (*wj_encoded_frame_fn)(void * opaque, const struct wj_encoded_frame * frame); + /** * The function table this library exposes to native extensions. It is a * singleton with static storage duration, so its address stays valid for the @@ -216,6 +271,35 @@ struct webrtc_java_api { * not be used afterwards. */ void (*audio_source_release)(void * source); + + /** + * Attaches an observer to the encoded frames of a sender or receiver. + * The observer keeps what it observes alive on its own, so the handle + * may be released right after this call. + * + * @param frames The handle from NativeApi.encodedFramesOf(). + * @param fn Called for every frame; see wj_encoded_frame_fn. + * @param opaque Passed to "fn" unchanged. + * + * @return The observer, for encoded_observer_remove(), or null if a + * handle or the function was null. + */ + void * (*encoded_observer_add)(void * frames, wj_encoded_frame_fn fn, void * opaque); + + /** + * Detaches an observer. When this returns, "fn" is not running and will + * not be called again, so whatever "opaque" points to may be freed. It + * must not be called from within "fn" itself. + * + * @param observer The observer from encoded_observer_add(). + */ + void (*encoded_observer_remove)(void * observer); + + /** + * Drops the reference that NativeApi.encodedFramesOf() handed out. The + * handle must not be used afterwards. + */ + void (*encoded_frames_release)(void * frames); }; #ifdef __cplusplus diff --git a/webrtc-jni/src/main/cpp/src/JNI_NativeApi.cpp b/webrtc-jni/src/main/cpp/src/JNI_NativeApi.cpp index 6f0ad9e4..fced0ee9 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_NativeApi.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_NativeApi.cpp @@ -17,6 +17,17 @@ #include "JNI_NativeApi.h" #include "api/ExtensionApi.h" +#include "api/EncodedFrameTransformer.h" + +namespace +{ + // Hands the reference over to the extension, which drops it again with + // encoded_frames_release(). + jlong Retained(webrtc::scoped_refptr transformer) + { + return reinterpret_cast(transformer.release()); + } +} JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_tableAddress (JNIEnv * env, jclass caller) @@ -29,3 +40,19 @@ JNIEXPORT jint JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_version { return static_cast(jni::GetExtensionApi()->version); } + +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_senderFrames +(JNIEnv * env, jclass caller, jlong handle) +{ + auto sender = reinterpret_cast(handle); + + return sender != nullptr ? Retained(jni::EncodedFrameTransformer::Of(sender)) : 0; +} + +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_receiverFrames +(JNIEnv * env, jclass caller, jlong handle) +{ + auto receiver = reinterpret_cast(handle); + + return receiver != nullptr ? Retained(jni::EncodedFrameTransformer::Of(receiver)) : 0; +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_RTCEncodedFrame.cpp b/webrtc-jni/src/main/cpp/src/JNI_RTCEncodedFrame.cpp new file mode 100644 index 00000000..1df04794 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_RTCEncodedFrame.cpp @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_RTCEncodedFrame.h" + +#include "api/frame_transformer_interface.h" + +#include +#include + +// The Java side checks the handle, the calling thread, the bounds and the +// capacity before calling any of these, since it can throw the right +// exception for each; what is left to check here is what only native code +// can see. + +namespace +{ + webrtc::TransformableFrameInterface * FrameOf(jlong handle) + { + return reinterpret_cast(handle); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCEncodedFrame_copyData +(JNIEnv * env, jclass caller, jlong handle, jobject target) +{ + webrtc::TransformableFrameInterface * frame = FrameOf(handle); + + if (frame == nullptr) { + return; + } + + uint8_t * address = static_cast(env->GetDirectBufferAddress(target)); + jlong capacity = env->GetDirectBufferCapacity(target); + std::span data = frame->GetData(); + + if (address != nullptr && !data.empty() && capacity >= static_cast(data.size())) { + std::memcpy(address, data.data(), data.size()); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCEncodedFrame_setDataBuffer +(JNIEnv * env, jclass caller, jlong handle, jobject source, jint offset, jint length) +{ + webrtc::TransformableFrameInterface * frame = FrameOf(handle); + const uint8_t * address = static_cast(env->GetDirectBufferAddress(source)); + + if (frame == nullptr || address == nullptr) { + return; + } + + frame->SetData(std::span(address + offset, static_cast(length))); +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCEncodedFrame_setDataArray +(JNIEnv * env, jclass caller, jlong handle, jbyteArray source, jint offset, jint length) +{ + webrtc::TransformableFrameInterface * frame = FrameOf(handle); + + if (frame == nullptr) { + return; + } + + // SetData() copies, so the array stays pinned for no longer than that. + void * elements = env->GetPrimitiveArrayCritical(source, nullptr); + + if (elements == nullptr) { + return; + } + + const uint8_t * address = static_cast(elements) + offset; + + frame->SetData(std::span(address, static_cast(length))); + + env->ReleasePrimitiveArrayCritical(source, elements, JNI_ABORT); +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_RTCRtpReceiver.cpp b/webrtc-jni/src/main/cpp/src/JNI_RTCRtpReceiver.cpp index 41a9ca0d..154da0b4 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_RTCRtpReceiver.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_RTCRtpReceiver.cpp @@ -22,6 +22,9 @@ #include "JavaList.h" #include "JavaUtils.h" +#include "api/EncodedFrameTransformer.h" + +#include "api/media_stream_interface.h" #include "api/rtp_receiver_interface.h" #include @@ -98,6 +101,54 @@ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_RTCRtpReceiver_getSynchronizati return list.release(); } +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpReceiver_setTransform +(JNIEnv * env, jobject caller, jobject jTransformer) +{ + webrtc::RtpReceiverInterface * receiver = GetHandle(env, caller); + CHECK_HANDLE(receiver); + + try { + // Clearing a transform that was never set installs nothing. + auto transformer = jTransformer != nullptr + ? jni::EncodedFrameTransformer::Of(receiver) + : jni::EncodedFrameTransformer::Find(receiver); + + if (transformer) { + transformer->SetTransformer(env, jTransformer); + } + } + catch (...) { + ThrowCxxJavaException(env); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpReceiver_requestKeyFrame +(JNIEnv * env, jobject caller) +{ + webrtc::RtpReceiverInterface * receiver = GetHandle(env, caller); + CHECK_HANDLE(receiver); + + if (receiver->media_type() != webrtc::MediaType::VIDEO) { + return; + } + + // A receiver has no way of its own to ask for a key frame, but the + // source of its track does: it is what the decoder feeds, and the one + // thing WebRTC lets ask the remote sender for a key frame on demand. + webrtc::scoped_refptr track = receiver->track(); + + if (track == nullptr) { + return; + } + + auto videoTrack = static_cast(track.get()); + webrtc::VideoTrackSourceInterface * source = videoTrack->GetSource(); + + if (source != nullptr) { + source->GenerateKeyFrame(); + } +} + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpReceiver_dispose (JNIEnv * env, jobject caller) { diff --git a/webrtc-jni/src/main/cpp/src/JNI_RTCRtpSender.cpp b/webrtc-jni/src/main/cpp/src/JNI_RTCRtpSender.cpp index 634abae1..a70435f1 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_RTCRtpSender.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_RTCRtpSender.cpp @@ -23,6 +23,7 @@ #include "JavaRuntimeException.h" #include "JavaUtils.h" +#include "api/EncodedFrameTransformer.h" #include "api/RTCDtmfSender.h" #include "api/rtp_sender_interface.h" @@ -122,6 +123,45 @@ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_RTCRtpSender_getDtmfSender return jDtmfSender.release(); } +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpSender_setTransform +(JNIEnv * env, jobject caller, jobject jTransformer) +{ + webrtc::RtpSenderInterface * sender = GetHandle(env, caller); + CHECK_HANDLE(sender); + + try { + // Clearing a transform that was never set installs nothing. + auto transformer = jTransformer != nullptr + ? jni::EncodedFrameTransformer::Of(sender) + : jni::EncodedFrameTransformer::Find(sender); + + if (transformer) { + transformer->SetTransformer(env, jTransformer); + } + } + catch (...) { + ThrowCxxJavaException(env); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpSender_generateKeyFrame +(JNIEnv * env, jobject caller) +{ + webrtc::RtpSenderInterface * sender = GetHandle(env, caller); + CHECK_HANDLE(sender); + + // Audio has no key frames, so there is nothing to ask for. + if (sender->media_type() != webrtc::MediaType::VIDEO) { + return; + } + + webrtc::RTCError result = sender->GenerateKeyFrame({}); + + if (!result.ok()) { + env->Throw(jni::JavaRuntimeException(env, jni::RTCErrorToString(result).c_str())); + } +} + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCRtpSender_dispose (JNIEnv * env, jobject caller) { diff --git a/webrtc-jni/src/main/cpp/src/api/EncodedFrameRouter.cpp b/webrtc-jni/src/main/cpp/src/api/EncodedFrameRouter.cpp new file mode 100644 index 00000000..c97376e3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/EncodedFrameRouter.cpp @@ -0,0 +1,213 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/EncodedFrameRouter.h" +#include "api/RTCEncodedFrameTransformer.h" + +#include "rtc_base/time_utils.h" + +#include +#include +#include + +namespace jni +{ + EncodedFrameRouter::EncodedFrameRouter(webrtc::MediaType mediaType, bool sender) : + mediaType(mediaType), + sender(sender), + observerCount(0) + { + } + + webrtc::MediaType EncodedFrameRouter::GetMediaType() const + { + return mediaType; + } + + bool EncodedFrameRouter::IsSender() const + { + return sender; + } + + void EncodedFrameRouter::SetCallback(webrtc::scoped_refptr callback) + { + std::lock_guard lock(callbackMutex); + + this->callback = std::move(callback); + } + + void EncodedFrameRouter::SetSinkCallback(uint32_t ssrc, webrtc::scoped_refptr callback) + { + std::lock_guard lock(callbackMutex); + + sinkCallbacks[ssrc] = std::move(callback); + } + + void EncodedFrameRouter::RemoveCallback() + { + webrtc::scoped_refptr removed; + + { + std::lock_guard lock(callbackMutex); + + removed = std::move(callback); + } + + // Released outside the lock: the last reference may take WebRTC state + // down with it, which is none of this lock's business. + } + + void EncodedFrameRouter::RemoveSinkCallback(uint32_t ssrc) + { + webrtc::scoped_refptr removed; + + { + std::lock_guard lock(callbackMutex); + + auto found = sinkCallbacks.find(ssrc); + + if (found != sinkCallbacks.end()) { + removed = std::move(found->second); + sinkCallbacks.erase(found); + } + } + } + + void EncodedFrameRouter::SetTransformer(std::shared_ptr transformer) + { + std::shared_ptr previous; + + { + std::lock_guard lock(transformerMutex); + + previous = std::move(this->transformer); + this->transformer = std::move(transformer); + } + + // The previous transform holds a global reference, which is deleted + // here, outside the lock, or on the worker once it finishes a frame + // it is still running. + } + + std::shared_ptr EncodedFrameRouter::GetTransformer() const + { + std::lock_guard lock(transformerMutex); + + return transformer; + } + + EncodedFrameRouter::Observer * EncodedFrameRouter::AddObserver(wj_encoded_frame_fn fn, void * opaque) + { + auto observer = std::make_unique(Observer{ fn, opaque }); + Observer * handle = observer.get(); + + std::lock_guard lock(observerMutex); + + observers.push_back(std::move(observer)); + observerCount.store(observers.size(), std::memory_order_release); + + return handle; + } + + void EncodedFrameRouter::RemoveObserver(Observer * observer) + { + std::lock_guard lock(observerMutex); + + auto found = std::find_if(observers.begin(), observers.end(), + [observer](const std::unique_ptr & o) { return o.get() == observer; }); + + if (found != observers.end()) { + observers.erase(found); + } + + observerCount.store(observers.size(), std::memory_order_release); + } + + void EncodedFrameRouter::Observe(const webrtc::TransformableFrameInterface & frame) + { + if (observerCount.load(std::memory_order_acquire) == 0) { + return; + } + + std::lock_guard lock(observerMutex); + + if (observers.empty()) { + return; + } + + const std::span data = frame.GetData(); + const std::string mimeType = frame.GetMimeType(); + + wj_encoded_frame encoded = {}; + encoded.data = data.data(); + encoded.size = data.size(); + encoded.mime_type = mimeType.c_str(); + encoded.payload_type = frame.GetPayloadType(); + encoded.ssrc = frame.GetSsrc(); + encoded.rtp_timestamp = std::visit([](auto timestamp) { return timestamp.value; }, + frame.GetRtpTimestampInfo()); + encoded.time_us = webrtc::TimeMicros(); + + if (mediaType == webrtc::MediaType::VIDEO) { + // The frames of a video sender or receiver are always video + // frames, which is what makes this cast safe. + const auto & videoFrame = static_cast(frame); + const webrtc::VideoFrameMetadata metadata = videoFrame.Metadata(); + + encoded.media_type = WEBRTC_JAVA_MEDIA_VIDEO; + encoded.key_frame = videoFrame.IsKeyFrame() ? 1 : 0; + encoded.width = metadata.GetWidth(); + encoded.height = metadata.GetHeight(); + } + else { + encoded.media_type = WEBRTC_JAVA_MEDIA_AUDIO; + } + + for (const auto & observer : observers) { + observer->fn(observer->opaque, &encoded); + } + } + + void EncodedFrameRouter::Forward(std::unique_ptr frame) + { + webrtc::scoped_refptr target; + + { + std::lock_guard lock(callbackMutex); + + auto found = sinkCallbacks.find(frame->GetSsrc()); + + if (found != sinkCallbacks.end()) { + target = found->second; + } + else if (callback) { + target = callback; + } + else if (sinkCallbacks.size() == 1) { + // A receiver of an unsignaled stream registers before it + // knows the SSRC it will get. + target = sinkCallbacks.begin()->second; + } + } + + // Called outside the lock, since WebRTC may unregister from within. + // A frame with nowhere to go belongs to a stream that is gone, and is + // simply released. + if (target) { + target->OnTransformedFrame(std::move(frame)); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/api/EncodedFrameTransformer.cpp b/webrtc-jni/src/main/cpp/src/api/EncodedFrameTransformer.cpp new file mode 100644 index 00000000..bbe2ba95 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/EncodedFrameTransformer.cpp @@ -0,0 +1,264 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/EncodedFrameTransformer.h" +#include "api/RTCEncodedFrameTransformer.h" + +#include +#include + +namespace jni +{ + namespace + { + using RegistryKey = std::pair; + using RegistryMap = std::map; + + // Never destroyed: a transformer may be released while the process + // shuts down, after static destructors have run. + std::mutex & RegistryMutex() + { + static std::mutex * mutex = new std::mutex(); + return *mutex; + } + + RegistryMap & Registry() + { + static RegistryMap * registry = new RegistryMap(); + return *registry; + } + } + + webrtc::scoped_refptr EncodedFrameTransformer::Of(webrtc::RtpSenderInterface * sender) + { + return Lookup(sender, true); + } + + webrtc::scoped_refptr EncodedFrameTransformer::Of(webrtc::RtpReceiverInterface * receiver) + { + return Lookup(receiver, true); + } + + webrtc::scoped_refptr EncodedFrameTransformer::Find(webrtc::RtpSenderInterface * sender) + { + return Lookup(sender, false); + } + + webrtc::scoped_refptr EncodedFrameTransformer::Find(webrtc::RtpReceiverInterface * receiver) + { + return Lookup(receiver, false); + } + + template + webrtc::scoped_refptr EncodedFrameTransformer::Lookup(T * owner, bool install) + { + if (owner == nullptr) { + return nullptr; + } + + constexpr bool isSender = std::is_base_of_v; + + // Asked before taking the lock: the owner is a proxy, which runs this + // on the signaling thread and waits for it. + Key key(static_cast(owner), owner->id()); + + auto find = [&key]() -> EncodedFrameTransformer * { + // Only a transformer that is not already on its way out counts; + // one whose last reference is being dropped is as good as gone, + // and its Release() leaves a replacement in the registry alone. + auto found = Registry().find(key); + + if (found != Registry().end() && found->second->TryAddRef()) { + return found->second; + } + + return nullptr; + }; + + EncodedFrameTransformer * existing = nullptr; + + { + std::lock_guard lock(RegistryMutex()); + + existing = find(); + } + + if (existing == nullptr && !install) { + return nullptr; + } + + webrtc::scoped_refptr transformer; + + if (existing == nullptr) { + transformer = webrtc::scoped_refptr( + new EncodedFrameTransformer(key, owner->media_type(), isSender)); + + std::lock_guard lock(RegistryMutex()); + + // Another thread may have installed one in the meantime, and that + // one wins; this one goes again as soon as it is let go of. + existing = find(); + + if (existing == nullptr) { + Registry()[key] = transformer.get(); + } + } + + if (existing != nullptr) { + // TryAddRef() took the reference this pointer now carries. + transformer = webrtc::scoped_refptr(existing); + existing->Release(); + + return transformer; + } + + owner->SetFrameTransformer(transformer); + + return transformer; + } + + EncodedFrameTransformer::EncodedFrameTransformer(Key key, webrtc::MediaType mediaType, bool sender) : + key(std::move(key)), + router(std::make_shared(mediaType, sender)), + refCount(0), + worker(nullptr) + { + } + + EncodedFrameTransformer::~EncodedFrameTransformer() + { + // Stops the worker without waiting for it; the router it shares lives + // on until the worker is done with it. + delete worker.load(std::memory_order_acquire); + } + + void EncodedFrameTransformer::SetTransformer(JNIEnv * env, jobject transformer) + { + std::shared_ptr javaTransformer; + + if (transformer != nullptr) { + javaTransformer = std::make_shared(env, transformer); + + std::lock_guard lock(workerMutex); + + if (worker.load(std::memory_order_acquire) == nullptr) { + JavaVM * vm = nullptr; + + if (env->GetJavaVM(&vm) == JNI_OK) { + worker.store(new EncodedFrameWorker(vm, router), std::memory_order_release); + } + } + } + + router->SetTransformer(std::move(javaTransformer)); + } + + EncodedFrameRouter::Observer * EncodedFrameTransformer::AddObserver(wj_encoded_frame_fn fn, void * opaque) + { + return router->AddObserver(fn, opaque); + } + + void EncodedFrameTransformer::RemoveObserver(EncodedFrameRouter::Observer * observer) + { + router->RemoveObserver(observer); + } + + void EncodedFrameTransformer::Transform(std::unique_ptr frame) + { + if (!frame) { + return; + } + + if (router->IsSender()) { + router->Observe(*frame); + } + + EncodedFrameWorker * frameWorker = worker.load(std::memory_order_acquire); + + if (frameWorker != nullptr) { + frameWorker->Post(std::move(frame)); + return; + } + + if (!router->IsSender()) { + router->Observe(*frame); + } + + router->Forward(std::move(frame)); + } + + void EncodedFrameTransformer::RegisterTransformedFrameCallback(webrtc::scoped_refptr callback) + { + router->SetCallback(std::move(callback)); + } + + void EncodedFrameTransformer::RegisterTransformedFrameSinkCallback(webrtc::scoped_refptr callback, uint32_t ssrc) + { + router->SetSinkCallback(ssrc, std::move(callback)); + } + + void EncodedFrameTransformer::UnregisterTransformedFrameCallback() + { + router->RemoveCallback(); + } + + void EncodedFrameTransformer::UnregisterTransformedFrameSinkCallback(uint32_t ssrc) + { + router->RemoveSinkCallback(ssrc); + } + + void EncodedFrameTransformer::AddRef() const + { + refCount.fetch_add(1, std::memory_order_relaxed); + } + + webrtc::RefCountReleaseStatus EncodedFrameTransformer::Release() const + { + if (refCount.fetch_sub(1, std::memory_order_acq_rel) != 1) { + return webrtc::RefCountReleaseStatus::kOtherRefsRemained; + } + + { + std::lock_guard lock(RegistryMutex()); + + auto found = Registry().find(key); + + // A lookup that found this transformer dying may have put a new + // one in its place, which stays. + if (found != Registry().end() && found->second == this) { + Registry().erase(found); + } + } + + delete this; + + return webrtc::RefCountReleaseStatus::kDroppedLastRef; + } + + bool EncodedFrameTransformer::TryAddRef() const + { + int count = refCount.load(std::memory_order_relaxed); + + while (count > 0) { + if (refCount.compare_exchange_weak(count, count + 1, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + return true; + } + } + + return false; + } +} diff --git a/webrtc-jni/src/main/cpp/src/api/EncodedFrameWorker.cpp b/webrtc-jni/src/main/cpp/src/api/EncodedFrameWorker.cpp new file mode 100644 index 00000000..8281918b --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/EncodedFrameWorker.cpp @@ -0,0 +1,156 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/EncodedFrameWorker.h" +#include "api/RTCEncodedFrameTransformer.h" + +#include "rtc_base/logging.h" + +#include +#include +#include +#include +#include + +namespace jni +{ + namespace + { + // A few seconds of video and audio together. A transform this far + // behind is not going to catch up. + constexpr size_t kMaxQueuedFrames = 256; + } + + struct EncodedFrameWorker::Queue + { + std::mutex mutex; + std::condition_variable condition; + std::deque> frames; + bool stopped = false; + bool reportedOverflow = false; + }; + + EncodedFrameWorker::EncodedFrameWorker(JavaVM * vm, std::shared_ptr router) : + queue(std::make_shared()) + { + std::thread(&EncodedFrameWorker::Run, vm, queue, std::move(router)).detach(); + } + + EncodedFrameWorker::~EncodedFrameWorker() + { + std::deque> dropped; + + { + std::lock_guard lock(queue->mutex); + + queue->stopped = true; + dropped.swap(queue->frames); + } + + queue->condition.notify_one(); + + // The queued frames are released here, outside the lock. + } + + void EncodedFrameWorker::Post(std::unique_ptr frame) + { + { + std::lock_guard lock(queue->mutex); + + if (queue->stopped) { + return; + } + + if (queue->frames.size() >= kMaxQueuedFrames) { + if (!queue->reportedOverflow) { + queue->reportedOverflow = true; + + RTC_LOG(LS_WARNING) << "Encoded frame transform does not keep up, dropping frames"; + } + + return; + } + + queue->frames.push_back(std::move(frame)); + } + + queue->condition.notify_one(); + } + + void EncodedFrameWorker::Run(JavaVM * vm, std::shared_ptr queue, + std::shared_ptr router) + { + JNIEnv * env = nullptr; + + JavaVMAttachArgs args; + args.version = JNI_VERSION_1_6; + args.name = const_cast("EncodedFrameTransform"); + args.group = nullptr; + + if (vm->AttachCurrentThreadAsDaemon(reinterpret_cast(&env), &args) != JNI_OK) { + env = nullptr; + + RTC_LOG(LS_ERROR) << "Encoded frame transform cannot attach its thread to the JVM"; + } + + while (true) { + std::unique_ptr frame; + + { + std::unique_lock lock(queue->mutex); + + queue->condition.wait(lock, [&queue] { + return queue->stopped || !queue->frames.empty(); + }); + + if (queue->stopped) { + break; + } + + frame = std::move(queue->frames.front()); + queue->frames.pop_front(); + } + + std::shared_ptr transformer = router->GetTransformer(); + + if (transformer) { + // A thread that could not attach cannot run the transform, and + // the frame must then not go out untransformed. + if (env == nullptr || !transformer->Transform(env, *frame, router->GetMediaType())) { + continue; + } + + // Released while still attached: it may be the last reference + // to a transform that was replaced in the meantime. + transformer.reset(); + } + + if (!router->IsSender()) { + router->Observe(*frame); + } + + router->Forward(std::move(frame)); + } + + // Everything that may hold a Java reference goes before the thread + // leaves the JVM. + router.reset(); + + if (env != nullptr) { + vm->DetachCurrentThread(); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/api/ExtensionApi.cpp b/webrtc-jni/src/main/cpp/src/api/ExtensionApi.cpp index e7bc858f..b65e2742 100644 --- a/webrtc-jni/src/main/cpp/src/api/ExtensionApi.cpp +++ b/webrtc-jni/src/main/cpp/src/api/ExtensionApi.cpp @@ -15,6 +15,7 @@ */ #include "api/ExtensionApi.h" +#include "api/EncodedFrameTransformer.h" #include "media/audio/CustomAudioSource.h" #include "media/video/CustomVideoSource.h" @@ -185,6 +186,49 @@ namespace jni } } + // What encoded_observer_add() hands out: the observer, together with + // a reference to the transformer it is attached to, so that removing + // it needs nothing else and the transformer outlives it. + struct EncodedObserver + { + webrtc::scoped_refptr transformer; + EncodedFrameRouter::Observer * observer; + }; + + void * ApiEncodedObserverAdd(void * frames, wj_encoded_frame_fn fn, void * opaque) + { + if (frames == nullptr || fn == nullptr) { + return nullptr; + } + + webrtc::scoped_refptr transformer( + static_cast(frames)); + + EncodedFrameRouter::Observer * observer = transformer->AddObserver(fn, opaque); + + return new EncodedObserver{ std::move(transformer), observer }; + } + + void ApiEncodedObserverRemove(void * observer) + { + if (observer == nullptr) { + return; + } + + EncodedObserver * encodedObserver = static_cast(observer); + + encodedObserver->transformer->RemoveObserver(encodedObserver->observer); + + delete encodedObserver; + } + + void ApiEncodedFramesRelease(void * frames) + { + if (frames != nullptr) { + static_cast(frames)->Release(); + } + } + const webrtc_java_api kExtensionApi = { WEBRTC_JAVA_API_VERSION, static_cast(sizeof(webrtc_java_api)), @@ -194,7 +238,10 @@ namespace jni &ApiVideoSourceRetain, &ApiVideoSourceRelease, &ApiAudioSourceRetain, - &ApiAudioSourceRelease + &ApiAudioSourceRelease, + &ApiEncodedObserverAdd, + &ApiEncodedObserverRemove, + &ApiEncodedFramesRelease }; } diff --git a/webrtc-jni/src/main/cpp/src/api/RTCEncodedFrameTransformer.cpp b/webrtc-jni/src/main/cpp/src/api/RTCEncodedFrameTransformer.cpp new file mode 100644 index 00000000..daf7c6c2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/RTCEncodedFrameTransformer.cpp @@ -0,0 +1,220 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/RTCEncodedFrameTransformer.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include +#include + +namespace jni +{ + namespace + { + constexpr const char * kVideoFrameCtorSig = + "(JIJJIL" "java/lang/String;" "JZIIL" "java/lang/String;" "JII)V"; + + constexpr const char * kAudioFrameCtorSig = + "(JIJJIL" "java/lang/String;" "JII[I)V"; + + jlong RtpTimestamp(const webrtc::TransformableFrameInterface & frame) + { + uint32_t timestamp = std::visit([](auto t) { return t.value; }, frame.GetRtpTimestampInfo()); + + return static_cast(timestamp); + } + + jlong CaptureTimeUs(const webrtc::TransformableFrameInterface & frame) + { + std::optional time = frame.CaptureTime(); + + return time.has_value() ? static_cast(time->us()) : -1; + } + + // Drops whatever the Java side left pending. Nothing called on the + // worker may throw into C++, and a pending exception must not reach + // the next JNI call either. + bool ClearException(JNIEnv * env) + { + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + + return true; + } + + return false; + } + } + + RTCEncodedFrameTransformer::RTCEncodedFrameTransformer(JNIEnv * env, jobject transformer) : + transformer(env, transformer), + javaFrameClass(JavaClasses::get(env)), + javaVideoFrameClass(JavaClasses::get(env)), + javaAudioFrameClass(JavaClasses::get(env)) + { + } + + bool RTCEncodedFrameTransformer::Transform(JNIEnv * env, webrtc::TransformableFrameInterface & frame, + webrtc::MediaType mediaType) + { + // The worker stays attached for as long as it runs, so the local + // references of one frame must not pile up behind the next. + if (env->PushLocalFrame(8) != JNI_OK) { + ClearException(env); + + return false; + } + + jobject jFrame = mediaType == webrtc::MediaType::VIDEO + ? NewVideoFrame(env, static_cast(frame)) + : NewAudioFrame(env, static_cast(frame)); + + bool forward = false; + + if (jFrame != nullptr && !ClearException(env)) { + // The Java side catches whatever the transform throws, reports it + // and treats the frame as dropped; it also invalidates the Java + // frame before returning, so nothing can reach the native frame + // once it has moved on. + forward = env->CallStaticBooleanMethod(javaFrameClass->cls, javaFrameClass->dispatch, + transformer.get(), jFrame) == JNI_TRUE; + + if (ClearException(env)) { + forward = false; + } + } + else { + ClearException(env); + } + + env->PopLocalFrame(nullptr); + + return forward; + } + + jstring RTCEncodedFrameTransformer::MimeType(JNIEnv * env, const std::string & mimeType) + { + auto found = mimeTypes.find(mimeType); + + if (found != mimeTypes.end()) { + return found->second->get(); + } + + jstring text = env->NewStringUTF(mimeType.c_str()); + + if (text == nullptr) { + return nullptr; + } + + auto ref = std::make_unique>(env, text); + jstring global = ref->get(); + + env->DeleteLocalRef(text); + + mimeTypes.emplace(mimeType, std::move(ref)); + + return global; + } + + jobject RTCEncodedFrameTransformer::NewVideoFrame(JNIEnv * env, webrtc::TransformableVideoFrameInterface & frame) + { + const webrtc::VideoFrameMetadata metadata = frame.Metadata(); + const std::optional rid = frame.Rid(); + const std::optional frameId = metadata.GetFrameId(); + + jstring mimeType = MimeType(env, frame.GetMimeType()); + jstring jRid = nullptr; + + if (rid.has_value() && !rid->empty()) { + jRid = env->NewStringUTF(rid->c_str()); + } + + return env->NewObject(javaVideoFrameClass->cls, javaVideoFrameClass->ctor, + reinterpret_cast(&frame), + static_cast(frame.GetData().size()), + RtpTimestamp(frame), + static_cast(frame.GetSsrc()), + static_cast(frame.GetPayloadType()), + mimeType, + CaptureTimeUs(frame), + frame.IsKeyFrame() ? JNI_TRUE : JNI_FALSE, + static_cast(metadata.GetWidth()), + static_cast(metadata.GetHeight()), + jRid, + static_cast(frameId.value_or(-1)), + static_cast(metadata.GetSpatialIndex()), + static_cast(metadata.GetTemporalIndex())); + } + + jobject RTCEncodedFrameTransformer::NewAudioFrame(JNIEnv * env, webrtc::TransformableAudioFrameInterface & frame) + { + const std::span sources = frame.GetContributingSources(); + const std::optional sequenceNumber = frame.SequenceNumber(); + const std::optional audioLevel = frame.AudioLevel(); + + jstring mimeType = MimeType(env, frame.GetMimeType()); + jintArray csrcs = nullptr; + + if (!sources.empty()) { + csrcs = env->NewIntArray(static_cast(sources.size())); + + if (csrcs == nullptr) { + return nullptr; + } + + std::vector values(sources.begin(), sources.end()); + + env->SetIntArrayRegion(csrcs, 0, static_cast(values.size()), values.data()); + } + + return env->NewObject(javaAudioFrameClass->cls, javaAudioFrameClass->ctor, + reinterpret_cast(&frame), + static_cast(frame.GetData().size()), + RtpTimestamp(frame), + static_cast(frame.GetSsrc()), + static_cast(frame.GetPayloadType()), + mimeType, + CaptureTimeUs(frame), + static_cast(sequenceNumber.has_value() ? *sequenceNumber : -1), + static_cast(audioLevel.has_value() ? *audioLevel : -1), + csrcs); + } + + RTCEncodedFrameTransformer::JavaEncodedFrameClass::JavaEncodedFrameClass(JNIEnv * env) + { + cls = FindClass(env, PKG"RTCEncodedFrame"); + + dispatch = GetStaticMethod(env, cls, "dispatch", + "(L" PKG "RTCEncodedFrameTransformer;L" PKG "RTCEncodedFrame;)Z"); + } + + RTCEncodedFrameTransformer::JavaEncodedVideoFrameClass::JavaEncodedVideoFrameClass(JNIEnv * env) + { + cls = FindClass(env, PKG"RTCEncodedVideoFrame"); + + ctor = GetMethod(env, cls, "", kVideoFrameCtorSig); + } + + RTCEncodedFrameTransformer::JavaEncodedAudioFrameClass::JavaEncodedAudioFrameClass(JNIEnv * env) + { + cls = FindClass(env, PKG"RTCEncodedAudioFrame"); + + ctor = GetMethod(env, cls, "", kAudioFrameCtorSig); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedAudioFrame.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedAudioFrame.java new file mode 100644 index 00000000..8e3f5f77 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedAudioFrame.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +/** + * An encoded audio frame, as an {@link RTCEncodedFrameTransformer} sees it. + * + * @author Alex Andres + */ +public class RTCEncodedAudioFrame extends RTCEncodedFrame { + + private static final long[] NO_SOURCES = new long[0]; + + private final int sequenceNumber; + + private final int audioLevel; + + private final int[] contributingSources; + + + /** + * Constructor to be used by the native api. + */ + RTCEncodedAudioFrame(long handle, int size, long timestamp, long ssrc, + int payloadType, String mimeType, long captureTimeUs, + int sequenceNumber, int audioLevel, int[] contributingSources) { + super(handle, size, timestamp, ssrc, payloadType, mimeType, captureTimeUs); + + this.sequenceNumber = sequenceNumber; + this.audioLevel = audioLevel; + this.contributingSources = contributingSources; + } + + /** + * @return The RTP sequence number of a received frame, or -1 for a frame + * about to be sent, which has none yet. + */ + public int getSequenceNumber() { + return sequenceNumber; + } + + /** + * Returns the audio level of this frame, in -dBov: 0 is the loudest, 127 + * digital silence. Received frames carry it only if the audio level + * header extension is in use. + * + * @return The audio level, or -1 if unknown. + */ + public int getAudioLevel() { + return audioLevel; + } + + /** + * @return The CSRCs of the sources mixed into this frame, as unsigned + * 32-bit values; empty if none were. + */ + public long[] getContributingSources() { + if (contributingSources == null) { + return NO_SOURCES; + } + + long[] sources = new long[contributingSources.length]; + + for (int i = 0; i < sources.length; i++) { + sources[i] = Integer.toUnsignedLong(contributingSources[i]); + } + + return sources; + } + + @Override + public String toString() { + return String.format("%s@%d [mimeType=%s, size=%d, timestamp=%d, ssrc=%d]", + getClass().getSimpleName(), hashCode(), getMimeType(), + getSize(), getTimestamp(), getSsrc()); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedFrame.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedFrame.java new file mode 100644 index 00000000..1b98ac8e --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedFrame.java @@ -0,0 +1,339 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * One encoded audio or video frame, as an {@link RTCEncodedFrameTransformer} + * sees it: the encoded payload and what WebRTC knows about it. + *

+ * A frame is backed by the native frame WebRTC is about to send or decode, + * and is valid only while the transform it was handed to runs, and only on + * that transform's thread. Reading or changing the payload after that throws + * an {@link IllegalStateException}; the metadata stays readable. + *

+ * The payload is not copied into Java unless asked for: a transform that only + * looks at the metadata, or that {@link #setData(ByteBuffer) replaces} the + * payload without reading it, costs no copy at all. + * + * @author Alex Andres + * + * @see RTCEncodedVideoFrame + * @see RTCEncodedAudioFrame + */ +public abstract class RTCEncodedFrame { + + /** + * Where the payload is copied to. There is one per transform thread, which + * grows to the largest frame seen and is reused for every frame after, so + * reading the payload allocates nothing. + */ + private static final ThreadLocal SCRATCH = new ThreadLocal<>(); + + /** The smallest scratch buffer, enough for any audio frame. */ + private static final int MIN_SCRATCH_CAPACITY = 4096; + + private final Thread owner; + + private final long timestamp; + + private final long ssrc; + + private final int payloadType; + + private final String mimeType; + + private final long captureTimeUs; + + /** The native frame, or 0 once the transform has returned. */ + private long handle; + + private int size; + + /** A copy of the current payload, or null if none was made yet. */ + private ByteBuffer data; + + private boolean dropped; + + + RTCEncodedFrame(long handle, int size, long timestamp, long ssrc, + int payloadType, String mimeType, long captureTimeUs) { + this.owner = Thread.currentThread(); + this.handle = handle; + this.size = size; + this.timestamp = timestamp; + this.ssrc = ssrc; + this.payloadType = payloadType; + this.mimeType = mimeType; + this.captureTimeUs = captureTimeUs; + } + + /** + * Returns the encoded payload, from position zero to its size. + *

+ * The buffer holds a copy: changing it changes nothing until it is passed + * to {@link #setData(ByteBuffer)}. Its memory is reused for the frames + * that follow, so it must not be kept beyond the transform; copy what is + * needed later. + * + * @return The payload of this frame. + * + * @throws IllegalStateException If the transform has returned, or if + * called from another thread. + */ + public ByteBuffer getData() { + checkAccess(); + + if (data == null) { + ByteBuffer scratch = scratch(size); + + if (size > 0) { + copyData(handle, scratch); + } + + data = scratch; + } + + ByteBuffer view = data.duplicate(); + view.clear(); + view.limit(size); + + return view; + } + + /** + * Replaces the payload with the remaining bytes of the given buffer. The + * bytes are copied, and the buffer's position is left unchanged. + * + * @param data The new payload. + * + * @throws IllegalStateException If the transform has returned, or if + * called from another thread. + */ + public void setData(ByteBuffer data) { + Objects.requireNonNull(data, "Data is null"); + checkAccess(); + + int length = data.remaining(); + + if (data.isDirect()) { + setDataBuffer(handle, data, data.position(), length); + } + else if (data.hasArray()) { + setDataArray(handle, data.array(), data.arrayOffset() + data.position(), length); + } + else { + // A read-only heap buffer exposes no array. + byte[] copy = new byte[length]; + data.duplicate().get(copy); + + setDataArray(handle, copy, 0, length); + } + + payloadChanged(length); + } + + /** + * Replaces the payload with the given bytes, which are copied. + * + * @param data The new payload. + * + * @throws IllegalStateException If the transform has returned, or if + * called from another thread. + */ + public void setData(byte[] data) { + Objects.requireNonNull(data, "Data is null"); + + setData(data, 0, data.length); + } + + /** + * Replaces the payload with a range of the given bytes, which are copied. + * + * @param data The array holding the new payload. + * @param offset Where the payload starts in the array. + * @param length The payload size in bytes. + * + * @throws IndexOutOfBoundsException If the range lies outside the array. + * @throws IllegalStateException If the transform has returned, or if + * called from another thread. + */ + public void setData(byte[] data, int offset, int length) { + Objects.requireNonNull(data, "Data is null"); + + if (offset < 0 || length < 0 || offset > data.length - length) { + throw new IndexOutOfBoundsException(String.format( + "Range [%d, %d) out of bounds for length %d", + offset, offset + length, data.length)); + } + + checkAccess(); + + setDataArray(handle, data, offset, length); + + payloadChanged(length); + } + + /** + * Drops this frame: it is released instead of being sent on. A receiver + * that misses a video frame cannot decode what depends on it, so dropping + * video frames usually means waiting for the next key frame. + * + * @throws IllegalStateException If the transform has returned, or if + * called from another thread. + */ + public void drop() { + checkAccess(); + + dropped = true; + } + + /** + * @return True if this frame was {@link #drop() dropped}. + */ + public boolean isDropped() { + return dropped; + } + + /** + * @return The current payload size in bytes. + */ + public int getSize() { + return size; + } + + /** + * @return The RTP timestamp of this frame, an unsigned 32-bit value in + * the clock rate of the codec. + */ + public long getTimestamp() { + return timestamp; + } + + /** + * @return The SSRC of the RTP stream this frame belongs to, an unsigned + * 32-bit value. + */ + public long getSsrc() { + return ssrc; + } + + /** + * @return The RTP payload type of this frame. + */ + public int getPayloadType() { + return payloadType; + } + + /** + * @return The codec of this frame as a MIME type, e.g. {@code video/VP8} + * or {@code audio/opus}. + */ + public String getMimeType() { + return mimeType; + } + + /** + * Returns the time the frame was captured, in microseconds. A sender's + * frames carry the local capture time. A receiver's carry the capture + * time of the remote capturer, relative to the NTP epoch, and only if the + * absolute capture time header extension is in use. + * + * @return The capture time in microseconds, or -1 if unknown. + */ + public long getCaptureTimeUs() { + return captureTimeUs; + } + + /** + * Runs the transform on the frame, on behalf of native code, and says + * whether the frame is to be sent on. Whatever happens, the frame is no + * longer usable afterwards, which is what makes it safe for native code to + * move on with the native frame. + */ + static boolean dispatch(RTCEncodedFrameTransformer transformer, RTCEncodedFrame frame) { + try { + transformer.transform(frame); + + return !frame.dropped; + } + catch (Throwable e) { + // Reported the way an exception on any other thread would be, + // which honors whatever handler the application installed. + Thread thread = Thread.currentThread(); + + try { + thread.getUncaughtExceptionHandler().uncaughtException(thread, e); + } + catch (Throwable ignored) { + // Nothing more can be done about it here. + } + + return false; + } + finally { + frame.handle = 0; + frame.data = null; + } + } + + private void payloadChanged(int length) { + size = length; + data = null; + } + + private void checkAccess() { + if (handle == 0) { + throw new IllegalStateException( + "An encoded frame is only valid while its transform runs"); + } + if (Thread.currentThread() != owner) { + throw new IllegalStateException( + "An encoded frame is only valid on the thread running its transform"); + } + } + + private static ByteBuffer scratch(int size) { + ByteBuffer scratch = SCRATCH.get(); + + if (scratch == null || scratch.capacity() < size) { + int capacity = MIN_SCRATCH_CAPACITY; + + if (scratch != null) { + capacity = Math.max(capacity, scratch.capacity()); + } + while (capacity < size) { + capacity = capacity > Integer.MAX_VALUE / 2 ? Integer.MAX_VALUE : capacity * 2; + } + + scratch = ByteBuffer.allocateDirect(capacity); + + SCRATCH.set(scratch); + } + + return scratch; + } + + private static native void copyData(long handle, ByteBuffer target); + + private static native void setDataBuffer(long handle, ByteBuffer source, int offset, int length); + + private static native void setDataArray(long handle, byte[] source, int offset, int length); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedFrameTransformer.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedFrameTransformer.java new file mode 100644 index 00000000..ab0b5623 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedFrameTransformer.java @@ -0,0 +1,69 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +/** + * Transforms the encoded frames of an {@link RTCRtpSender} or an {@link + * RTCRtpReceiver}, the way WebRTC Encoded Transforms (insertable streams) do + * in a browser. A sender's transform sees every frame after the encoder and + * before the packetizer, a receiver's after the depacketizer and before the + * decoder. This is where end-to-end encryption, frame metadata or custom + * frame analysis go. + *

+ * A frame the transform returns from is sent on, with whatever changes were + * made to it, unless the transform {@link RTCEncodedFrame#drop() dropped} it. + * If the transform throws, the frame is dropped: sending it on unchanged + * could send in the clear what an encryption transform was meant to protect. + *

+ * The transform runs on a thread of its own, one per sender or receiver, and + * never on a thread that carries media, so a slow transform delays only its + * own frames and may safely call back into the peer connection. Frames arrive + * in order, one at a time. A transform that falls several seconds behind has + * frames dropped until it catches up. + *

+ * Example, a (deliberately trivial) encryption of the payload: + * 

{@code
+ * sender.setTransform(frame -> {
+ *     ByteBuffer data = frame.getData();
+ *
+ *     for (int i = 0; i < data.limit(); i++) {
+ *         data.put(i, (byte) (data.get(i) ^ 0x5A));
+ *     }
+ *
+ *     frame.setData(data);
+ * });
+ * }
+ * + * @author Alex Andres + * + * @see RTCRtpSender#setTransform(RTCEncodedFrameTransformer) + * @see RTCRtpReceiver#setTransform(RTCEncodedFrameTransformer) + */ +@FunctionalInterface +public interface RTCEncodedFrameTransformer { + + /** + * Transforms one encoded frame in place. The frame, and any buffer it + * returned, is valid only until this method returns, and only on the + * thread that called it. + * + * @param frame The frame to transform, an {@link RTCEncodedVideoFrame} or + * an {@link RTCEncodedAudioFrame}. + */ + void transform(RTCEncodedFrame frame); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedVideoFrame.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedVideoFrame.java new file mode 100644 index 00000000..42b069dc --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCEncodedVideoFrame.java @@ -0,0 +1,119 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +/** + * An encoded video frame, as an {@link RTCEncodedFrameTransformer} sees it. + * + * @author Alex Andres + */ +public class RTCEncodedVideoFrame extends RTCEncodedFrame { + + private final boolean keyFrame; + + private final int width; + + private final int height; + + private final String rid; + + private final long frameId; + + private final int spatialIndex; + + private final int temporalIndex; + + + /** + * Constructor to be used by the native api. + */ + RTCEncodedVideoFrame(long handle, int size, long timestamp, long ssrc, + int payloadType, String mimeType, long captureTimeUs, + boolean keyFrame, int width, int height, String rid, long frameId, + int spatialIndex, int temporalIndex) { + super(handle, size, timestamp, ssrc, payloadType, mimeType, captureTimeUs); + + this.keyFrame = keyFrame; + this.width = width; + this.height = height; + this.rid = rid; + this.frameId = frameId; + this.spatialIndex = spatialIndex; + this.temporalIndex = temporalIndex; + } + + /** + * @return True if this frame can be decoded without any frame before it. + */ + public boolean isKeyFrame() { + return keyFrame; + } + + /** + * @return The frame width in pixels, or 0 if the codec does not say for + * this frame, which is common for frames other than key frames. + */ + public int getWidth() { + return width; + } + + /** + * @return The frame height in pixels, or 0 if the codec does not say for + * this frame, which is common for frames other than key frames. + */ + public int getHeight() { + return height; + } + + /** + * @return The RTP stream ID of the simulcast layer this frame belongs to, + * or {@code null} without simulcast. + */ + public String getRid() { + return rid; + } + + /** + * @return The frame ID from the dependency descriptor, or -1 if there is + * none. + */ + public long getFrameId() { + return frameId; + } + + /** + * @return The spatial layer of this frame, for scalable codecs. + */ + public int getSpatialIndex() { + return spatialIndex; + } + + /** + * @return The temporal layer of this frame, for scalable codecs. + */ + public int getTemporalIndex() { + return temporalIndex; + } + + @Override + public String toString() { + return String.format("%s@%d [mimeType=%s, keyFrame=%s, size=%d, %dx%d, timestamp=%d, ssrc=%d]", + getClass().getSimpleName(), hashCode(), getMimeType(), keyFrame, + getSize(), width, height, getTimestamp(), getSsrc()); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiver.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiver.java index 03778a79..7e120f76 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiver.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiver.java @@ -83,6 +83,30 @@ private RTCRtpReceiver() { */ public native List getSynchronizationSources(); + /** + * Sets the transform that every encoded frame of this receiver passes + * through between the depacketizer and the decoder, replacing the one set + * before. A transform of {@code null} removes it, after which frames pass + * unchanged. + *

+ * The transform belongs to the native receiver, so it is shared by every + * RTCRtpReceiver instance standing for it, and it stays in place when + * this instance is disposed. + * + * @param transformer The transform, or {@code null} to remove it. + * + * @see RTCEncodedFrameTransformer + */ + public native void setTransform(RTCEncodedFrameTransformer transformer); + + /** + * Asks the remote sender for a video key frame, e.g. so that a recording + * can start decoding at once rather than wait for the sender to send one + * of its own accord, which WebRTC senders rarely do. Does nothing for an + * audio receiver. + */ + public native void requestKeyFrame(); + /** * Releases the native reference held by this RTCRtpReceiver instance. *

diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSender.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSender.java index 155bb896..b30c010e 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSender.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpSender.java @@ -112,6 +112,34 @@ private RTCRtpSender() { */ public native RTCDtmfSender getDtmfSender(); + /** + * Sets the transform that every encoded frame of this sender passes + * through between the encoder and the packetizer, replacing the one set + * before. A transform of {@code null} removes it, after which frames pass + * unchanged. + *

+ * The transform belongs to the native sender, so it is shared by every + * RTCRtpSender instance standing for it, and it stays in place when this + * instance is disposed. Setting it before the connection is negotiated + * avoids the key frame the encoder otherwise sends when the first + * transform is set on a running video sender. + * + * @param transformer The transform, or {@code null} to remove it. + * + * @see RTCEncodedFrameTransformer + */ + public native void setTransform(RTCEncodedFrameTransformer transformer); + + /** + * Asks the encoder to make the next video frame a key frame, e.g. so that + * a newly joined receiver or a recording can start decoding at once. Does + * nothing for an audio sender. + * + * @throws RuntimeException If the sender has no encoder yet, before the + * connection is negotiated. + */ + public native void generateKeyFrame(); + /** * Releases the native reference held by this RTCRtpSender instance. *

diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeApi.java b/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeApi.java index b6cb7dd7..93b0a576 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeApi.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeApi.java @@ -18,6 +18,9 @@ import java.util.Objects; +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.RTCRtpSender; + /** * The entry point a native extension library uses to reach this library's * native side directly, without going through Java for every frame. @@ -88,4 +91,45 @@ public static long handleOf(NativeObject object) { return object.getNativeHandle(); } + /** + * Returns a handle through which a native extension observes the encoded + * frames of the given sender, as they leave the encoder and before any + * {@code RTCEncodedFrameTransformer} runs on them. The extension attaches + * to it with {@code encoded_observer_add()} of the function table. + *

+ * The handle carries a reference, which the extension must drop with + * {@code encoded_frames_release()} once it has attached its observers, + * or once it decides not to. + * + * @param sender The sender whose frames are to be observed. + * + * @return The handle, or {@code 0} if the sender was disposed. + * + * @throws NullPointerException If the sender is {@code null}. + */ + public static long encodedFramesOf(RTCRtpSender sender) { + return senderFrames(handleOf(sender)); + } + + /** + * Returns a handle through which a native extension observes the encoded + * frames of the given receiver, as they go to the decoder and after any + * {@code RTCEncodedFrameTransformer} ran on them. See {@link + * #encodedFramesOf(RTCRtpSender)} for how the handle is used and + * released. + * + * @param receiver The receiver whose frames are to be observed. + * + * @return The handle, or {@code 0} if the receiver was disposed. + * + * @throws NullPointerException If the receiver is {@code null}. + */ + public static long encodedFramesOf(RTCRtpReceiver receiver) { + return receiverFrames(handleOf(receiver)); + } + + private static native long senderFrames(long senderHandle); + + private static native long receiverFrames(long receiverHandle); + } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/RTCEncodedFrameTransformTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/RTCEncodedFrameTransformTests.java new file mode 100644 index 00000000..39fd24ba --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/RTCEncodedFrameTransformTests.java @@ -0,0 +1,372 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.nio.ByteBuffer; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests encoded frame transforms on a call between two local peer + * connections. + */ +@Execution(ExecutionMode.SAME_THREAD) +class RTCEncodedFrameTransformTests extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + + @Test + void encryptedVideoRoundTrips() throws Exception { + AtomicInteger encrypted = new AtomicInteger(); + AtomicInteger decrypted = new AtomicInteger(); + AtomicReference mimeType = new AtomicReference<>(); + CountDownLatch decoded = new CountDownLatch(10); + + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.getVideoSender().setTransform(frame -> { + mimeType.compareAndSet(null, frame.getMimeType()); + xor(frame); + encrypted.incrementAndGet(); + }); + + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + receiver.setTransform(frame -> { + xor(frame); + decrypted.incrementAndGet(); + }); + + // Frames only decode if the receiver undid what the sender did. + // The remote track belongs to the receiver and is not disposed. + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + decoded.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(decoded.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), + "too few frames decoded: encrypted " + encrypted.get() + + ", decrypted " + decrypted.get()); + + assertTrue(mimeType.get().startsWith("video/"), mimeType.get()); + assertTrue(decrypted.get() > 0); + + track.removeSink(sink); + receiver.dispose(); + } + } + + @Test + void videoFramesCarryMetadata() throws Exception { + AtomicReference first = new AtomicReference<>(); + AtomicReference firstSize = new AtomicReference<>(); + CountDownLatch seen = new CountDownLatch(1); + + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.getVideoSender().setTransform(frame -> { + if (first.compareAndSet(null, (RTCEncodedVideoFrame) frame)) { + firstSize.set(frame.getData().remaining()); + seen.countDown(); + } + }); + + call.negotiate(); + call.awaitConnected(); + call.startMedia(); + + assertTrue(seen.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + RTCEncodedVideoFrame frame = first.get(); + + // A stream starts with a key frame, which says how large it is. + assertTrue(frame.isKeyFrame()); + assertEquals(320, frame.getWidth()); + assertEquals(240, frame.getHeight()); + assertEquals(frame.getSize(), (int) firstSize.get()); + assertTrue(frame.getSize() > 0); + assertTrue(frame.getSsrc() > 0); + assertTrue(frame.getPayloadType() > 0); + } + } + + @Test + void frameIsInvalidAfterTransform() throws Exception { + AtomicReference kept = new AtomicReference<>(); + CountDownLatch seen = new CountDownLatch(1); + + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.getVideoSender().setTransform(frame -> { + if (kept.compareAndSet(null, frame)) { + seen.countDown(); + } + }); + + call.negotiate(); + call.awaitConnected(); + call.startMedia(); + + assertTrue(seen.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + RTCEncodedFrame frame = kept.get(); + + // Waits for the transform to have returned from this frame. + Thread.sleep(200); + + assertThrows(IllegalStateException.class, frame::getData); + assertThrows(IllegalStateException.class, () -> frame.setData(new byte[1])); + assertThrows(IllegalStateException.class, frame::drop); + + // The metadata is a copy, and stays. + assertNotNull(frame.getMimeType()); + } + } + + @Test + void droppedFramesNeverArrive() throws Exception { + AtomicInteger dropped = new AtomicInteger(); + AtomicInteger received = new AtomicInteger(); + + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.getVideoSender().setTransform(frame -> { + frame.drop(); + dropped.incrementAndGet(); + }); + + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + receiver.setTransform(frame -> received.incrementAndGet()); + + call.awaitConnected(); + call.startMedia(); + + waitFor(() -> dropped.get() >= 30); + Thread.sleep(500); + + assertEquals(0, received.get()); + + // Without a transform, frames flow again. + call.getVideoSender().setTransform(null); + + waitFor(() -> received.get() > 0); + + receiver.dispose(); + } + } + + @Test + void throwingTransformDropsFrames() throws Exception { + AtomicInteger thrown = new AtomicInteger(); + AtomicInteger received = new AtomicInteger(); + Thread.UncaughtExceptionHandler previous = Thread.getDefaultUncaughtExceptionHandler(); + + Thread.setDefaultUncaughtExceptionHandler((thread, e) -> { + if (e instanceof IllegalArgumentException) { + thrown.incrementAndGet(); + } + }); + + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.getVideoSender().setTransform(frame -> { + throw new IllegalArgumentException("test"); + }); + + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + receiver.setTransform(frame -> received.incrementAndGet()); + + call.awaitConnected(); + call.startMedia(); + + waitFor(() -> thrown.get() >= 30); + + assertEquals(0, received.get()); + + receiver.dispose(); + } + finally { + Thread.setDefaultUncaughtExceptionHandler(previous); + } + } + + @Test + void audioFramesAreTransformed() throws Exception { + AtomicReference first = new AtomicReference<>(); + AtomicInteger received = new AtomicInteger(); + + try (TestMediaCall call = new TestMediaCall(factory, false, true)) { + call.getAudioSender().setTransform(frame -> { + first.compareAndSet(null, frame); + xor(frame); + }); + + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("audio"); + receiver.setTransform(frame -> { + xor(frame); + received.incrementAndGet(); + }); + + call.awaitConnected(); + call.startMedia(); + + waitFor(() -> received.get() >= 50); + + assertTrue(first.get() instanceof RTCEncodedAudioFrame); + assertEquals("audio/opus", first.get().getMimeType().toLowerCase()); + + receiver.dispose(); + } + } + + @Test + void requestKeyFrameReachesSender() throws Exception { + AtomicInteger keyFrames = new AtomicInteger(); + AtomicInteger frames = new AtomicInteger(); + + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.getVideoSender().setTransform(frame -> { + frames.incrementAndGet(); + + if (((RTCEncodedVideoFrame) frame).isKeyFrame()) { + keyFrames.incrementAndGet(); + } + }); + + call.negotiate(); + call.awaitConnected(); + call.startMedia(); + + waitFor(() -> frames.get() >= 30); + + int before = keyFrames.get(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + receiver.requestKeyFrame(); + + waitFor(() -> keyFrames.get() > before); + + int afterRequest = keyFrames.get(); + + call.getVideoSender().generateKeyFrame(); + + waitFor(() -> keyFrames.get() > afterRequest); + + receiver.dispose(); + } + } + + @Test + void clearingUnsetTransformDoesNothing() { + try (TestMediaCall call = new TestMediaCall(factory, true, true)) { + call.getVideoSender().setTransform(null); + call.getAudioSender().setTransform(null); + + // Audio has no key frames; asking for one is not an error. + call.getAudioSender().generateKeyFrame(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Test + void transformIsSharedBetweenInstances() throws Exception { + AtomicInteger first = new AtomicInteger(); + AtomicInteger second = new AtomicInteger(); + + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + RTCRtpSender videoSender = call.getVideoSender(); + + call.getVideoSender().setTransform(frame -> first.incrementAndGet()); + + call.negotiate(); + call.awaitConnected(); + call.startMedia(); + + waitFor(() -> first.get() > 0); + + // Another instance for the same native sender replaces the + // transform, rather than installing a second one. + for (RTCRtpSender sender : call.getCallerSenders()) { + if (sender.equals(videoSender)) { + sender.setTransform(frame -> second.incrementAndGet()); + } + sender.dispose(); + } + + int firstCount = first.get(); + + waitFor(() -> second.get() > 0); + Thread.sleep(200); + + assertTrue(first.get() - firstCount < 10, "the old transform still runs"); + } + } + + private static void xor(RTCEncodedFrame frame) { + ByteBuffer data = frame.getData(); + + for (int i = 0; i < data.limit(); i++) { + data.put(i, (byte) (data.get(i) ^ 0x5A)); + } + + frame.setData(data); + } + + private static void waitFor(Condition condition) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (!condition.met()) { + if (System.nanoTime() > deadline) { + fail("timed out waiting"); + } + + Thread.sleep(20); + } + } + + @FunctionalInterface + private interface Condition { + + boolean met(); + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java new file mode 100644 index 00000000..aa5987b3 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java @@ -0,0 +1,234 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import java.nio.ByteBuffer; +import java.util.Collections; + +import dev.onvoid.webrtc.media.MediaStreamTrack; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoTrack; + +/** + * A call between two local peer connections that sends generated video and + * audio from the caller to the callee, for tests that need media flowing. + *

+ * The call is set up in steps, so that a test can prepare the senders before + * anything is negotiated and the receivers before media flows: construct it, + * {@link #negotiate()}, {@link #awaitConnected()}, then {@link #startMedia()}. + */ +class TestMediaCall implements AutoCloseable { + + private static final int WIDTH = 320; + private static final int HEIGHT = 240; + + private final CustomVideoSource videoSource; + private final CustomAudioSource audioSource; + private final VideoTrack videoTrack; + private final AudioTrack audioTrack; + + private final TestPeerConnection caller; + private final TestPeerConnection callee; + + private final RTCRtpSender videoSender; + private final RTCRtpSender audioSender; + + private volatile boolean feeding; + private Thread feeder; + + + TestMediaCall(PeerConnectionFactory factory, boolean video, boolean audio) { + caller = new TestPeerConnection(factory); + callee = new TestPeerConnection(factory); + + caller.setRemotePeerConnection(callee); + callee.setRemotePeerConnection(caller); + + if (video) { + videoSource = new CustomVideoSource(); + videoTrack = factory.createVideoTrack("video", videoSource); + videoSender = caller.getPeerConnection().addTrack(videoTrack, + Collections.singletonList("stream")); + } + else { + videoSource = null; + videoTrack = null; + videoSender = null; + } + + if (audio) { + audioSource = new CustomAudioSource(); + audioTrack = factory.createAudioTrack("audio", audioSource); + audioSender = caller.getPeerConnection().addTrack(audioTrack, + Collections.singletonList("stream")); + } + else { + audioSource = null; + audioTrack = null; + audioSender = null; + } + } + + /** + * Negotiates the call. Once this returns, the callee has its receivers, + * though media only starts with {@link #startMedia()}. + */ + void negotiate() throws Exception { + callee.setRemoteDescription(caller.createOffer()); + caller.setRemoteDescription(callee.createAnswer()); + } + + /** + * Waits for both ends to connect. + */ + void awaitConnected() throws InterruptedException { + caller.waitUntilConnected(); + callee.waitUntilConnected(); + } + + /** + * Starts pushing video at 30 frames and audio in 10 ms chunks per second. + */ + void startMedia() { + feeding = true; + feeder = new Thread(this::feed, "TestMediaCall-feeder"); + feeder.setDaemon(true); + feeder.start(); + } + + RTCRtpSender getVideoSender() { + return videoSender; + } + + RTCRtpSender getAudioSender() { + return audioSender; + } + + /** + * Returns new instances of all of the caller's senders, which are the + * caller's to dispose. + */ + RTCRtpSender[] getCallerSenders() { + return caller.getPeerConnection().getSenders(); + } + + /** + * Returns the callee's receiver of the given kind. The instance is the + * caller's to dispose. + */ + RTCRtpReceiver getReceiver(String kind) { + boolean video = MediaStreamTrack.VIDEO_TRACK_KIND.equals(kind); + + if (video ? videoSender == null : audioSender == null) { + throw new IllegalStateException("The call has no " + kind); + } + + // The callee's transceivers follow the order of the offer, which has + // video first. A remote track belongs to its receiver, so its kind is + // not asked for here: disposing of it again would not be allowed. + int index = video || videoSender == null ? 0 : 1; + + RTCRtpTransceiver[] transceivers = callee.getPeerConnection().getTransceivers(); + RTCRtpReceiver receiver = transceivers[index].getReceiver(); + + for (RTCRtpTransceiver transceiver : transceivers) { + transceiver.dispose(); + } + + return receiver; + } + + @Override + public void close() throws InterruptedException { + feeding = false; + + if (feeder != null) { + feeder.join(); + } + + if (videoSender != null) { + videoSender.dispose(); + } + if (audioSender != null) { + audioSender.dispose(); + } + + caller.close(); + callee.close(); + + if (videoTrack != null) { + videoTrack.dispose(); + videoSource.dispose(); + } + if (audioTrack != null) { + audioTrack.dispose(); + audioSource.dispose(); + } + } + + private void feed() { + byte[] audio = new byte[480 * 2]; + long startNs = System.nanoTime(); + long audioChunks = 0; + long videoFrames = 0; + int shade = 0; + + while (feeding) { + long elapsedMs = (System.nanoTime() - startNs) / 1_000_000; + + while (audioSource != null && audioChunks * 10 <= elapsedMs) { + // A tone, so that the encoder has something to encode. + for (int i = 0; i < 480; i++) { + short sample = (short) (Math.sin(2 * Math.PI * 440 * (audioChunks * 480 + i) / 48000) * 8000); + audio[2 * i] = (byte) sample; + audio[2 * i + 1] = (byte) (sample >> 8); + } + + audioSource.pushAudio(audio, 16, 48000, 1, 480); + audioChunks++; + } + + if (videoSource != null && videoFrames * 1000 / 30 <= elapsedMs) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(WIDTH, HEIGHT); + ByteBuffer y = buffer.getDataY(); + + // A moving gradient, so that frames differ from each other. + for (int i = 0; i < y.capacity(); i++) { + y.put(i, (byte) (i + shade)); + } + shade += 3; + + VideoFrame frame = new VideoFrame(buffer, 0); + videoSource.pushFrame(frame); + frame.release(); + videoFrames++; + } + + try { + Thread.sleep(2); + } + catch (InterruptedException e) { + return; + } + } + } + +} From 6d45f0b1a40383692538ec48dceb4b266a213245 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 27 Sep 2026 16:15:22 +0200 Subject: [PATCH 2/3] docs: advertise call recording and encoded transforms --- README.md | 2 ++ docs/index.md | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 43dc8f77..827525c1 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ The library provides a comprehensive set of Java classes that map to the WebRTC - **Media capabilities** - Audio and video capture from cameras and microphones - **Desktop capture** - Screen and application window sharing - **Media file playback** - Send video and audio files over a peer connection in place of a camera and microphone, with the optional FFmpeg-based `webrtc-java-media` module +- **Call recording** - Record what a peer connection sends or receives into MKV, WebM or MP4 files, without re-encoding, with the `webrtc-java-media` module +- **Encoded transforms** - Read, change or drop encoded audio and video frames on their way through a sender or receiver, e.g. for end-to-end encryption, like insertable streams in the browser - **Data channels** - Bidirectional peer-to-peer data exchange - **Statistics API** - Detailed metrics for monitoring connection quality - **Simple integration** - Available as a Maven dependency diff --git a/docs/index.md b/docs/index.md index 3e3d6dc6..1dd4739c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,6 +28,10 @@ features: details: Audio and video capture from cameras and microphones devices, with support for custom media sources for flexible streaming solutions. - title: Media File Playback details: Send video and audio files over a peer connection. The optional media module decodes with FFmpeg in native code and paces playback in real time, keeping audio and video in sync. + - title: Call Recording + details: Record what a peer connection sends or receives into MKV, WebM or MP4 files. Encoded frames go into the file as they are, without re-encoding, so recording costs next to no CPU and keeps the exact quality of the call. + - title: End-to-End Encryption + details: Encoded transforms let Java code read, change or drop every encoded frame between encoder and network, like insertable streams in the browser; the building block for end-to-end encryption, frame metadata and stream analysis. - title: Screen Sharing details: Share application windows or the full desktop with minimal setup; integrate screen capture streams like any other media stream. - title: Data Channels From cf7c87003fb6eaa3015776311861ae77bd09fd55 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 27 Sep 2026 16:39:30 +0200 Subject: [PATCH 3/3] test: prefer VP8 in the transform and recorder test calls macOS prefers H.264 through VideoToolbox by default. A transform that changes the whole payload breaks H.264, whose packetizer splits frames at start codes in it, so nothing reached the receiver there; the recorder tests check for VP8 as well. VideoToolbox on CI runners also encodes far fewer frames, so the tests wait for fewer. --- .../webrtc/media/recorder/TestCall.java | 32 +++++++++++++++++++ .../webrtc/RTCEncodedFrameTransformTests.java | 10 +++--- .../java/dev/onvoid/webrtc/TestMediaCall.java | 32 +++++++++++++++++++ 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java index d105e4cf..e282e797 100644 --- a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java @@ -17,7 +17,10 @@ package dev.onvoid.webrtc.media.recorder; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -31,11 +34,13 @@ import dev.onvoid.webrtc.RTCOfferOptions; import dev.onvoid.webrtc.RTCPeerConnection; import dev.onvoid.webrtc.RTCPeerConnectionState; +import dev.onvoid.webrtc.RTCRtpCodecCapability; import dev.onvoid.webrtc.RTCRtpReceiver; import dev.onvoid.webrtc.RTCRtpSender; import dev.onvoid.webrtc.RTCRtpTransceiver; import dev.onvoid.webrtc.RTCSessionDescription; import dev.onvoid.webrtc.SetSessionDescriptionObserver; +import dev.onvoid.webrtc.media.MediaType; import dev.onvoid.webrtc.media.audio.AudioTrack; import dev.onvoid.webrtc.media.audio.CustomAudioSource; import dev.onvoid.webrtc.media.video.CustomVideoSource; @@ -81,6 +86,8 @@ class TestCall implements AutoCloseable { videoSender = caller.connection.addTrack(videoTrack, Collections.singletonList("stream")); audioSender = caller.connection.addTrack(audioTrack, Collections.singletonList("stream")); + preferVp8(factory, caller.connection, videoSender); + RTCSessionDescription offer = caller.createOffer(); callee.setRemoteDescription(offer); RTCSessionDescription answer = callee.createAnswer(); @@ -137,6 +144,31 @@ public void close() throws InterruptedException { audioSource.dispose(); } + /** + * Makes VP8 the preferred video codec. Platforms prefer different codecs by + * default, macOS H.264 through VideoToolbox, and the tests should not + * depend on which: VP8 is available everywhere, and it is what the + * recordings are checked for. + */ + private static void preferVp8(PeerConnectionFactory factory, RTCPeerConnection connection, + RTCRtpSender sender) { + List codecs = new ArrayList<>( + factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()); + + codecs.sort(Comparator.comparing(codec -> !"VP8".equalsIgnoreCase(codec.getName()))); + + for (RTCRtpTransceiver transceiver : connection.getTransceivers()) { + RTCRtpSender transceiverSender = transceiver.getSender(); + + if (transceiverSender.equals(sender)) { + transceiver.setCodecPreferences(codecs); + } + + transceiverSender.dispose(); + transceiver.dispose(); + } + } + private void feed() { byte[] audio = new byte[480 * 2]; long startNs = System.nanoTime(); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/RTCEncodedFrameTransformTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/RTCEncodedFrameTransformTests.java index 39fd24ba..bc46cba9 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/RTCEncodedFrameTransformTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/RTCEncodedFrameTransformTests.java @@ -80,8 +80,8 @@ void encryptedVideoRoundTrips() throws Exception { call.startMedia(); assertTrue(decoded.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), - "too few frames decoded: encrypted " + encrypted.get() - + ", decrypted " + decrypted.get()); + "too few frames decoded: " + mimeType.get() + ", encrypted " + + encrypted.get() + ", decrypted " + decrypted.get()); assertTrue(mimeType.get().startsWith("video/"), mimeType.get()); assertTrue(decrypted.get() > 0); @@ -175,7 +175,7 @@ void droppedFramesNeverArrive() throws Exception { call.awaitConnected(); call.startMedia(); - waitFor(() -> dropped.get() >= 30); + waitFor(() -> dropped.get() >= 10); Thread.sleep(500); assertEquals(0, received.get()); @@ -214,7 +214,7 @@ void throwingTransformDropsFrames() throws Exception { call.awaitConnected(); call.startMedia(); - waitFor(() -> thrown.get() >= 30); + waitFor(() -> thrown.get() >= 10); assertEquals(0, received.get()); @@ -274,7 +274,7 @@ void requestKeyFrameReachesSender() throws Exception { call.awaitConnected(); call.startMedia(); - waitFor(() -> frames.get() >= 30); + waitFor(() -> frames.get() >= 10); int before = keyFrames.get(); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java index aa5987b3..a96fe56a 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java @@ -17,9 +17,13 @@ package dev.onvoid.webrtc; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; +import java.util.List; import dev.onvoid.webrtc.media.MediaStreamTrack; +import dev.onvoid.webrtc.media.MediaType; import dev.onvoid.webrtc.media.audio.AudioTrack; import dev.onvoid.webrtc.media.audio.CustomAudioSource; import dev.onvoid.webrtc.media.video.CustomVideoSource; @@ -67,6 +71,8 @@ class TestMediaCall implements AutoCloseable { videoTrack = factory.createVideoTrack("video", videoSource); videoSender = caller.getPeerConnection().addTrack(videoTrack, Collections.singletonList("stream")); + + preferVp8(factory, caller.getPeerConnection(), videoSender); } else { videoSource = null; @@ -184,6 +190,32 @@ public void close() throws InterruptedException { } } + /** + * Makes VP8 the preferred video codec. Platforms prefer different codecs by + * default, macOS H.264 through VideoToolbox, and the tests should not + * depend on which: VP8 is available everywhere, and it is the codec that + * survives a transform changing the whole payload, which H.264 does not, + * since its packetizer splits frames at start codes in the payload. + */ + private static void preferVp8(PeerConnectionFactory factory, RTCPeerConnection connection, + RTCRtpSender sender) { + List codecs = new ArrayList<>( + factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()); + + codecs.sort(Comparator.comparing(codec -> !"VP8".equalsIgnoreCase(codec.getName()))); + + for (RTCRtpTransceiver transceiver : connection.getTransceivers()) { + RTCRtpSender transceiverSender = transceiver.getSender(); + + if (transceiverSender.equals(sender)) { + transceiver.setCodecPreferences(codecs); + } + + transceiverSender.dispose(); + transceiver.dispose(); + } + } + private void feed() { byte[] audio = new byte[480 * 2]; long startNs = System.nanoTime();