diff --git a/Package.swift b/Package.swift index dec1f6d..585797c 100644 --- a/Package.swift +++ b/Package.swift @@ -13,12 +13,23 @@ var targets: [Target] = [ // the package is VoiceMeetCore and its tests. #if os(macOS) products.append(.executable(name: "VoiceMeetMac", targets: ["VoiceMeetMac"])) -targets.append(.executableTarget(name: "VoiceMeetMac", dependencies: ["VoiceMeetCore", "Record"])) +targets.append(.executableTarget(name: "VoiceMeetMac", dependencies: ["VoiceMeetCore", "Record", "Live"])) // The recorder: screens, cameras, microphones and speakers into 30-second files in a spool // (README.md, "The record"). `record-run` drives it from a terminal, for tests. targets.append(.target(name: "Record")) targets.append(.executableTarget(name: "record-run", dependencies: ["Record"])) targets.append(.testTarget(name: "RecordTests", dependencies: ["Record"])) +// Live: the Mac as a WebRTC peer, fed by the recorder's own capture (README.md, "Live"). Everything +// that touches WebRTC is here; the recorder does not import it. The library is LiveKit's build of +// WebRTC, its macOS slice, a release asset of this repository. `live-run` drives it from a +// terminal, for tests. +targets.append(.binaryTarget( + name: "LiveKitWebRTC", + url: "https://github.com/iterate/mac-app/releases/download/libwebrtc-150.7871.06/LiveKitWebRTC-macos-150.7871.06.xcframework.zip", + checksum: "8d0c0c33a1f5db7600ef2c74033a7ce815056d444976e2519f4596a2a813252c" +)) +targets.append(.target(name: "Live", dependencies: ["Record", "VoiceMeetCore", "LiveKitWebRTC"])) +targets.append(.executableTarget(name: "live-run", dependencies: ["Live", "Record", "VoiceMeetCore"])) #endif let package = Package( diff --git a/README.md b/README.md index 5bfaecc..0a8b890 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,8 @@ CLI's own sign-in; the Mac's key lends and does nothing else): | `device/app-focused` | Always while the Mac is lent: the front app or its window's title changed | `app`, `bundle`, `title` | | `device/window-read` | While **Window text** is ticked under Record: on a new app or window, a click, a pause after typing, the end of a scroll; at most every 2 s, only when the text changed | `app`, `title`, `text` (at most 20,000 characters), `trigger` | | `record/slot-stored` | While a switch under **Record** is on: once for each 30 seconds, after its files are in the mind | `t`, `tracks` (below) | +| `device/live-watch-started` | A member's page began to watch this Mac live (below, "Live") | `watcher`, `who`, `tracks` | +| `device/live-watch-ended` | The watch ended | `watcher`, `who`, `seconds`, `why` | A key event holds the key itself, so what you type goes to the mind, a password too. Keyboard and mouse is off until you turn it on. Its events are ephemeral: they reach whoever listens at that @@ -316,6 +318,44 @@ offset, ms from the slot's start], …]`, one per fragment), `picture`, and a sc `scale`; an audio track has `rate`, `channels`, `rms`, `peak` and `levels` (60 half seconds, in dBFS). +## Live: a member watches this Mac now + +A member of the mind (or a member's agent) opens the device on `record.` and presses +**Watch live**: the page shows the screens, the camera and hears the microphone of this Mac, about +a tenth of a second behind, peer to peer over WebRTC. The mind carries only the signalling: a few +ephemeral `device/live-signal` events on `/devices/`. The contract is the mind app's README, +"Live". + +- **Your choices decide.** A watch is a use of each display, the camera and the microphone under + your choice for it, as a snapshot is: Off, that track is not offered; Ask, you are asked once + for the whole watch (Touch ID); Shared, it is sent. A Mac that may show nothing answers with the + reason, and the page shows nothing. There is no switch of its own. +- **You see it.** While somebody watches, Activity shows "Watched live by " as a running act + (the pill stays at the top right), and the panel lists each watcher under **Watched live** with + an **End** button. Each watch is also on the device's log in the mind: + `device/live-watch-started { watcher, who, tracks }` and + `device/live-watch-ended { watcher, who, seconds, why }`. +- **One capture, two sinks.** The record's own capture feeds live (`CaptureSink` in + `Sources/Record`): the same frames, no copy. A source that only live wants starts with the first + watcher and stops with the last. While somebody watches, a screen captures 30 frames a second and + the record keeps its 10 by time, so its files are the same with or without a watcher. +- **What a watcher gets:** each display (`screen-`, at its own pixels), one camera + (`camera-0`, 1280×720 at most) and one sound (`mic-0`), H.264 in hardware, at most 4 watchers. + The page is told to show each picture at once, and the sound is in a stream of its own. +- **On the same network** the page connects to this Mac directly, and macOS asks once whether + iterate may find devices on the local network. From another network it goes through STUN (or + TURN, when the mind gives it), and macOS does not ask. + +A mind's agent reads and ends watches: `mac.live.state()` (each watcher, its tracks, since when) +and `mac.live.end({ watcher })`. A watch starts only from a page. + +How it is built: `Sources/Live` holds everything that touches WebRTC (LiveKit's build of WebRTC, +a binary target of `Package.swift`, carried in the bundle's `Contents/Frameworks`). `LivePeer` is +the sink and one peer connection per watcher; the signalling's rules are `LiveWatchers` in +`VoiceMeetCore`, tested on their own. `bridge/device-live.mjs` connects to the mind as you, +subscribes to `device/live-signal`, appends the Mac's signals in order, and asks the facet +`record` for the ICE servers (STUN alone when it has none). + ## Capture: show an agent what you mean Press **⌥⇧S** in any app, then talk, click, and hold **⇧** and move the mouse to draw on the @@ -425,7 +465,7 @@ script of the bridge parses. Every push to `main` publishes a version to npm ./run-on-mac.sh # build, install in /Applications, open, follow the logs ./run-on-mac.sh --build-only # build and sign .build/iterate.app only ./run-on-mac.sh --logs # follow the app's log -swift test # the state machines' tests (needs Xcode): microphone, camera, lend, banner, the record, the mark and record.set's rule +swift test # the state machines' tests (needs Xcode): microphone, camera, lend, banner, the record, the mark, record.set's rule, live npx . # the installer from this folder npx . init /tmp/mac --no-install # a copy, without building or installing it ``` @@ -442,6 +482,13 @@ permission; `--screens --cameras --mics --speakers` record the real sources, whi permissions (run it through the app's `run` method). `node bridge/record-upload.mjs /tmp/spool --name test.mac --iterate --mind
` sends a spool to a mind. +`swift build --product live-run` drives live from a terminal with the same libraries and bridge: +`.build/debug/live-run --device test.mac --node $(which node) --bridge bridge/device-live.mjs --iterate +--mind
--synthetic --seconds 90` answers every watcher of `/devices/test.mac` with the +test picture and beep (no switch is asked); `--record ` records them too; `--screens +--cameras --mics` are the real sources (through the app's `run` method), and `--loopback-only +--no-stun` keeps the media on this Mac for a watcher here. stderr has a line of statistics a second. + `.build/release/VoiceMeetMac --render-panel panel.png` draws the panel to a file without opening anything. A third word draws the lend in one phase: `live`, `reconnecting`, `stopped` or `not-allowed`; `ringing` draws a call's banner at the top. `--render-banner banner.png ring` diff --git a/Sources/Live/Bridge.swift b/Sources/Live/Bridge.swift new file mode 100644 index 0000000..25cfca0 --- /dev/null +++ b/Sources/Live/Bridge.swift @@ -0,0 +1,82 @@ +import Foundation +import VoiceMeetCore + +/// `bridge/device-live.mjs` as a child process: it carries a `LivePeer`'s signals to the mind and +/// back, as JSON lines. +/// +/// - from the bridge: `{ signal: { from, want | description | candidate | bye }, who }`, +/// `{ iceServers: [...] }`, and `{ failed }` for the log; +/// - to the bridge: `{ signal: { to, description, tracks } | { to, candidate } | { to, bye, reason } }`, +/// `{ started: { watcher, who, tracks } }` and `{ ended: { watcher, who, seconds, why } }`. +public final class LiveBridge: @unchecked Sendable { + private let process = Process() + private let input = Pipe() + private let lock = NSLock() + private var pending = "" + /// A line of the bridge for the log: `{ failed }`, `{ connected }`. + public var said: ([String: Any]) -> Void = { _ in } + public var exited: (Int32) -> Void = { _ in } + + /// `node script arguments…`; `peer` hears the watchers' signals and says its own through here. + public init(node: String, script: String, arguments: [String], peer: LivePeer) { + process.executableURL = URL(fileURLWithPath: node) + process.arguments = [script] + arguments + process.standardInput = input + let output = Pipe() + process.standardOutput = output + process.standardError = FileHandle.nullDevice + output.fileHandleForReading.readabilityHandler = { [weak self, weak peer] handle in + let data = handle.availableData + guard let self, let peer, !data.isEmpty else { return } + for line in self.lines(data) { + if let signal = line["signal"] as? [String: Any] { + peer.heard(signal, who: line["who"] as? String ?? "someone") + } else if let servers = line["iceServers"] as? [[String: Any]] { + peer.setIceServers(servers) + } else { + self.said(line) + } + } + } + process.terminationHandler = { [weak self] ended in + output.fileHandleForReading.readabilityHandler = nil + self?.exited(ended.terminationStatus) + } + peer.send = { [weak self] signal in self?.write(["signal": signal.json]) } + peer.kept = { [weak self] facts in self?.write(facts) } + } + + public var pid: Int32 { process.processIdentifier } + + public func start() throws { try process.run() } + + /// Its stdin closes: it appends what waits, then exits. + public func stop() { try? input.fileHandleForWriting.close() } + + private func write(_ line: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: line, options: [.withoutEscapingSlashes]) else { return } + lock.withLock { try? input.fileHandleForWriting.write(contentsOf: data + Data("\n".utf8)) } + } + + private func lines(_ data: Data) -> [[String: Any]] { + lock.withLock { + var lines = (pending + String(decoding: data, as: UTF8.self)).components(separatedBy: "\n") + pending = lines.removeLast() + return lines.compactMap { try? JSONSerialization.jsonObject(with: Data($0.utf8)) as? [String: Any] } + } + } +} + +extension LiveSignal { + /// The payload of `device/live-signal` without its `from: "device"` (the bridge adds it). + public var json: [String: Any] { + switch self { + case let .offer(to, sdp, tracks): return ["to": to, "description": ["type": "offer", "sdp": sdp], "tracks": tracks] + case let .candidate(to, c): + var candidate: [String: Any] = ["candidate": c.candidate, "sdpMLineIndex": c.sdpMLineIndex] + if let mid = c.sdpMid { candidate["sdpMid"] = mid } + return ["to": to, "candidate": candidate] + case let .bye(to, reason): return ["to": to, "bye": true, "reason": reason] + } + } +} diff --git a/Sources/Live/Peer.swift b/Sources/Live/Peer.swift new file mode 100644 index 0000000..3970113 --- /dev/null +++ b/Sources/Live/Peer.swift @@ -0,0 +1,329 @@ +import CoreMedia +import Foundation +import LiveKitWebRTC +import Record +import VoiceMeetCore + +/// THIS MAC AS A WEBRTC PEER (README, "Live"): a watcher's page sees its screens, its camera and +/// hears its microphone, peer to peer, while the mind carries only the signalling. It is a sink of +/// the recorder's capture (`CaptureSink`): the same frames as the record, no copy. One peer +/// connection per watcher (at most `LiveWatchers.most`), send only; the device offers. The signalling's +/// rules are `LiveWatchers` (VoiceMeetCore); this class runs their effects. +/// +/// Track ids are the record's: `screen-`, `camera-0`, `mic-0`, and `test-1`, `test-2` for +/// the synthetic source. Live shows one camera and one sound: the first camera and the first +/// microphone that deliver are `camera-0` and `mic-0`. +/// +/// Everything runs on one queue; the callbacks are called there. +public final class LivePeer: NSObject, CaptureSink, @unchecked Sendable { + /// Which tracks a watcher gets: the app asks the owner's switches. Answer `(tracks, reason)`; + /// no track ends the watch with `bye` and the reason. + public var decide: (_ watcher: String, _ who: String, _ answer: @escaping ([String], String) -> Void) -> Void = { _, _, answer in answer([], "nothing to show") } + /// A signal for one watcher: the bridge appends it to `/devices/`. + public var send: (LiveSignal) -> Void = { _ in } + /// The two kept events of a watch, for the bridge: `{ started: { watcher, who, tracks } }` and + /// `{ ended: { watcher, who, seconds, why } }`. + public var kept: ([String: Any]) -> Void = { _ in } + public var started: (_ watcher: String, _ who: String, _ tracks: [String]) -> Void = { _, _, _ in } + public var ended: (_ watcher: String, _ who: String, _ seconds: Int, _ why: String) -> Void = { _, _, _, _ in } + /// The watchers changed: the capture runs for their tracks; the panel lists them. + public var changed: (_ watchers: LiveWatchers) -> Void = { _ in } + public var log: (String) -> Void = { _ in } + + private let work = DispatchQueue(label: "live") + private var watchers = LiveWatchers() + private let factory: LKRTCPeerConnectionFactory + private let soundDevice = SoundDevice() + private let soundTrack: LKRTCAudioTrack + private var connections: [String: Connection] = [:] + private var iceServers: [LKRTCIceServer] + private let stun: [LKRTCIceServer] + private let loopbackOnly: Bool + private let constraints = LKRTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil) + + /// Touched from the capture queues: under `lock`. + private let lock = NSLock() + private var pictures: [String: (source: LKRTCVideoSource, capturer: LKRTCVideoCapturer, track: LKRTCVideoTrack)] = [:] + /// The sound track offered now (`mic-0` or `test-2`), and the capture tracks that feed + /// `camera-0` and `mic-0`. + private var hearing: String? + private var feeds: [String: String] = [:] + + /// `loopbackOnly`: candidates on 127.0.0.1 alone, for a watcher on this Mac (no packet to the + /// local network). `stun`: false for no STUN server until `setIceServers`. + public init(loopbackOnly: Bool = false, stun: Bool = true) { + _ = LivePeer.initializeOnce + self.loopbackOnly = loopbackOnly + factory = LKRTCPeerConnectionFactory( + encoderFactory: LKRTCDefaultVideoEncoderFactory(), decoderFactory: LKRTCDefaultVideoDecoderFactory(), audioDevice: soundDevice) + if loopbackOnly { + let options = LKRTCPeerConnectionFactoryOptions() + options.ignoreLoopbackNetworkAdapter = false + options.ignoreWiFiNetworkAdapter = true + options.ignoreEthernetNetworkAdapter = true + options.ignoreVPNNetworkAdapter = true + options.ignoreCellularNetworkAdapter = true + factory.setOptions(options) + } + self.stun = stun ? [LKRTCIceServer(urlStrings: ["stun:stun.cloudflare.com:3478", "stun:stun.l.google.com:19302"])] : [] + iceServers = self.stun + soundTrack = factory.audioTrack(with: factory.audioSource(with: nil), trackId: "sound") + // The samples are the capture's own (Apple's voice processing, or a test tone): libwebrtc's + // echo canceller, noise suppressor, gain control and high-pass filter stay off. + _ = soundTrack.setAudioProcessingOptions(.raw()) + super.init() + } + + /// Once per process, before the first factory: every video packet asks the watcher to show it + /// at once (the RTP playout-delay extension, min 0, max 0). Measured: 15 to 40 ms from drawn + /// to shown, against about 180 ms with the browser's own playout. + private static let initializeOnce: Void = { + // deprecated, and the one the measurements were made with + LKRTCInitFieldTrialDictionary(["WebRTC-ForceSendPlayoutDelay": "min_ms:0,max_ms:0"]) + LKRTCInitializeSSL() + LKRTCSetMinDebugLogLevel(.error) + }() + + // MARK: - From the bridge and the app + + /// One `device/live-signal` to the device: `{ from, want | description | candidate | bye }`. + /// `who` is the event's source as a short name. + public func heard(_ signal: [String: Any], who: String) { + work.async { + guard let from = signal["from"] as? String, !from.isEmpty else { return } + let now = Date() + if signal["want"] as? Bool == true { + self.run(self.watchers.want(from: from, who: who, now: now)) + } else if signal["bye"] != nil { + self.run(self.watchers.end(from, why: "the watcher left", tell: false, now: now)) + } else if let d = signal["description"] as? [String: Any], d["type"] as? String == "answer", let sdp = d["sdp"] as? String { + self.run(self.watchers.answer(from: from, sdp: sdp)) + } else if let c = signal["candidate"] as? [String: Any], let line = c["candidate"] as? String, !line.isEmpty, + // loopback alone: a watcher's `.local` name is not resolved (that asks the local network) + !(self.loopbackOnly && line.contains(".local")) + { + let candidate = LiveCandidate(candidate: line, sdpMid: c["sdpMid"] as? String, sdpMLineIndex: (c["sdpMLineIndex"] as? Int) ?? 0) + self.run(self.watchers.candidate(from: from, candidate)) + } + } + } + + /// The addresses a connection may use (STUN, TURN): `[{ urls, username?, credential? }]`, for + /// every peer connection made after. None: STUN alone. + public func setIceServers(_ servers: [[String: Any]]) { + work.async { + let made = servers.compactMap { server -> LKRTCIceServer? in + let urls = (server["urls"] as? [String]) ?? (server["urls"] as? String).map { [$0] } ?? [] + return urls.isEmpty ? nil : LKRTCIceServer(urlStrings: urls, username: server["username"] as? String, credential: server["credential"] as? String) + } + self.iceServers = made.isEmpty ? self.stun : made + self.log("live: \(self.iceServers.count) ICE servers: \(self.iceServers.flatMap(\.urlStrings).map { $0.components(separatedBy: "?")[0] }.joined(separator: ", "))") + } + } + + /// The owner's button, or the Mac is no longer lent: the watcher hears `bye` with `why`. + public func end(_ watcher: String, why: String) { + work.async { self.run(self.watchers.end(watcher, why: why, tell: true, now: Date())) } + } + + public func endAll(why: String) { + work.sync { run(watchers.endAll(why: why, now: Date())) } + } + + /// The watchers now. + public var state: LiveWatchers { work.sync { watchers } } + + /// The peer connection of a watcher, for its statistics (`live-run`). + public func connection(_ watcher: String) -> LKRTCPeerConnection? { work.sync { connections[watcher]?.pc } } + + // MARK: - The effects of the signalling + + private func run(_ effects: [LiveEffect]) { + for effect in effects { + switch effect { + case let .decide(id, who): + decide(id, who) { tracks, reason in + self.work.async { + self.run(self.watchers.decided(id, tracks: tracks, reason: reason)) + self.watchersChanged() + } + } + case let .makeOffer(id, tracks): + offer(id, tracks) + case let .send(signal): + send(signal) + case let .setAnswer(id, sdp): + guard let connection = connections[id] else { continue } + connection.remoteSet = false + connection.pc.setRemoteDescription(LKRTCSessionDescription(type: .answer, sdp: sdp)) { error in + self.work.async { + if let error { self.log("live: \(id.prefix(8)): the answer was refused: \(error.localizedDescription)") } + connection.remoteSet = true + for candidate in connection.early { connection.pc.add(candidate) { _ in } } + connection.early = [] + } + } + case let .addCandidate(id, c): + guard let connection = connections[id] else { continue } + let candidate = LKRTCIceCandidate(sdp: c.candidate, sdpMLineIndex: Int32(c.sdpMLineIndex), sdpMid: c.sdpMid) + if connection.remoteSet { connection.pc.add(candidate) { _ in } } else { connection.early.append(candidate) } + case let .close(id): + connections.removeValue(forKey: id)?.pc.close() + watchersChanged() + case let .started(id, who, tracks): + log("live: \(who) watches (\(id.prefix(8))): \(tracks.joined(separator: ", "))") + kept(["started": ["watcher": id, "who": who, "tracks": tracks]]) + started(id, who, tracks) + watchersChanged() + case let .ended(id, who, seconds, why): + log("live: \(who) watched \(seconds) s (\(id.prefix(8))): \(why)") + kept(["ended": ["watcher": id, "who": who, "seconds": seconds, "why": why]]) + ended(id, who, seconds, why) + } + } + } + + /// On `work`: the pictures and the sound follow the watchers' tracks; the app hears of it. + private func watchersChanged() { + let tracks = watchers.tracks + lock.withLock { + pictures = pictures.filter { tracks.contains($0.key) } + hearing = tracks.first { $0 == "mic-0" || $0 == "test-2" } + } + changed(watchers) + } + + /// On `work`: the watcher's peer connection: each of its tracks, send only, and the offer. + private func offer(_ id: String, _ tracks: [String]) { + let configuration = LKRTCConfiguration() + configuration.sdpSemantics = .unifiedPlan + configuration.iceServers = iceServers + configuration.tcpCandidatePolicy = .disabled + configuration.bundlePolicy = .maxBundle + configuration.rtcpMuxPolicy = .require + configuration.continualGatheringPolicy = .gatherOnce + let connection = Connection(id: id, peer: self) + guard let pc = factory.peerConnection(with: configuration, constraints: constraints, delegate: connection) else { + return run(watchers.end(id, why: "no peer connection", tell: true, now: Date())) + } + connection.pc = pc + connections[id] = connection + // H.264 in hardware: High before Constrained Baseline, the other codecs after. + let codecs = factory.rtpSenderCapabilities(forKind: "video").codecs + let h264 = codecs.filter { $0.name.uppercased() == "H264" } + let high = h264.filter { ($0.parameters["profile-level-id"] ?? "").hasPrefix("64") } + let preferred = high + h264.filter { !high.contains($0) } + codecs.filter { $0.name.uppercased() != "H264" } + var sent: [(LKRTCRtpTransceiver, String)] = [] + for track in tracks { + let setup = LKRTCRtpTransceiverInit() + setup.direction = .sendOnly + if track == "mic-0" || track == "test-2" { + // The sound in a stream of its own: the watcher does not hold it back for lip sync + // (measured: lip sync held it about 140 ms, with the picture shown at once). + setup.streamIds = ["device-sound"] + if let transceiver = pc.addTransceiver(with: soundTrack, init: setup) { sent.append((transceiver, track)) } + continue + } + setup.streamIds = ["device"] + guard let transceiver = pc.addTransceiver(with: picture(track), init: setup) else { continue } + try? transceiver.setCodecPreferences(preferred, error: ()) + // A screen of text keeps its size and gives up frames first. + let parameters = transceiver.sender.parameters + parameters.degradationPreference = NSNumber(value: LKRTCDegradationPreference.maintainResolution.rawValue) + transceiver.sender.parameters = parameters + sent.append((transceiver, track)) + } + pc.offer(for: constraints) { offer, error in + guard let offer else { + return self.work.async { self.run(self.watchers.end(id, why: "no offer: \(String(describing: error))", tell: true, now: Date())) } + } + pc.setLocalDescription(offer) { error in + self.work.async { + if let error { return self.run(self.watchers.end(id, why: "no offer: \(error.localizedDescription)", tell: true, now: Date())) } + var mids: [String: String] = [:] + for (transceiver, track) in sent where !transceiver.mid.isEmpty { mids[transceiver.mid] = track } + self.run(self.watchers.offerMade(id, sdp: offer.sdp, mids: mids)) + } + } + } + } + + /// On `work`: the video track of a capture track, shared by every watcher. + private func picture(_ id: String) -> LKRTCVideoTrack { + lock.withLock { + if let made = pictures[id] { return made.track } + // a camera is a person's picture; a screen or the test picture is a screencast + let source = factory.videoSource(forScreenCast: !id.hasPrefix("camera-")) + let made = (source: source, capturer: LKRTCVideoCapturer(delegate: source), track: factory.videoTrack(with: source, trackId: id)) + pictures[id] = made + return made.track + } + } + + fileprivate func connection(_ id: String, changed state: LKRTCPeerConnectionState) { + work.async { + guard self.connections[id] != nil else { return } + if state == .connected { self.run(self.watchers.connected(id, now: Date())) } + if state == .failed { self.run(self.watchers.end(id, why: "connection failed", tell: true, now: Date())) } + } + } + + fileprivate func connection(_ id: String, made candidate: LKRTCIceCandidate) { + let c = LiveCandidate(candidate: candidate.sdp, sdpMid: candidate.sdpMid, sdpMLineIndex: Int(candidate.sdpMLineIndex)) + work.async { self.run(self.watchers.deviceCandidate(id, c)) } + } + + // MARK: - CaptureSink: on the capture's queues + + /// The live track a capture track feeds: the first camera and the first microphone that + /// deliver are `camera-0` and `mic-0`; the others are themselves. + private func live(_ track: String) -> String { + for (prefix, name) in [("camera-", "camera-0"), ("mic-", "mic-0")] where track.hasPrefix(prefix) { + if feeds[name] == nil { feeds[name] = track } + return feeds[name] == track ? name : "" + } + return track + } + + public func video(_ track: String, _ frame: CVPixelBuffer, hostTime: CMTime) { + guard let picture = lock.withLock({ pictures[live(track)] }) else { return } + // libwebrtc's clock is the host clock too + let nanos = CMTimeConvertScale(hostTime, timescale: 1_000_000_000, method: .roundHalfAwayFromZero).value + picture.source.capturer(picture.capturer, didCapture: LKRTCVideoFrame(buffer: LKRTCCVPixelBuffer(pixelBuffer: frame), rotation: ._0, timeStampNs: nanos)) + } + + public func sound(_ track: String, _ sample: CMSampleBuffer, hostTime: CMTime) { + guard lock.withLock({ hearing != nil && live(track) == hearing }) else { return } + soundDevice.put(sample) + } + + public func ended(_ track: String) { + lock.withLock { feeds = feeds.filter { $0.value != track } } + } +} + +/// One watcher's peer connection and its delegate. +private final class Connection: NSObject, LKRTCPeerConnectionDelegate { + let id: String + weak var peer: LivePeer? + var pc: LKRTCPeerConnection! + /// Candidates of the watcher that wait for the answer to be set. + var early: [LKRTCIceCandidate] = [] + var remoteSet = false + + init(id: String, peer: LivePeer) { + self.id = id + self.peer = peer + } + + func peerConnection(_ peerConnection: LKRTCPeerConnection, didGenerate candidate: LKRTCIceCandidate) { peer?.connection(id, made: candidate) } + func peerConnection(_ peerConnection: LKRTCPeerConnection, didChange newState: LKRTCPeerConnectionState) { peer?.connection(id, changed: newState) } + func peerConnection(_ peerConnection: LKRTCPeerConnection, didChange stateChanged: LKRTCSignalingState) {} + func peerConnection(_ peerConnection: LKRTCPeerConnection, didAdd stream: LKRTCMediaStream) {} + func peerConnection(_ peerConnection: LKRTCPeerConnection, didRemove stream: LKRTCMediaStream) {} + func peerConnectionShouldNegotiate(_ peerConnection: LKRTCPeerConnection) {} + func peerConnection(_ peerConnection: LKRTCPeerConnection, didChange newState: LKRTCIceConnectionState) {} + func peerConnection(_ peerConnection: LKRTCPeerConnection, didChange newState: LKRTCIceGatheringState) {} + func peerConnection(_ peerConnection: LKRTCPeerConnection, didRemove candidates: [LKRTCIceCandidate]) {} + func peerConnection(_ peerConnection: LKRTCPeerConnection, didOpen dataChannel: LKRTCDataChannel) {} +} diff --git a/Sources/Live/Sound.swift b/Sources/Live/Sound.swift new file mode 100644 index 0000000..b64e34a --- /dev/null +++ b/Sources/Live/Sound.swift @@ -0,0 +1,137 @@ +import AVFoundation +import Foundation +import LiveKitWebRTC + +/// The sound of a watch, as libwebrtc sees it: an audio device of our own that only sends. +/// libwebrtc opens no microphone and no speaker. Its model is PUSH: whoever has sound calls +/// `deliverRecordedData`, always from the same thread (RTCAudioDevice.h). The capture's buffers go +/// to one thread of ours (`put`), which turns them into 16-bit mono at 48 kHz and hands libwebrtc +/// each exact 10 ms block as soon as it is full. No timer: the capture's clock paces it. +final class SoundDevice: NSObject, LKRTCAudioDevice, @unchecked Sendable { + static let rate = 48_000.0 + static let block = 480 // 10 ms + + private var delegate: LKRTCAudioDeviceDelegate? + private let lock = NSLock() + private var sending = false + private let condition = NSCondition() + private var waiting: [CMSampleBuffer] = [] + // Touched only by the push thread. + private var pending = [Int16]() + private var time = AudioTimeStamp() + private let out = UnsafeMutablePointer.allocate(capacity: SoundDevice.block) + private let mono = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SoundDevice.rate, channels: 1, interleaved: true)! + private var converter: AVAudioConverter? + + override init() { + super.init() + time.mFlags = .sampleTimeValid + let thread = Thread { [weak self] in while let self { self.next() } } + thread.name = "live.sound" + thread.qualityOfService = .userInteractive + thread.start() + } + + /// From the capture's callback: hand the buffer to the push thread. While nobody listens it is + /// dropped. At most 50 wait (half a second or more): the oldest go. + func put(_ sample: CMSampleBuffer) { + guard lock.withLock({ sending }) else { return } + condition.lock() + waiting.append(sample) + if waiting.count > 50 { waiting.removeFirst(waiting.count - 50) } + condition.signal() + condition.unlock() + } + + /// On the push thread: the next buffers, as 10 ms blocks. + private func next() { + condition.lock() + while waiting.isEmpty { condition.wait() } + let samples = waiting + waiting = [] + condition.unlock() + for sample in samples { push(sample) } + } + + private func push(_ sample: CMSampleBuffer) { + guard let pcm = convert(sample), let data = pcm.int16ChannelData else { return } + pending.append(contentsOf: UnsafeBufferPointer(start: data[0], count: Int(pcm.frameLength))) + var used = 0 + while pending.count - used >= Self.block { + pending.withUnsafeBufferPointer { _ = memcpy(out, $0.baseAddress! + used, Self.block * 2) } + used += Self.block + guard let deliver = lock.withLock({ sending ? delegate?.deliverRecordedData : nil }) else { continue } + var list = AudioBufferList(mNumberBuffers: 1, mBuffers: AudioBuffer(mNumberChannels: 1, mDataByteSize: UInt32(Self.block * 2), mData: out)) + var flags = AudioUnitRenderActionFlags() + _ = deliver(&flags, &time, 1, UInt32(Self.block), &list, nil, nil) + time.mSampleTime += Double(Self.block) + } + pending.removeFirst(used) + } + + /// The buffer as 16-bit mono at 48 kHz, whatever the capture gave: a microphone's own format, + /// another rate, several channels (they are mixed). + private func convert(_ sample: CMSampleBuffer) -> AVAudioPCMBuffer? { + guard let description = sample.formatDescription, sample.numSamples > 0 else { return nil } + let format = AVAudioFormat(cmAudioFormatDescription: description) + let frames = AVAudioFrameCount(sample.numSamples) + guard let source = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames) else { return nil } + source.frameLength = frames + guard CMSampleBufferCopyPCMDataIntoAudioBufferList(sample, at: 0, frameCount: Int32(frames), into: source.mutableAudioBufferList) == noErr + else { return nil } + if converter?.inputFormat != format { + converter = AVAudioConverter(from: format, to: mono) + converter?.downmix = true + } + let capacity = AVAudioFrameCount(Double(frames) * Self.rate / format.sampleRate) + 64 + guard let converter, let result = AVAudioPCMBuffer(pcmFormat: mono, frameCapacity: capacity) else { return nil } + var given = false + _ = converter.convert(to: result, error: nil) { _, status in + status.pointee = given ? .noDataNow : .haveData + defer { given = true } + return given ? nil : source + } + return result.frameLength > 0 ? result : nil + } + + var deviceInputSampleRate: Double { Self.rate } + var inputIOBufferDuration: TimeInterval { Double(Self.block) / Self.rate } + var inputNumberOfChannels: Int { 1 } + var inputLatency: TimeInterval { 0 } + var deviceOutputSampleRate: Double { Self.rate } + var outputIOBufferDuration: TimeInterval { Double(Self.block) / Self.rate } + var outputNumberOfChannels: Int { 1 } + var outputLatency: TimeInterval { 0 } + var isInitialized: Bool { lock.withLock { delegate != nil } } + var isPlayoutInitialized = false + var isRecordingInitialized = false + var isPlaying = false + var isRecording: Bool { lock.withLock { sending } } + + func initialize(with delegate: LKRTCAudioDeviceDelegate) -> Bool { + lock.withLock { self.delegate = delegate } + return true + } + + func terminateDevice() -> Bool { + lock.withLock { (sending, delegate) = (false, nil) } + return true + } + + func initializePlayout() -> Bool { isPlayoutInitialized = true; return true } + // the device only sends: nothing is pulled + func startPlayout() -> Bool { isPlaying = true; return true } + func stopPlayout() -> Bool { isPlaying = false; return true } + func initializeRecording() -> Bool { isRecordingInitialized = true; return true } + + /// libwebrtc wants sound (a watcher has the sound track): from now on each block goes to it. + func startRecording() -> Bool { + lock.withLock { sending = delegate != nil } + return true + } + + func stopRecording() -> Bool { + lock.withLock { sending = false } + return true + } +} diff --git a/Sources/Record/Recorder.swift b/Sources/Record/Recorder.swift index 2209a64..5310475 100644 --- a/Sources/Record/Recorder.swift +++ b/Sources/Record/Recorder.swift @@ -1,4 +1,5 @@ import AppKit +import CoreMedia import Foundation /// What a recorder may record. Each one is a switch of its own. @@ -17,6 +18,9 @@ public enum RecordSource: String, CaseIterable, Sendable { case .synthetic: return "test-" } } + + /// The source of a track id: `screen-1` is `screens`. + public static func of(track: String) -> RecordSource? { allCases.first { track.hasPrefix($0.prefix) } } } public struct RecordOptions: Sendable { @@ -55,6 +59,11 @@ public final class Recorder: @unchecked Sendable { private var sources: [RecordSource: Source] = [:] private var tracks: [Track] = [] private var paused = false + /// What the record wants and what live wants (`watch`): the sources run for both. + private var recorded: Set = [] + private var watched: Set = [] + /// Who gets each frame and sound buffer: the record's own tracks first, then live. + private lazy var sinks: [CaptureSink] = [RecordSink(recorder: self)] private let control = DispatchQueue(label: "record.control") private let slots = DispatchQueue(label: "record.slots") private var entries: [Int64: [String: [String: Any]]] = [:] @@ -70,7 +79,11 @@ public final class Recorder: @unchecked Sendable { } var isPaused: Bool { lock.withLock { paused } } - func add(_ track: Track) { lock.withLock { tracks.append(track) } } + /// A track writes files only while the record wants its source; one made for live alone is + /// not added, so it gets no frame and writes nothing. + func add(_ track: Track) { + lock.withLock { if recorded.contains(where: { track.id.hasPrefix($0.prefix) }) { tracks.append(track) } } + } func track(_ id: String) -> Track? { lock.withLock { tracks.first { $0.id == id } } } /// The ids of the tracks that record now. @@ -103,37 +116,86 @@ public final class Recorder: @unchecked Sendable { log("record: device \(options.device), spool \(options.spool.path), slots sealed up to \(sealed)") } - /// Starts the sources that are in `wanted` and stops the others, each with its tracks: a track - /// that stops writes what it has, and its slot is sealed when its other tracks close. - public func set(_ wanted: Set) { - control.async { - let running = Set(self.lock.withLock { self.sources.keys }) - for kind in running.subtracting(wanted) { - guard let source = self.lock.withLock({ self.sources.removeValue(forKey: kind) }) else { continue } + /// The sources the record wants: each records into the spool. A source that live alone wants + /// runs, and writes nothing. + public func set(_ wanted: Set) { control.async { self.apply(record: wanted, live: nil) } } + + /// The sources live wants (`Sources/Live`): they run while somebody watches. A screen then + /// captures `CaptureRate.live` frames a second; the record keeps its own `fps` of them by time. + public func watch(_ wanted: Set) { control.async { self.apply(record: nil, live: wanted) } } + + /// Another receiver of every frame and sound buffer (live). It gets them on the capture's queue. + public func add(sink: CaptureSink) { lock.withLock { sinks.append(sink) } } + public func remove(sink: CaptureSink) { lock.withLock { sinks.removeAll { $0 === sink } } } + + /// On `control`: starts and stops the sources by `CapturePlan`. A source that stops takes its + /// tracks with it: each writes what it has, and its slot is sealed when its other tracks close. + private func apply(record: Set?, live: Set?) { + let (running, recordBefore, liveBefore) = lock.withLock { (Set(sources.keys), recorded, watched) } + let plan = CapturePlan.changes(running: running, record: (recordBefore, record ?? recordBefore), live: (liveBefore, live ?? liveBefore)) + lock.withLock { (recorded, watched) = (record ?? recordBefore, live ?? liveBefore) } + for change in plan { + switch change { + case .stop(let kind), .restart(let kind): + guard let source = lock.withLock({ sources.removeValue(forKey: kind) }) else { continue } source.stop() - let stopping = self.lock.withLock { () -> [Track] in - defer { self.tracks.removeAll { $0.id.hasPrefix(kind.prefix) } } - return self.tracks.filter { $0.id.hasPrefix(kind.prefix) } - } - for track in stopping { track.stop() } + endTracks(kind) log("record: \(kind.rawValue) off") + if case .restart = change { start(kind) } + case .start(let kind): + start(kind) + case .stopRecording(let kind): + endTracks(kind) + log("record: \(kind.rawValue) off (live goes on)") + case .rate(let kind): + lock.withLock { sources[kind] }?.rate(captureFps(kind)) } - for kind in wanted.subtracting(running) { - let source: Source - switch kind { - case .screens: source = Screens(recorder: self) - case .cameras: source = DeviceCapture(recorder: self, media: .video) - case .mics: source = DeviceCapture(recorder: self, media: .audio) - case .speakers: source = Speakers(recorder: self) - case .synthetic: source = Synthetic(recorder: self) - } - self.lock.withLock { self.sources[kind] = source } - source.start() - log("record: \(kind.rawValue) on") - } } } + private func start(_ kind: RecordSource) { + let source: Source + switch kind { + case .screens: source = Screens(recorder: self) + case .cameras: source = DeviceCapture(recorder: self, media: .video) + case .mics: source = DeviceCapture(recorder: self, media: .audio) + case .speakers: source = Speakers(recorder: self) + case .synthetic: source = Synthetic(recorder: self) + } + lock.withLock { sources[kind] = source } + source.start() + let (records, watches) = lock.withLock { (recorded.contains(kind), watched.contains(kind)) } + let fast = watches && (kind == .screens || kind == .synthetic) + log("record: \(kind.rawValue) on\(records ? "" : " for live alone")\(fast ? ", watched at \(captureFps(kind)) fps" : "")") + } + + /// The tracks of a source stop: each writes what it has. + private func endTracks(_ kind: RecordSource) { + let stopping = lock.withLock { () -> [Track] in + defer { tracks.removeAll { $0.id.hasPrefix(kind.prefix) } } + return tracks.filter { $0.id.hasPrefix(kind.prefix) } + } + for track in stopping { track.stop() } + } + + /// The frames a second a screen or the synthetic picture captures now. + func captureFps(_ kind: RecordSource) -> Int { + CaptureRate.capture(record: options.fps, watched: lock.withLock { watched.contains(kind) }) + } + + /// Hands a frame to every sink (from a source, on its capture queue). + func video(_ track: String, _ frame: CVPixelBuffer, hostTime: CMTime) { + for sink in lock.withLock({ sinks }) { sink.video(track, frame, hostTime: hostTime) } + } + + func sound(_ track: String, _ sample: CMSampleBuffer, hostTime: CMTime) { + for sink in lock.withLock({ sinks }) { sink.sound(track, sample, hostTime: hostTime) } + } + + func ended(_ track: String) { + for sink in lock.withLock({ sinks }) { sink.ended(track) } + } + /// The Mac is locked or its displays sleep: screens and cameras pause and start again by /// themselves; a camera's capture stops, so its light is off. Microphones and speakers go on. public func pause(_ on: Bool) { @@ -149,7 +211,7 @@ public final class Recorder: @unchecked Sendable { /// Stops every source and track, writes what is left and seals every slot. public func stop() { - set([]) + control.async { self.apply(record: [], live: []) } control.sync {} timer?.cancel() tickQueue.sync {} @@ -299,9 +361,12 @@ protocol Source: AnyObject { func paused() /// After an unlock or a wake: open again what could not open (a locked Mac lists no display). func resumed() + /// A watcher came or the last one left: a screen captures this many frames a second now. + func rate(_ fps: Int) } extension Source { func paused() {} func resumed() {} + func rate(_ fps: Int) {} } diff --git a/Sources/Record/Sink.swift b/Sources/Record/Sink.swift new file mode 100644 index 0000000..bed33d2 --- /dev/null +++ b/Sources/Record/Sink.swift @@ -0,0 +1,115 @@ +import CoreMedia +import CoreVideo +import Foundation + +/// A receiver of what the sources capture: every frame and every sound buffer, as captured, with +/// its host time. The record's own tracks are one sink (`RecordSink`); live (`Sources/Live`) is +/// another. A sink is called on the capture's own queue and must not block it. +public protocol CaptureSink: AnyObject { + /// `track` is the record's track id: `screen-`, `camera-`, `test-1`. + func video(_ track: String, _ frame: CVPixelBuffer, hostTime: CMTime) + /// PCM in the capture's own format: `mic-`, `speaker-0`, `test-2`. + func sound(_ track: String, _ sample: CMSampleBuffer, hostTime: CMTime) + /// The source of `track` is gone: an unplugged screen or camera. + func ended(_ track: String) +} + +/// How many frames a second a screen captures, and which of them the record keeps. +public enum CaptureRate { + /// While somebody watches: a screen captures 30 frames a second. + public static let live = 30 + /// A frame is dropped when it comes less than this before its time (`keeps`): 4 ms, so that + /// every third frame of 30 passes for 10, and every second of 30 for 15. + static let slack = 0.004 + + public static func capture(record fps: Int, watched: Bool) -> Int { watched ? max(fps, live) : fps } + + /// Whether the record keeps a frame at `now` (seconds), when it kept the last one at `last`: + /// at most `fps` a second, by time. A frame from before the last one (a clock that went back) + /// is kept. + public static func keeps(last: Double?, now: Double, fps: Int) -> Bool { + guard let last, now >= last else { return true } + return now - last >= 1 / Double(fps) - slack + } +} + +/// What the recorder does when what the record wants or what live wants changes. A source runs +/// while either wants it. Pure: the recorder applies it (`Recorder.apply`). +public enum CapturePlan { + public enum Change: Equatable { + /// Start the source (its tracks record when the record wants it). + case start(RecordSource) + /// Stop the source and its tracks. + case stop(RecordSource) + /// The record no longer wants a source that live still watches: its tracks end, the source runs on. + case stopRecording(RecordSource) + /// The record wants a source that ran for live alone: it starts again, with its tracks. + case restart(RecordSource) + /// Live began or ended watching a source the record has: its frames a second change. + case rate(RecordSource) + } + + /// Stops first, then the rest, each in the order of `RecordSource.allCases`. + public static func changes( + running: Set, record: (before: Set, now: Set), live: (before: Set, now: Set) + ) -> [Change] { + var stops: [Change] = [], others: [Change] = [] + for kind in RecordSource.allCases { + let wanted = record.now.contains(kind) || live.now.contains(kind) + let recordedBefore = record.before.contains(kind), records = record.now.contains(kind) + if running.contains(kind) { + if !wanted { + stops.append(.stop(kind)) + } else if records && !recordedBefore { + others.append(.restart(kind)) + } else if !records && recordedBefore { + others.append(.stopRecording(kind)) + } else if live.before.contains(kind) != live.now.contains(kind) { + others.append(.rate(kind)) + } + } else if wanted { + others.append(.start(kind)) + } + } + return stops + others + } +} + +/// The record's own sink: each frame and buffer to its track, when the record has that track. A +/// screen that captures faster for live is kept at the record's `fps` by time; a camera always at +/// `cameraFps` (a camera's format belongs to the whole Mac: another program can set it to 30). +final class RecordSink: CaptureSink { + private unowned let r: Recorder + private let lock = NSLock() + /// The time of the last frame kept, per track. + private var kept: [String: Double] = [:] + + init(recorder: Recorder) { r = recorder } + + func video(_ id: String, _ frame: CVPixelBuffer, hostTime: CMTime) { + guard let track = r.track(id) as? VideoTrack else { return } + let fps: Int? + if id.hasPrefix(RecordSource.cameras.prefix) { + fps = r.options.cameraFps + } else { + let kind: RecordSource = id.hasPrefix(RecordSource.synthetic.prefix) ? .synthetic : .screens + fps = r.captureFps(kind) > r.options.fps ? r.options.fps : nil + } + if let fps { + let now = CMTimeGetSeconds(hostTime) + let keep = lock.withLock { () -> Bool in + guard CaptureRate.keeps(last: kept[id], now: now, fps: fps) else { return false } + kept[id] = now + return true + } + guard keep else { return } + } + track.append(frame, hostTime: hostTime) + } + + func sound(_ id: String, _ sample: CMSampleBuffer, hostTime: CMTime) { + (r.track(id) as? AudioTrack)?.append(sample, hostTime: hostTime) + } + + func ended(_ id: String) { (r.track(id) as? VideoTrack)?.end() } +} diff --git a/Sources/Record/Sources.swift b/Sources/Record/Sources.swift index f737a93..3211f6c 100644 --- a/Sources/Record/Sources.swift +++ b/Sources/Record/Sources.swift @@ -12,7 +12,7 @@ import ScreenCaptureKit final class Screens: NSObject, Source, SCStreamOutput, SCStreamDelegate { private let r: Recorder private let queue = DispatchQueue(label: "record.screens") - private var streams: [SCStream: (track: VideoTrack, display: CGDirectDisplayID)] = [:] + private var streams: [SCStream: (track: VideoTrack, display: CGDirectDisplayID, config: SCStreamConfiguration)] = [:] private var tracks: [String: VideoTrack] = [:] private var restartPending = false private var stopped = false @@ -62,7 +62,7 @@ final class Screens: NSObject, Source, SCStreamOutput, SCStreamDelegate { let ids = Set(content.displays.map { "screen-\($0.displayID)" }) for (id, track) in queue.sync(execute: { tracks }) where !ids.contains(id) { log("\(id): the display is gone; its track ends") - track.end() + r.ended(track.id) } for display in content.displays { let id = "screen-\(display.displayID)" @@ -76,7 +76,8 @@ final class Screens: NSObject, Source, SCStreamOutput, SCStreamDelegate { let config = SCStreamConfiguration() config.width = Int(width) & ~1 config.height = Int(height) & ~1 - config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(r.options.fps)) + let fps = r.captureFps(.screens) + config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps)) config.showsCursor = true // allow-high-entropy-next-line: the name of Apple's pixel format constant, not a secret config.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange @@ -88,7 +89,7 @@ final class Screens: NSObject, Source, SCStreamOutput, SCStreamDelegate { guard !stopped else { return nil } let track = tracks[id] ?? VideoTrack(id: id, recorder: r) tracks[id] = track - streams[stream] = (track, display.displayID) + streams[stream] = (track, display.displayID, config) return track }) else { return } // Global display points and pixels per point, so that a mouse position maps to a screen. @@ -96,7 +97,7 @@ final class Screens: NSObject, Source, SCStreamOutput, SCStreamDelegate { do { try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue) try await stream.startCapture() - log("\(id): capturing \(config.width)×\(config.height) at up to \(r.options.fps) fps") + log("\(id): capturing \(config.width)×\(config.height) at up to \(fps) fps") } catch { log("\(id): not started, \(error.localizedDescription)") _ = queue.sync { streams.removeValue(forKey: stream) } @@ -108,7 +109,17 @@ final class Screens: NSObject, Source, SCStreamOutput, SCStreamDelegate { guard type == .screen, let track = streams[stream]?.track, let buffer = sample.imageBuffer else { return } let info = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]] guard let raw = info?.first?[.status] as? Int, SCFrameStatus(rawValue: raw) == .complete else { return } - track.append(buffer, hostTime: sample.presentationTimeStamp) + r.video(track.id, buffer, hostTime: sample.presentationTimeStamp) + } + + /// A watcher came or the last one left: each stream captures `fps` frames a second from now on. + func rate(_ fps: Int) { + for (stream, s) in queue.sync(execute: { streams }) { + s.config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps)) + stream.updateConfiguration(s.config) { error in + log("\(s.track.id): \(error.map { "frames a second not changed: \($0.localizedDescription)" } ?? "up to \(fps) fps")") + } + } } func stream(_ stream: SCStream, didStopWithError error: Error) { @@ -174,8 +185,6 @@ final class DeviceCapture: NSObject, Source, AVCaptureVideoDataOutputSampleBuffe private let media: AVMediaType private var open: [String: (session: AVCaptureSession, track: Track)] = [:] // by unique id private var byOutput: [AVCaptureOutput: (Track, AVCaptureSession)] = [:] - /// The time of the last camera frame that was kept, per output. - private var lastKept: [AVCaptureOutput: CMTime] = [:] private var observers: [NSObjectProtocol] = [] private let queue: DispatchQueue @@ -258,7 +267,7 @@ final class DeviceCapture: NSObject, Source, AVCaptureVideoDataOutputSampleBuffe guard let (session, track) = open.removeValue(forKey: uid) else { return } session.stopRunning() byOutput = byOutput.filter { $0.value.1 !== session } - (track as? VideoTrack)?.end() + r.ended(track.id) log("\(track.id): the device is gone; its track ends") } @@ -266,15 +275,11 @@ final class DeviceCapture: NSObject, Source, AVCaptureVideoDataOutputSampleBuffe guard let (track, session) = byOutput[output] else { return } // The session's clock is the host clock on macOS; convert anyway, in case it is not. let host = CMSyncConvertTime(sample.presentationTimeStamp, from: session.synchronizationClock ?? CMClockGetHostTimeClock(), to: CMClockGetHostTimeClock()) - if let video = track as? VideoTrack, let buffer = sample.imageBuffer { - // At most `cameraFps`: 4 ms of slack, so that every second frame of 30 passes for 15. - if let last = lastKept[output], CMTimeGetSeconds(host - last) < 1 / Double(r.options.cameraFps) - 0.004, CMTimeCompare(host, last) >= 0 { - return - } - lastKept[output] = host - video.append(buffer, hostTime: host) - } else if let audio = track as? AudioTrack { - audio.append(sample, hostTime: host) + // Every frame to the sinks: the record keeps at most `cameraFps` of them (`RecordSink`). + if track is VideoTrack, let buffer = sample.imageBuffer { + r.video(track.id, buffer, hostTime: host) + } else if track is AudioTrack { + r.sound(track.id, sample, hostTime: host) } } @@ -388,7 +393,7 @@ final class Speakers: NSObject, Source, SCStreamOutput, SCStreamDelegate { func stream(_ stream: SCStream, didOutputSampleBuffer sample: CMSampleBuffer, of type: SCStreamOutputType) { guard type == .audio, sample.isValid, sample.numSamples > 0 else { return } if restarts > 0, ProcessInfo.processInfo.systemUptime - openedAt >= 60 { restarts = 0 } - track.append(sample, hostTime: sample.presentationTimeStamp) + r.sound(track.id, sample, hostTime: sample.presentationTimeStamp) } func stream(_ stream: SCStream, didStopWithError error: Error) { diff --git a/Sources/Record/Synthetic.swift b/Sources/Record/Synthetic.swift index c55e3e4..72b76ea 100644 --- a/Sources/Record/Synthetic.swift +++ b/Sources/Record/Synthetic.swift @@ -15,6 +15,7 @@ final class Synthetic: Source { private let audio: AudioTrack private let queue = DispatchQueue(label: "record.synthetic") private var timers: [DispatchSourceTimer] = [] + private var frames: DispatchSourceTimer? private var pool: CVPixelBufferPool? private var audioStart = CMTime.zero private var samplesMade: Int64 = 0 @@ -33,21 +34,34 @@ final class Synthetic: Source { func start() { audioStart = r.clock.hostNow() - every(1.0 / Double(r.options.fps)) { self.frame() } - every(0.02) { self.sound() } - log("synthetic: test-1 video 1280×720 \(r.options.fps) fps, test-2 audio") + let fps = r.captureFps(.synthetic) + queue.sync { frames = every(1.0 / Double(fps)) { self.frame() } } + timers.append(every(0.02) { self.sound() }) + log("synthetic: test-1 video 1280×720 \(fps) fps, test-2 audio") } - private func every(_ seconds: Double, _ body: @escaping () -> Void) { + private func every(_ seconds: Double, _ body: @escaping () -> Void) -> DispatchSourceTimer { let timer = DispatchSource.makeTimerSource(flags: .strict, queue: queue) timer.schedule(deadline: .now() + seconds, repeating: seconds, leeway: .milliseconds(1)) timer.setEventHandler(handler: body) timer.resume() - timers.append(timer) + return timer + } + + /// Live watches it, or no longer: the picture is drawn `fps` times a second from now on. + func rate(_ fps: Int) { + queue.async { + guard let old = self.frames else { return } + old.cancel() + self.frames = self.every(1.0 / Double(fps)) { self.frame() } + } } func stop() { - for timer in timers { timer.cancel() } + queue.sync { + for timer in timers + [frames].compactMap({ $0 }) { timer.cancel() } + frames = nil + } queue.sync {} // a handler that is running now ends before the writers close } @@ -79,7 +93,7 @@ final class Synthetic: Source { } } CVPixelBufferUnlockBaseAddress(buffer, []) - video.append(buffer, hostTime: host) + r.video(video.id, buffer, hostTime: host) } private func sound() { @@ -115,6 +129,6 @@ final class Synthetic: Source { var list = AudioBufferList(mNumberBuffers: 1, mBuffers: AudioBuffer(mNumberChannels: 1, mDataByteSize: UInt32(raw.count), mData: raw.baseAddress)) return CMSampleBufferSetDataBufferFromAudioBufferList(sample, blockBufferAllocator: nil, blockBufferMemoryAllocator: nil, flags: 0, bufferList: &list) == noErr } - if ok { audio.append(sample, hostTime: host) } + if ok { r.sound(audio.id, sample, hostTime: host) } } } diff --git a/Sources/VoiceMeetCore/LiveRule.swift b/Sources/VoiceMeetCore/LiveRule.swift new file mode 100644 index 0000000..c481e11 --- /dev/null +++ b/Sources/VoiceMeetCore/LiveRule.swift @@ -0,0 +1,195 @@ +import Foundation + +/// LIVE: a watcher's page sees this Mac now, over WebRTC (README, "Live"). Pure: who gets which +/// track, and the signalling of each watch. `Sources/Live` runs the peer connections by these +/// rules; the app (`DeviceLive`) asks the owner's switches. + +extension UseGate { + /// One gate for a power that has several things (the cameras): the strictest of theirs. No + /// thing at all is off. + public static func strictest(_ gates: [UseGate]) -> UseGate { + for gate in [UseGate.off, .notPermitted, .ask] where gates.contains(gate) { return gate } + return gates.isEmpty ? .off : .go + } +} + +/// Which tracks a watcher gets, by the owner's switches: Off and a permission macOS has not given: +/// not offered. Ask: offered when the person approved the watch (one question for all of them). +/// Shared: offered. +public enum LiveTracks { + /// Whether the watch needs the person's approval first. + public static func asks(_ gates: [(track: String, gate: UseGate)]) -> Bool { gates.contains { $0.gate == .ask } } + + public static func offered(_ gates: [(track: String, gate: UseGate)], approved: Bool) -> [String] { + gates.filter { $0.gate == .go || ($0.gate == .ask && approved) }.map(\.track) + } + + /// The reason in the `bye` to a watcher who may see nothing. + public static func nothing(asked: Bool, approved: Bool) -> String { + asked && !approved ? "the owner did not approve the watch" : "nothing to show: the owner shares no screen, camera or microphone" + } +} + +/// One ICE candidate, as the contract carries it. +public struct LiveCandidate: Equatable, Sendable { + public let candidate: String + public let sdpMid: String? + public let sdpMLineIndex: Int + + public init(candidate: String, sdpMid: String?, sdpMLineIndex: Int) { + (self.candidate, self.sdpMid, self.sdpMLineIndex) = (candidate, sdpMid, sdpMLineIndex) + } +} + +/// A signal from the device to one watcher: `device/live-signal { from: "device", to, … }`. +public enum LiveSignal: Equatable, Sendable { + case offer(to: String, sdp: String, tracks: [String: String]) + case candidate(to: String, LiveCandidate) + case bye(to: String, reason: String) +} + +/// What the device does next, in order. +public enum LiveEffect: Equatable, Sendable { + /// Ask the owner's switches which tracks this watcher gets; answer with `decided`. + case decide(watcher: String, who: String) + /// Make a peer connection with these tracks, send only, and its offer; answer with `offerMade`. + case makeOffer(watcher: String, tracks: [String]) + case send(LiveSignal) + case setAnswer(watcher: String, sdp: String) + case addCandidate(watcher: String, LiveCandidate) + /// Close the watcher's peer connection. + case close(watcher: String) + /// The watch began: `device/live-watch-started`, and a running act in Activity. + case started(watcher: String, who: String, tracks: [String]) + /// The watch ended: `device/live-watch-ended`, and the act ends. + case ended(watcher: String, who: String, seconds: Int, why: String) +} + +/// The watchers of this device and the signalling of each. The watcher asks (`want`); the device +/// always offers, so there is no glare. The device's candidates wait for its offer, the watcher's +/// for its answer: each side's signals go out and are used in order. At most `most` watchers. +public struct LiveWatchers: Equatable, Sendable { + public static let most = 4 + + public enum Phase: Equatable, Sendable { + /// The owner's switches are asked (an Ask may wait for the person). + case deciding + /// The peer connection and its offer are being made. + case offering + /// The offer is sent; the answer has not come. + case offered + case answered + case connected + } + + public struct Watch: Equatable, Sendable { + public let id: String + /// Who asked: a short name or an email, from the `want` event's source. + public let who: String + public internal(set) var phase: Phase + public internal(set) var tracks: [String] = [] + public let asked: Date + public internal(set) var connected: Date? + /// The device's candidates made before its offer went out. + var held: [LiveCandidate] = [] + /// The watcher's candidates that came before its answer. + var early: [LiveCandidate] = [] + } + + public private(set) var watches: [Watch] = [] + + public init() {} + + /// Every track some watcher gets (from the owner's decision on): the capture runs for these. + public var tracks: Set { Set(watches.flatMap(\.tracks)) } + + public func watch(_ id: String) -> Watch? { watches.first { $0.id == id } } + + private func index(_ id: String) -> Int? { watches.firstIndex { $0.id == id } } + + /// `device/live-signal { from, to: "device", want: true }`. + public mutating func want(from id: String, who: String, now: Date) -> [LiveEffect] { + let before = end(id, why: "asked again", tell: false, now: now) + guard watches.count < Self.most else { return before + [.send(.bye(to: id, reason: "too many watchers (\(Self.most))"))] } + watches.append(Watch(id: id, who: who, phase: .deciding, asked: now)) + return before + [.decide(watcher: id, who: who)] + } + + /// The owner's switches answered for this watcher. No track: `bye` with `reason`. + public mutating func decided(_ id: String, tracks: [String], reason: String) -> [LiveEffect] { + guard let i = index(id), watches[i].phase == .deciding else { return [] } + guard !tracks.isEmpty else { + watches.remove(at: i) + return [.send(.bye(to: id, reason: reason))] + } + watches[i].tracks = tracks + watches[i].phase = .offering + return [.makeOffer(watcher: id, tracks: tracks)] + } + + /// The offer is made: it goes out, then the candidates made meanwhile. `mids` maps each `mid` + /// of the offer to a track id. + public mutating func offerMade(_ id: String, sdp: String, mids: [String: String]) -> [LiveEffect] { + guard let i = index(id), watches[i].phase == .offering else { return [] } + watches[i].phase = .offered + let held = watches[i].held + watches[i].held = [] + return [.send(.offer(to: id, sdp: sdp, tracks: mids))] + held.map { .send(.candidate(to: id, $0)) } + } + + /// The device's peer connection made a candidate. + public mutating func deviceCandidate(_ id: String, _ candidate: LiveCandidate) -> [LiveEffect] { + guard let i = index(id) else { return [] } + if watches[i].phase == .offering { + watches[i].held.append(candidate) + return [] + } + return [.send(.candidate(to: id, candidate))] + } + + /// `{ description: { type: "answer", sdp } }`: then the watcher's candidates that came early. + public mutating func answer(from id: String, sdp: String) -> [LiveEffect] { + guard let i = index(id), watches[i].phase == .offered else { return [] } + watches[i].phase = .answered + let early = watches[i].early + watches[i].early = [] + return [.setAnswer(watcher: id, sdp: sdp)] + early.map { .addCandidate(watcher: id, $0) } + } + + /// `{ candidate }` from the watcher. Before the answer it waits (at most 40). + public mutating func candidate(from id: String, _ candidate: LiveCandidate) -> [LiveEffect] { + guard let i = index(id) else { return [] } + switch watches[i].phase { + case .answered, .connected: + return [.addCandidate(watcher: id, candidate)] + default: + if watches[i].early.count < 40 { watches[i].early.append(candidate) } + return [] + } + } + + /// The peer connection is connected: the watch begins. + public mutating func connected(_ id: String, now: Date) -> [LiveEffect] { + guard let i = index(id), watches[i].connected == nil, watches[i].phase == .answered else { return [] } + watches[i].phase = .connected + watches[i].connected = now + return [.started(watcher: id, who: watches[i].who, tracks: watches[i].tracks)] + } + + /// The watch ends: the watcher's `bye` ("the watcher left", `tell` false), a failed connection, + /// the owner's button, the device's stop (`tell` true: the watcher hears `bye`). + public mutating func end(_ id: String, why: String, tell: Bool, now: Date) -> [LiveEffect] { + guard let i = index(id) else { return [] } + let watch = watches.remove(at: i) + var effects: [LiveEffect] = [.close(watcher: id)] + if tell { effects.append(.send(.bye(to: id, reason: why))) } + if let connected = watch.connected { + effects.append(.ended(watcher: id, who: watch.who, seconds: Int(now.timeIntervalSince(connected).rounded()), why: why)) + } + return effects + } + + public mutating func endAll(why: String, now: Date) -> [LiveEffect] { + watches.map(\.id).flatMap { end($0, why: why, tell: true, now: now) } + } +} diff --git a/Sources/VoiceMeetMac/AppDelegate.swift b/Sources/VoiceMeetMac/AppDelegate.swift index 19eb975..a0353ec 100644 --- a/Sources/VoiceMeetMac/AppDelegate.swift +++ b/Sources/VoiceMeetMac/AppDelegate.swift @@ -72,6 +72,11 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegate { return DeviceRecord.Destination(node: config.node, iterateBin: config.iterateBin, uploader: uploader, mind: mind.mind, name: mind.name) } model.record.onChange = { [weak self] in self?.stateDidChange() } + model.live.destination = { [weak self] in + guard let config = self?.lend.config, let mind = config.activeMind else { return nil } + let script = URL(fileURLWithPath: config.bridge).deletingLastPathComponent().appendingPathComponent("device-live.mjs").path + return DeviceLive.Destination(node: config.node, iterateBin: config.iterateBin, script: script, mind: mind.mind, name: mind.name) + } model.banners.onChange = { [weak self] in self?.stateDidChange() } model.start() lend.isBusy = { [weak self] in @@ -314,6 +319,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegate { private func choiceChanged() { log.info("sharing: " + Shareable.all.map(Self.line).joined(separator: "; ")) stateDidChange() + model.liveChoicesChanged() if popover.isShown { syncPreviews() } // A thing was turned on whose permission macOS has not given: the guide starts. if let permission = Shareable.all.first(where: { $0.sharing != .off && !$0.isPermitted })?.power.permission, diff --git a/Sources/VoiceMeetMac/AppModel.swift b/Sources/VoiceMeetMac/AppModel.swift index 4a44b54..1194286 100644 --- a/Sources/VoiceMeetMac/AppModel.swift +++ b/Sources/VoiceMeetMac/AppModel.swift @@ -37,6 +37,8 @@ final class AppModel { let record = DeviceRecord() /// The window in front, as events: `device/app-focused`, `device/window-read` (`DeviceWindow`). let deviceWindow = DeviceWindow() + /// Live: a member's page watches this Mac now (`DeviceLive`). + let live = DeviceLive() private var presenceWatch: AnyCancellable? private var activity: NSObjectProtocol? private var microphoneHolders = 0 @@ -68,11 +70,24 @@ final class AppModel { presenceWatch = deviceEvents.$presence.removeDuplicates().sink { [weak self] _ in DispatchQueue.main.async { MainActor.assumeIsolated { self?.record.tick() } } } + record.sink = live.peer + live.decide = { [weak self] who in await self?.liveTracks(who: who) ?? ([], "the app is closing") } + live.sourcesChanged = { [weak self] sources in + guard let self, sources != self.record.live else { return } + self.record.live = sources + self.record.tick() + } + live.began = { [weak self] who, tracks in + self?.acts.began(.live, doing: "Watched live by \(who)", did: "Watched live by \(who)", detail: LiveSection.words(tracks)) ?? 0 + } + live.finished = { [weak self] act, failure in self?.acts.ended(act, failure: failure) } record.start() + live.start() } func shutDown() { endpoint.stop() + live.stop() acts.stop() deviceEvents.stop() deviceWindow.stop() @@ -119,11 +134,21 @@ final class AppModel { "microphone": microphone.statusLine, "presence": Self.presence().dictionary, "record": ["recording": record.isRecording, "paused": record.isPaused, "line": record.line], + "live": ["watchers": live.watchers.map(\.who)], ] case "record.state": return recordState() case "record.set": return try await setRecord(parameters) + case "live.state": + return live.state() + case "live.end": + // Ending a watch is no use of the Mac: always allowed. + guard let watcher = parameters["watcher"] as? String, live.watchers.contains(where: { $0.id == watcher }) else { + throw EndpointRefusal("live.end needs a watcher of live.state(): no such watcher now.") + } + live.end(watcher, why: "ended by a call to live.end") + return live.state() case "screen.snapshot": let display = try Screen.resolve(parameters["display"] as? Int) return try await act(display, what: "A still picture of the display \"\(display.title)\".", doing: "Taking a picture of the display “\(display.title)”", did: "Took a picture of the display “\(display.title)”") { @@ -208,7 +233,7 @@ final class AppModel { return [:] default: throw EndpointRefusal( - "No method \"\(method)\". The methods are: state, screen.snapshot, camera.snapshot, microphone.listen, speaker.play, speaker.say, accessibility.frontWindow, run, banner.show, call.ring, record.state, record.set." + "No method \"\(method)\". The methods are: state, screen.snapshot, camera.snapshot, microphone.listen, speaker.play, speaker.say, accessibility.frontWindow, run, banner.show, call.ring, record.state, record.set, live.state, live.end." ) } } @@ -231,6 +256,50 @@ final class AppModel { count(thing) } + // MARK: - Live + + /// The owner changed a choice in the panel: a watch that has a track whose thing is now Off + /// (or lost its permission) ends at once. Ask and Shared go on: the watch was approved. + func liveChoicesChanged() { + let cameras = Camera.devices.map(Shareable.camera) + let things = { (track: String) -> [Shareable] in + track == "camera-0" ? cameras : track == "mic-0" ? [.microphone] : Screen.displays.filter { "screen-\($0.id)" == track }.map(Shareable.display) + } + for watcher in live.watchers { + let off = watcher.tracks.first { track in things(track).contains { $0.sharing == .off || !$0.isPermitted } } + if let off { live.end(watcher.id, why: "the owner turned \(off) off") } + } + } + + /// The tracks a watcher gets: a watch is a use of each display, the cameras and the + /// microphone under their choices, through the same gate as every use (`UseGate`). Off: not + /// offered. Ask: one question for the whole watch. Shared: offered. Live shows one camera, + /// and goes by every camera's choice, as the record's switch does. + private func liveTracks(who: String) async -> (tracks: [String], reason: String) { + let gate = { (thing: Shareable) in + UseGate.decide(sharing: thing.sharing.rawValue, permitted: thing.isPermitted, approved: self.approval.isApproved(thing) != nil) + } + let cameras = Camera.devices.map(Shareable.camera) + let things: [(track: String, things: [Shareable])] = + Screen.displays.map { ("screen-\($0.id)", [Shareable.display($0)]) } + [("camera-0", cameras), ("mic-0", [.microphone])] + let gates = things.map { (track: $0.track, gate: UseGate.strictest($0.things.map(gate))) } + let asked = LiveTracks.asks(gates) + var approved = false + if asked { + let asking = zip(things, gates).filter { $0.1.gate == .ask }.flatMap { $0.0.things } + let names = asking.map(\.title).joined(separator: ", ") + do { + try await approval.ask(for: asking, does: "show this Mac live to \(who)", what: "\(who) wants to watch this Mac live: \(names). It lasts until they leave or you end it in the panel.") + approved = true + } catch { + log.info("live: \(who) is not approved: \(AppLog.describe(error))") + } + } + let tracks = LiveTracks.offered(gates, approved: approved) + for (item, _) in zip(things, gates) where tracks.contains(item.track) { item.things.forEach(count) } + return (tracks, LiveTracks.nothing(asked: asked, approved: approved)) + } + // MARK: - The record's switches /// The record's switches and its state, for `state()` and `record.state()`. Reading it is no use. diff --git a/Sources/VoiceMeetMac/Approval.swift b/Sources/VoiceMeetMac/Approval.swift index 06b7721..945f78f 100644 --- a/Sources/VoiceMeetMac/Approval.swift +++ b/Sources/VoiceMeetMac/Approval.swift @@ -26,14 +26,20 @@ final class Approval { /// Returns when the person approved; throws the refusal a caller reads when they did not. func ask(for thing: Shareable, what: String) async throws { + try await ask(for: [thing], does: thing.does, what: what) + } + + /// One question for several things (a live watch: the screens, the camera, the microphone). + /// "Approve for 30 Minutes" holds for each of them. + func ask(for things: [Shareable], does: String, what: String) async throws { // One question at a time: a second caller waits for the first answer, which may cover it. while isAsking { try await Task.sleep(nanoseconds: 200_000_000) } - if isApproved(thing) != nil { return } + if things.allSatisfy({ isApproved($0) != nil }) { return } isAsking = true defer { isAsking = false } let alert = NSAlert() - alert.messageText = "Your mind wants to \(thing.does)" + alert.messageText = "Your mind wants to \(does)" alert.informativeText = what alert.icon = NSApp.applicationIconImage alert.addButton(withTitle: "Approve Once") @@ -50,23 +56,23 @@ final class Approval { guard answer == .alertFirstButtonReturn || answer == .alertSecondButtonReturn else { let words = answer == .alertThirdButtonReturn ? "denied it" : "gave no answer in \(Int(Self.patience)) seconds" - log.info("to \(thing.does): the person \(words)") - throw EndpointRefusal("The Mac's owner \(words) to \(thing.does).") + log.info("to \(does): the person \(words)") + throw EndpointRefusal("The Mac's owner \(words) to \(does).") } do { - let reason = "approve that your mind may \(thing.does)" + let reason = "approve that your mind may \(does)" guard try await LAContext().evaluatePolicy(.deviceOwnerAuthentication, localizedReason: reason) else { - throw EndpointRefusal("The Mac's owner did not confirm with Touch ID to \(thing.does).") + throw EndpointRefusal("The Mac's owner did not confirm with Touch ID to \(does).") } } catch let refusal as EndpointRefusal { throw refusal } catch { - log.info("to \(thing.does): no confirmation: \(AppLog.describe(error))") - throw EndpointRefusal("The Mac's owner did not confirm with Touch ID to \(thing.does).") + log.info("to \(does): no confirmation: \(AppLog.describe(error))") + throw EndpointRefusal("The Mac's owner did not confirm with Touch ID to \(does).") } if answer == .alertSecondButtonReturn { - approvedUntil[thing.id] = Date().addingTimeInterval(Self.window) + for thing in things { approvedUntil[thing.id] = Date().addingTimeInterval(Self.window) } } - log.info("to \(thing.does): the person approved \(answer == .alertSecondButtonReturn ? "for 30 minutes" : "this one use")") + log.info("to \(does): the person approved \(answer == .alertSecondButtonReturn ? "for 30 minutes" : "this one use")") } } diff --git a/Sources/VoiceMeetMac/DeviceLive.swift b/Sources/VoiceMeetMac/DeviceLive.swift new file mode 100644 index 0000000..e581f69 --- /dev/null +++ b/Sources/VoiceMeetMac/DeviceLive.swift @@ -0,0 +1,181 @@ +import Live +import Record +import SwiftUI +import VoiceMeetCore + +/// LIVE: a member's page (record.) watches this Mac now: its screens, its camera, its +/// microphone, peer to peer over WebRTC (`Sources/Live`). The mind carries only the signalling, on +/// `/devices/`, through `bridge/device-live.mjs`, which runs while the Mac is lent. +/// +/// A watch is a use of the screen, the camera and the microphone under their switches, through +/// the same gate as every other use (`AppModel.liveTracks`): Off, the track is not offered; Ask, +/// the person is asked once for the watch; Shared, it is sent. While a watcher is connected, +/// Activity shows it as a running act, and the panel lists it with a button that ends it. +@MainActor +final class DeviceLive: ObservableObject { + struct Destination: Equatable { + let node: String + let iterateBin: String + let script: String + let mind: String + let name: String + } + + /// A connected watcher, for the panel and `live.state()`. + struct Watcher: Identifiable, Equatable { + let id: String + let who: String + let tracks: [String] + let since: Date + } + + @Published private(set) var watchers: [Watcher] = [] + + var destination: () -> Destination? = { nil } + /// The tracks a watcher gets, by the owner's switches, and the reason when it gets none. + var decide: (_ who: String) async -> (tracks: [String], reason: String) = { _ in ([], "nothing to show") } + /// The sources the watchers need changed: the recorder runs them (`DeviceRecord.live`). + var sourcesChanged: (Set) -> Void = { _ in } + var began: (_ who: String, _ tracks: [String]) -> Int = { _, _ in 0 } + var finished: (_ act: Int, _ failure: String?) -> Void = { _, _ in } + + /// The capture's sink: made once, at the start. + let peer = LivePeer() + private let log = AppLog("live") + private var bridge: LiveBridge? + private var bridgeTo: Destination? + private var bridgeEndedAt = Date.distantPast + private var acts: [String: Int] = [:] + private var timer: Timer? + + func start() { + peer.log = { line in AppLog("live").info(line) } + peer.decide = { [weak self] _, who, answer in + DispatchQueue.main.async { + MainActor.assumeIsolated { + guard let self else { return answer([], "the app is closing") } + Task { @MainActor in + let decided = await self.decide(who) + answer(decided.tracks, decided.reason) + } + } + } + } + peer.started = { [weak self] watcher, who, tracks in + DispatchQueue.main.async { MainActor.assumeIsolated { self?.watchBegan(watcher, who: who, tracks: tracks) } } + } + peer.ended = { [weak self] watcher, who, seconds, why in + DispatchQueue.main.async { MainActor.assumeIsolated { self?.watchEnded(watcher, why: why) } } + } + peer.changed = { [weak self] state in + let sources = Set(state.tracks.compactMap(RecordSource.of(track:))) + let connected = state.watches.compactMap { w in w.connected.map { Watcher(id: w.id, who: w.who, tracks: w.tracks, since: $0) } } + DispatchQueue.main.async { + MainActor.assumeIsolated { + guard let self else { return } + if connected != self.watchers { self.watchers = connected } + self.sourcesChanged(sources) + } + } + } + let timer = Timer(timeInterval: 2, repeats: true) { [weak self] _ in MainActor.assumeIsolated { self?.tick() } } + RunLoop.main.add(timer, forMode: .common) + self.timer = timer + tick() + } + + func stop() { + timer?.invalidate() + peer.endAll(why: "the Mac stopped") + bridge?.stop() + } + + /// The panel's button, or `live.end` on the endpoint. + func end(_ watcher: String, why: String) { + log.info("\(watcher.prefix(8)): ended, \(why)") + peer.end(watcher, why: why) + } + + /// For `live.state()` on the endpoint. + func state() -> [String: Any] { + [ + "watchers": watchers.map { ["watcher": $0.id, "who": $0.who, "tracks": $0.tracks, "since": Int($0.since.timeIntervalSince1970 * 1000)] }, + "signalling": bridge != nil, + ] + } + + /// Every 2 s: the bridge follows the lend. A bridge that ended starts again after 10 s. + private func tick() { + let destination = destination() + if destination != bridgeTo, let old = bridge { + peer.endAll(why: "the Mac is no longer lent to this mind") + old.stop() + (bridge, bridgeTo, bridgeEndedAt) = (nil, nil, Date()) + } + guard let destination, bridge == nil, Date().timeIntervalSince(bridgeEndedAt) > 10 else { return } + let made = LiveBridge( + node: destination.node, script: destination.script, + arguments: [destination.name, "--iterate", destination.iterateBin, "--mind", destination.mind], peer: peer) + made.said = { line in AppLog("live").info("bridge: \(line)") } + made.exited = { [weak self] status in + DispatchQueue.main.async { + MainActor.assumeIsolated { + guard let self, self.bridge === made else { return } + self.log.info("the bridge ended (status \(status))") + self.peer.endAll(why: "the signalling stopped") + (self.bridge, self.bridgeTo, self.bridgeEndedAt) = (nil, nil, Date()) + } + } + } + do { + try made.start() + (bridge, bridgeTo) = (made, destination) + log.info("the bridge listens on \(destination.mind) at /devices/\(destination.name)") + } catch { + log.error("the bridge did not start: \(AppLog.describe(error))") + bridgeEndedAt = Date() + } + } + + private func watchBegan(_ watcher: String, who: String, tracks: [String]) { + acts[watcher] = began(who, tracks) + } + + private func watchEnded(_ watcher: String, why: String) { + guard let act = acts.removeValue(forKey: watcher) else { return } + finished(act, why == "connection failed" ? why : nil) + } +} + +/// The panel's watchers, while there are any: who, what, since when, and a button that ends it. +struct LiveSection: View { + @ObservedObject var live: DeviceLive + + var body: some View { + VStack(alignment: .leading, spacing: 9) { + Text("WATCHED LIVE").font(.caption2.weight(.semibold)).foregroundStyle(.secondary) + ForEach(live.watchers) { watcher in + HStack(spacing: 10) { + Image(systemName: "eye.fill").font(.system(size: 14)).frame(width: 20).foregroundStyle(.red) + VStack(alignment: .leading, spacing: 1) { + Text(watcher.who).lineLimit(1).truncationMode(.middle) + Text("\(Self.words(watcher.tracks)) · since \(watcher.since.formatted(date: .omitted, time: .shortened))") + .font(.caption).foregroundStyle(.secondary).lineLimit(1) + } + Spacer(minLength: 8) + Button("End") { live.end(watcher.id, why: "the owner ended it") }.controlSize(.small) + } + } + } + } + + /// "the screen, the camera, the microphone" + static func words(_ tracks: [String]) -> String { + let screens = tracks.filter { $0.hasPrefix("screen-") }.count + var words: [String] = [] + if screens > 0 { words.append(screens == 1 ? "the screen" : "\(screens) screens") } + if tracks.contains("camera-0") { words.append("the camera") } + if tracks.contains("mic-0") { words.append("the microphone") } + return words.joined(separator: ", ") + } +} diff --git a/Sources/VoiceMeetMac/DeviceRecord.swift b/Sources/VoiceMeetMac/DeviceRecord.swift index 9207874..6d31576 100644 --- a/Sources/VoiceMeetMac/DeviceRecord.swift +++ b/Sources/VoiceMeetMac/DeviceRecord.swift @@ -54,6 +54,11 @@ final class DeviceRecord: ObservableObject { private(set) var tracks: [String] = [] var destination: () -> Destination? = { nil } + /// The sources live wants now (`DeviceLive`): the recorder runs them too, and writes nothing + /// of those the record does not want. + var live: Set = [] + /// Live's sink of the capture (`LivePeer`): every recorder hands it each frame and buffer. + var sink: CaptureSink? /// The device's presence: "active", "idle", "locked", "asleep" (`DeviceEvents.presence`). var presence: () -> String? = { nil } var onChange: () -> Void = {} @@ -64,6 +69,7 @@ final class DeviceRecord: ObservableObject { private var recorder: Recorder? private var recordsTo: Destination? private var running: Set = [] + private var watching: Set = [] private var paused = false private var uploader: Process? private var uploaderInput: FileHandle? @@ -119,37 +125,46 @@ final class DeviceRecord: ObservableObject { /// Every 2 s, at each switch, and at once when the presence changes (`AppModel`). func tick() { let destination = destination() - let wanted = destination == nil ? [] : Set(on.filter { source in Self.switches.first { $0.source == source }?.permission.isGranted ?? true }) + let permitted = { (sources: Set) in + destination == nil ? [] : Set(sources.filter { source in Self.switches.first { $0.source == source }?.permission.isGranted ?? true }) + } + // the record's switches, and live's watchers: the capture runs for both + let wanted = permitted(on), watched = permitted(live) // another mind, or none: the record follows the lend - if destination != recordsTo || wanted.isEmpty, let old = recorder { + if destination != recordsTo || wanted.union(watched).isEmpty, let old = recorder { lifecycle.async { old.stop() } recorder = nil - running = [] + (running, watching) = ([], []) } if destination != recordsTo { endUploader() } recordsTo = destination - if let destination, !wanted.isEmpty { + if let destination, !wanted.union(watched).isEmpty { if recorder == nil { let recorder = Recorder(options: RecordOptions(spool: Self.spool(for: destination.mind), device: destination.name)) + if let sink { recorder.add(sink: sink) } lifecycle.async { recorder.start() } self.recorder = recorder paused = false } + let recorder = recorder! if wanted != running { - let recorder = recorder! lifecycle.async { recorder.set(wanted) } running = wanted } + if watched != watching { + lifecycle.async { recorder.watch(watched) } + watching = watched + } if let state = presence() { let shouldPause = state == "locked" || state == "asleep" - if shouldPause != paused, let recorder { + if shouldPause != paused { paused = shouldPause lifecycle.async { recorder.pause(shouldPause) } } } } let waiting = destination.map { Self.waiting(in: Self.spool(for: $0.mind)) } ?? 0 - if let destination, recorder != nil || waiting > 0 { + if let destination, !running.isEmpty || waiting > 0 { if uploader == nil, Date().timeIntervalSince(uploaderEndedAt) > 10 { startUploader(destination) } } else { endUploader() @@ -161,7 +176,7 @@ final class DeviceRecord: ObservableObject { var parts: [String] = [] if destination == nil { parts.append(on.isEmpty ? "Off" : "Waits for the Mac to be lent to a mind") - } else if let recorder { + } else if let recorder, !running.isEmpty { let tracks = recorder.trackIDs.count parts.append(paused ? "Paused: \(presence() ?? "locked")" : "Recording \(tracks) track\(tracks == 1 ? "" : "s")") } else { @@ -170,9 +185,9 @@ final class DeviceRecord: ObservableObject { if waiting > 0 { parts.append("\(waiting) slot\(waiting == 1 ? "" : "s") waiting") } if let uploadFailure, waiting > 0 { parts.append("upload failing: \(uploadFailure)") } let line = parts.joined(separator: " · ") - let isRecording = recorder != nil + let isRecording = recorder != nil && !running.isEmpty let isPaused = isRecording && paused - let tracks = recorder?.trackIDs ?? [] + let tracks = isRecording ? recorder?.trackIDs ?? [] : [] let pausedBecause = isPaused ? presence() ?? "locked" : nil // the mark shows these: it changes with them let markChanges = isRecording != self.isRecording || isPaused != self.isPaused || tracks.count != self.tracks.count || pausedBecause != self.pausedBecause diff --git a/Sources/VoiceMeetMac/Panel.swift b/Sources/VoiceMeetMac/Panel.swift index e5fd09c..8d5441e 100644 --- a/Sources/VoiceMeetMac/Panel.swift +++ b/Sources/VoiceMeetMac/Panel.swift @@ -86,6 +86,7 @@ struct Panel: View { section("Cameras", of: .camera) section("This Mac", of: .microphone, .speaker, .banners, .accessibility, .commands) DeviceEventsSection(events: model.deviceEvents) + LiveWatching(live: model.live) RecordSection(record: model.record, window: model.deviceWindow, allow: allow) } .padding(.horizontal, 16) @@ -394,3 +395,12 @@ private struct LayerView: NSViewRepresentable { func updateNSView(_ view: NSView, context: Context) {} } + +/// The panel's live watchers: nothing while nobody watches. +private struct LiveWatching: View { + @ObservedObject var live: DeviceLive + + var body: some View { + if !live.watchers.isEmpty { LiveSection(live: live) } + } +} diff --git a/Sources/VoiceMeetMac/Permissions.swift b/Sources/VoiceMeetMac/Permissions.swift index 84fbad0..33e9708 100644 --- a/Sources/VoiceMeetMac/Permissions.swift +++ b/Sources/VoiceMeetMac/Permissions.swift @@ -179,6 +179,9 @@ struct Shareable: Identifiable, Hashable { static let speaker = Shareable(id: "speaker", power: .speaker, title: "Speakers", does: "play sound through the speakers", symbol: "speaker.wave.2") static let accessibility = Shareable(id: "accessibility", power: .accessibility, title: "Window text", does: "read the window in front", symbol: "text.viewfinder") static let commands = Shareable(id: "commands", power: .commands, title: "Commands", does: "run commands", symbol: "terminal") + /// A live watch: its act in Activity. It has no switch: the screen, the camera and the + /// microphone decide (`AppModel.liveTracks`). + static let live = Shareable(id: "live", power: .screen, title: "Live", does: "watch this Mac live", symbol: "eye") /// Ask and Shared both show: a button is pressed by the person alone. static let banners = Shareable(id: "banners", power: .banners, title: "Banners", does: "show banners on the screen", symbol: "bell.badge") diff --git a/Sources/live-run/main.swift b/Sources/live-run/main.swift new file mode 100644 index 0000000..e63e5c7 --- /dev/null +++ b/Sources/live-run/main.swift @@ -0,0 +1,148 @@ +import CoreGraphics +import Foundation +import LiveKitWebRTC +import Live +import Record +import VoiceMeetCore + +// live-run: live from a terminal, for tests: the app's own Record and Live libraries and its +// bridge, with no switch and no question (every watcher gets every track asked for here). +// +// live-run --device --node --bridge --iterate --mind
+// [--synthetic] [--screens] [--cameras] [--mics] [--record ] +// [--loopback-only] [--no-stun] [--seconds N] +// +// --record also records the same sources into a spool, as the app does with a watcher and the +// record on together. Real sources need the app's permissions: run it through the app's `run` +// method. stderr: one JSON line of statistics a second, and the libraries' log lines. +var arguments = Array(CommandLine.arguments.dropFirst()) +func value(_ flag: String) -> String? { + guard let i = arguments.firstIndex(of: flag), i + 1 < arguments.count else { return nil } + return arguments[i + 1] +} +guard let device = value("--device"), let node = value("--node"), let bridgeScript = value("--bridge"), let iterate = value("--iterate"), let mind = value("--mind") else { + FileHandle.standardError.write(Data("live-run --device --node --bridge --iterate --mind
[--synthetic] [--screens] [--cameras] [--mics] [--record ] [--loopback-only] [--no-stun] [--seconds N]\n".utf8)) + exit(2) +} +let started = Date() +let outLock = NSLock() +func say(_ line: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: line, options: [.sortedKeys, .withoutEscapingSlashes]) else { return } + outLock.withLock { FileHandle.standardError.write(data + Data("\n".utf8)) } +} +recordLog = { say(["log": $0]) } + +let sources = Set(RecordSource.allCases.filter { arguments.contains("--\($0.rawValue)") }) +let spool = value("--record") ?? NSTemporaryDirectory() + "live-run-\(ProcessInfo.processInfo.processIdentifier)" +let recorder = Recorder(options: RecordOptions(spool: URL(fileURLWithPath: spool), device: device)) +let peer = LivePeer(loopbackOnly: arguments.contains("--loopback-only"), stun: !arguments.contains("--no-stun")) +peer.log = { say(["log": $0]) } +recorder.add(sink: peer) +recorder.start() +if value("--record") != nil { recorder.set(sources) } + +/// The tracks of the sources asked for: each display, the camera, the microphone, the test picture and beep. +func tracks() -> [String] { + var displays = [CGDirectDisplayID](repeating: 0, count: 16) + var count: UInt32 = 0 + CGGetOnlineDisplayList(16, &displays, &count) + var tracks: [String] = [] + if sources.contains(.synthetic) { tracks += ["test-1", "test-2"] } + if sources.contains(.screens) { tracks += displays.prefix(Int(count)).map { "screen-\($0)" } } + if sources.contains(.cameras) { tracks.append("camera-0") } + if sources.contains(.mics) { tracks.append("mic-0") } + return tracks +} +peer.decide = { _, _, answer in answer(tracks(), "nothing to show") } +peer.changed = { watchers in recorder.watch(Set(watchers.tracks.compactMap(RecordSource.of(track:)))) } + +let bridge = LiveBridge(node: node, script: bridgeScript, arguments: [device, "--iterate", iterate, "--mind", mind], peer: peer) +peer.started = { watcher, who, tracks in + say(["event": "live-watch-started", "watcher": watcher, "who": who, "tracks": tracks]) +} +peer.ended = { watcher, who, seconds, why in + say(["event": "live-watch-ended", "watcher": watcher, "who": who, "seconds": seconds, "why": why]) +} +bridge.said = { say(["bridge": $0]) } +let bridgeEnded = DispatchSemaphore(value: 0) +bridge.exited = { status in + say(["event": "bridge exited", "status": Int(status)]) + bridgeEnded.signal() +} +try bridge.start() +say(["event": "ready", "device": device, "bridge": Int(bridge.pid), "sources": sources.map(\.rawValue).sorted(), "record": value("--record") ?? "no"]) + +// MARK: statistics, one line a second: CPU of this process, and per watcher and track what is sent + +func cpuSeconds() -> Double { + var usage = rusage() + getrusage(RUSAGE_SELF, &usage) + let t = { (v: timeval) in Double(v.tv_sec) + Double(v.tv_usec) / 1e6 } + return t(usage.ru_utime) + t(usage.ru_stime) +} +var cpuBefore = (seconds: cpuSeconds(), at: Date()) +var sentBefore: [String: (bytes: Double, frames: Double)] = [:] +let statsQueue = DispatchQueue(label: "stats") +let timer = DispatchSource.makeTimerSource(queue: statsQueue) +timer.schedule(deadline: .now() + 1, repeating: 1) +timer.setEventHandler { + let now = Date(), cpu = cpuSeconds() + let percent = (cpu - cpuBefore.seconds) / now.timeIntervalSince(cpuBefore.at) * 100 + let seconds = max(0.001, now.timeIntervalSince(cpuBefore.at)) + cpuBefore = (cpu, now) + var rows: [String: Any] = [:] + for watch in peer.state.watches { + guard let pc = peer.connection(watch.id) else { continue } + let done = DispatchSemaphore(value: 0) + pc.statistics { report in + let all = report.statistics + let number = { (s: LKRTCStatistics, k: String) -> Double in (s.values[k] as? NSNumber)?.doubleValue ?? 0 } + var tracks: [String: Any] = [:] + for out in all.values where out.type == "outbound-rtp" { + let track = (out.values["mediaSourceId"] as? String).flatMap { all[$0]?.values["trackIdentifier"] as? String } ?? "?" + let key = "\(watch.id)/\(track)", bytes = number(out, "bytesSent"), frames = number(out, "framesSent") + let before = sentBefore[key] ?? (0, 0) + sentBefore[key] = (bytes, frames) + var row: [String: Any] = ["kbps": Int((bytes - before.bytes) * 8 / 1000 / seconds)] + if out.values["kind"] as? String == "video" { + row["fps"] = ((frames - before.frames) / seconds * 10).rounded() / 10 + row["size"] = "\(Int(number(out, "frameWidth")))x\(Int(number(out, "frameHeight")))" + row["encoder"] = out.values["encoderImplementation"] as? String ?? "?" + row["limit"] = out.values["qualityLimitationReason"] as? String ?? "?" + } + tracks[track] = row + } + rows[String(watch.id.prefix(8))] = ["phase": "\(watch.phase)", "tracks": tracks] + done.signal() + } + _ = done.wait(timeout: .now() + 0.5) + } + say(["stats": ["t": Int(now.timeIntervalSince(started)), "cpu": Int(percent.rounded()), "recording": recorder.trackIDs, "watchers": rows]]) +} +timer.resume() + +// MARK: run until --seconds, SIGINT or SIGTERM; each watcher hears bye first + +func finish(_ why: String) { + DispatchQueue.global().async { + timer.cancel() + peer.endAll(why: "the device stopped") + bridge.stop() + _ = bridgeEnded.wait(timeout: .now() + 10) + recorder.stop() + say(["event": "stopped", "why": why, "seconds": Int(Date().timeIntervalSince(started))]) + exit(0) + } +} +signal(SIGINT, SIG_IGN) +signal(SIGTERM, SIG_IGN) +let signalSources = [SIGINT, SIGTERM].map { number -> DispatchSourceSignal in + let source = DispatchSource.makeSignalSource(signal: number, queue: .main) + source.setEventHandler { finish("signal \(number)") } + source.resume() + return source +} +if let seconds = value("--seconds").flatMap(Double.init) { + DispatchQueue.main.asyncAfter(deadline: .now() + seconds) { finish("time") } +} +withExtendedLifetime(signalSources) { RunLoop.main.run() } diff --git a/Support/Info.plist b/Support/Info.plist index b3c5e07..d66c20b 100644 --- a/Support/Info.plist +++ b/Support/Info.plist @@ -32,6 +32,8 @@ iterate lets your mind see the camera, and shows it in its menu. You can forbid it in the menu. NSMicrophoneUsageDescription iterate lets your mind hear the microphone. You can forbid it in the menu. + NSLocalNetworkUsageDescription + iterate connects directly to a person of your mind on this network who watches this Mac live. You can forbid the screen, the camera and the microphone in the menu. NSAppleEventsUsageDescription iterate runs commands for your mind; a command may ask another app to do something. You can forbid commands in the menu. diff --git a/Tests/RecordTests/RecordTests.swift b/Tests/RecordTests/RecordTests.swift index f068b27..1bb0992 100644 --- a/Tests/RecordTests/RecordTests.swift +++ b/Tests/RecordTests/RecordTests.swift @@ -247,3 +247,56 @@ final class SpoolTests: XCTestCase { XCTAssertTrue(second.opening(track: "screen-1", slot: t + 30_000)) } } + +final class CaptureTests: XCTestCase { + func testTheRecordKeeps10FramesASecondByTimeOutOf30() { + // 30 frames a second for 3 s, with a millisecond of jitter + var frames: [Double] = [] + for n in 0..<90 { frames.append(Double(n) / 30 + (n % 2 == 0 ? 0.001 : -0.001)) } + let kept = { (fps: Int) -> [Double] in + var last: Double? + var kept: [Double] = [] + for t in frames where CaptureRate.keeps(last: last, now: t, fps: fps) { + last = t + kept.append(t) + } + return kept + } + XCTAssertEqual(kept(10).count, 30) + XCTAssertEqual(kept(10).prefix(4).map { ($0 * 1000).rounded() }, [1, 99, 201, 299]) + // a camera of 30 at 15 + XCTAssertEqual(kept(15).count, 45) + // a clock that went back: kept + XCTAssertTrue(CaptureRate.keeps(last: 5, now: 4, fps: 10)) + XCTAssertEqual(CaptureRate.capture(record: 10, watched: false), 10) + XCTAssertEqual(CaptureRate.capture(record: 10, watched: true), 30) + } + + func testTheCaptureRunsForTheUnionOfRecordAndLive() { + let none: Set = [] + // live alone starts a source; the record and live together run it once + XCTAssertEqual(CapturePlan.changes(running: none, record: (none, none), live: (none, [.screens, .mics])), [.start(.screens), .start(.mics)]) + XCTAssertEqual(CapturePlan.changes(running: none, record: (none, [.screens]), live: (none, [.screens])), [.start(.screens)]) + // the first watcher of a source the record has: faster; the last one leaves: as before + XCTAssertEqual(CapturePlan.changes(running: [.screens], record: ([.screens], [.screens]), live: (none, [.screens])), [.rate(.screens)]) + XCTAssertEqual(CapturePlan.changes(running: [.screens], record: ([.screens], [.screens]), live: ([.screens], none)), [.rate(.screens)]) + // the last watcher of a source only live wanted: it stops + XCTAssertEqual(CapturePlan.changes(running: [.cameras], record: (none, none), live: ([.cameras], none)), [.stop(.cameras)]) + // the record turned off while somebody watches: its tracks end, the capture goes on + XCTAssertEqual(CapturePlan.changes(running: [.screens], record: ([.screens], none), live: ([.screens], [.screens])), [.stopRecording(.screens)]) + // the record turned on for a source live alone ran: it starts again, with its tracks + XCTAssertEqual(CapturePlan.changes(running: [.mics], record: (none, [.mics]), live: ([.mics], [.mics])), [.restart(.mics)]) + // stops first + XCTAssertEqual( + CapturePlan.changes(running: [.screens], record: ([.screens], [.cameras]), live: (none, none)), [.stop(.screens), .start(.cameras)]) + XCTAssertEqual(CapturePlan.changes(running: [.screens], record: ([.screens], [.screens]), live: (none, none)), []) + } + + func testTheSourceOfATrack() { + XCTAssertEqual(RecordSource.of(track: "screen-1"), .screens) + XCTAssertEqual(RecordSource.of(track: "camera-0"), .cameras) + XCTAssertEqual(RecordSource.of(track: "mic-0"), .mics) + XCTAssertEqual(RecordSource.of(track: "test-2"), .synthetic) + XCTAssertNil(RecordSource.of(track: "nothing")) + } +} diff --git a/Tests/VoiceMeetCoreTests/LiveRuleTests.swift b/Tests/VoiceMeetCoreTests/LiveRuleTests.swift new file mode 100644 index 0000000..514bfa0 --- /dev/null +++ b/Tests/VoiceMeetCoreTests/LiveRuleTests.swift @@ -0,0 +1,128 @@ +import XCTest + +@testable import VoiceMeetCore + +final class LiveTracksTests: XCTestCase { + func testOffAskAndSharedDecideEachTrack() { + let gates: [(track: String, gate: UseGate)] = [("screen-1", .go), ("screen-2", .off), ("camera-0", .ask), ("mic-0", .notPermitted)] + XCTAssertTrue(LiveTracks.asks(gates)) + XCTAssertEqual(LiveTracks.offered(gates, approved: true), ["screen-1", "camera-0"]) + XCTAssertEqual(LiveTracks.offered(gates, approved: false), ["screen-1"]) + XCTAssertFalse(LiveTracks.asks([("screen-1", .go), ("mic-0", .off)])) + XCTAssertEqual(LiveTracks.offered([("screen-1", .off), ("mic-0", .off)], approved: true), []) + XCTAssertEqual(LiveTracks.nothing(asked: true, approved: false), "the owner did not approve the watch") + XCTAssertTrue(LiveTracks.nothing(asked: false, approved: false).hasPrefix("nothing to show")) + } + + func testSeveralCamerasGoByTheStrictest() { + XCTAssertEqual(UseGate.strictest([.go, .ask]), .ask) + XCTAssertEqual(UseGate.strictest([.go, .notPermitted, .ask]), .notPermitted) + XCTAssertEqual(UseGate.strictest([.ask, .off]), .off) + XCTAssertEqual(UseGate.strictest([.go, .go]), .go) + XCTAssertEqual(UseGate.strictest([]), .off) // no camera + } +} + +final class LiveWatchersTests: XCTestCase { + let t0 = Date(timeIntervalSince1970: 1_800_000_000) + let c = { (n: Int) in LiveCandidate(candidate: "candidate:\(n) 1 udp 1 127.0.0.1 \(5000 + n) typ host", sdpMid: "0", sdpMLineIndex: 0) } + + /// A watcher from `want` to connected; answers the effects of each step. + private func connect(_ w: inout LiveWatchers, _ id: String, at t: Date) { + _ = w.want(from: id, who: "\(id)@example.com", now: t) + _ = w.decided(id, tracks: ["screen-1", "mic-0"], reason: "") + _ = w.offerMade(id, sdp: "offer", mids: ["0": "screen-1", "1": "mic-0"]) + _ = w.answer(from: id, sdp: "answer") + _ = w.connected(id, now: t) + } + + func testWantOfferAnswerCandidatesInOrderAndBye() { + var w = LiveWatchers() + XCTAssertEqual(w.want(from: "a", who: "ann@example.com", now: t0), [.decide(watcher: "a", who: "ann@example.com")]) + XCTAssertEqual(w.watch("a")?.phase, .deciding) + XCTAssertEqual(w.tracks, []) // nothing captured before the owner's answer + XCTAssertEqual(w.decided("a", tracks: ["screen-1", "camera-0"], reason: ""), [.makeOffer(watcher: "a", tracks: ["screen-1", "camera-0"])]) + XCTAssertEqual(w.tracks, ["screen-1", "camera-0"]) + // the device's candidates before its offer wait; the watcher's before its answer too + XCTAssertEqual(w.deviceCandidate("a", c(1)), []) + XCTAssertEqual(w.deviceCandidate("a", c(2)), []) + XCTAssertEqual(w.candidate(from: "a", c(9)), []) + XCTAssertEqual( + w.offerMade("a", sdp: "offer", mids: ["0": "screen-1", "1": "camera-0"]), + [.send(.offer(to: "a", sdp: "offer", tracks: ["0": "screen-1", "1": "camera-0"])), .send(.candidate(to: "a", c(1))), .send(.candidate(to: "a", c(2)))]) + XCTAssertEqual(w.deviceCandidate("a", c(3)), [.send(.candidate(to: "a", c(3)))]) + XCTAssertEqual(w.candidate(from: "a", c(10)), []) + XCTAssertEqual(w.answer(from: "a", sdp: "answer"), [.setAnswer(watcher: "a", sdp: "answer"), .addCandidate(watcher: "a", c(9)), .addCandidate(watcher: "a", c(10))]) + XCTAssertEqual(w.answer(from: "a", sdp: "again"), []) // one answer + XCTAssertEqual(w.candidate(from: "a", c(11)), [.addCandidate(watcher: "a", c(11))]) + XCTAssertEqual(w.connected("a", now: t0), [.started(watcher: "a", who: "ann@example.com", tracks: ["screen-1", "camera-0"])]) + XCTAssertEqual(w.connected("a", now: t0), []) // once + // the watcher leaves: no bye back, the watch's end with its seconds + XCTAssertEqual( + w.end("a", why: "the watcher left", tell: false, now: t0.addingTimeInterval(42.4)), + [.close(watcher: "a"), .ended(watcher: "a", who: "ann@example.com", seconds: 42, why: "the watcher left")]) + XCTAssertEqual(w.watches, []) + XCTAssertEqual(w.end("a", why: "again", tell: true, now: t0), []) + } + + func testNothingToShowIsAByeWithTheReason() { + var w = LiveWatchers() + _ = w.want(from: "a", who: "ann", now: t0) + XCTAssertEqual(w.decided("a", tracks: [], reason: "nothing to show"), [.send(.bye(to: "a", reason: "nothing to show"))]) + XCTAssertEqual(w.watches, []) + } + + func testAWatcherWhoLeftBeforeTheAnswerIsForgotten() { + var w = LiveWatchers() + _ = w.want(from: "a", who: "ann", now: t0) + // it left while the owner was asked: no act, no kept event + XCTAssertEqual(w.end("a", why: "the watcher left", tell: false, now: t0), [.close(watcher: "a")]) + XCTAssertEqual(w.decided("a", tracks: ["screen-1"], reason: ""), []) + } + + func testASecondWatcherAndTheFifthIsRefused() { + var w = LiveWatchers() + connect(&w, "a", at: t0) + connect(&w, "b", at: t0) + XCTAssertEqual(w.watches.map(\.phase), [.connected, .connected]) + _ = w.want(from: "c", who: "c", now: t0) + _ = w.want(from: "d", who: "d", now: t0) + XCTAssertEqual(w.want(from: "e", who: "e", now: t0), [.send(.bye(to: "e", reason: "too many watchers (4)"))]) + XCTAssertEqual(w.watches.map(\.id), ["a", "b", "c", "d"]) + // one leaves: the next one may watch + _ = w.end("c", why: "the watcher left", tell: false, now: t0) + XCTAssertEqual(w.want(from: "e", who: "e", now: t0), [.decide(watcher: "e", who: "e")]) + } + + func testAskingAgainEndsTheWatchBefore() { + var w = LiveWatchers() + connect(&w, "a", at: t0) + XCTAssertEqual( + w.want(from: "a", who: "a@example.com", now: t0.addingTimeInterval(5)), + [.close(watcher: "a"), .ended(watcher: "a", who: "a@example.com", seconds: 5, why: "asked again"), .decide(watcher: "a", who: "a@example.com")]) + XCTAssertEqual(w.watch("a")?.phase, .deciding) + } + + func testTheDeviceEndsEveryWatchAndSaysBye() { + var w = LiveWatchers() + connect(&w, "a", at: t0) + _ = w.want(from: "b", who: "b", now: t0) + XCTAssertEqual( + w.endAll(why: "the device stopped", now: t0.addingTimeInterval(10)), + [ + .close(watcher: "a"), .send(.bye(to: "a", reason: "the device stopped")), + .ended(watcher: "a", who: "a@example.com", seconds: 10, why: "the device stopped"), + .close(watcher: "b"), .send(.bye(to: "b", reason: "the device stopped")), + ]) + XCTAssertEqual(w.tracks, []) + } + + func testSignalsOfAStrangerChangeNothing() { + var w = LiveWatchers() + XCTAssertEqual(w.answer(from: "x", sdp: "answer"), []) + XCTAssertEqual(w.candidate(from: "x", c(1)), []) + XCTAssertEqual(w.connected("x", now: t0), []) + XCTAssertEqual(w.deviceCandidate("x", c(1)), []) + XCTAssertEqual(w.watches, []) + } +} diff --git a/bridge/device-live.mjs b/bridge/device-live.mjs new file mode 100644 index 0000000..29283a6 --- /dev/null +++ b/bridge/device-live.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +// device-live.mjs: the signalling of live, between this Mac and the watchers' pages, through the +// context /devices/ of the mind (README, "Live"). Only signalling goes through the mind: the +// pictures and the sound go peer to peer. +// +// node device-live.mjs --iterate --mind
+// +// The app (Sources/VoiceMeetMac/DeviceLive.swift, through Sources/Live/Bridge.swift) starts it +// while the Mac is lent, and closes stdin to end it. It connects to the mind once, as the person +// (the iterate CLI's own sign-in: the Mac's key lends and does nothing else). +// +// - stdout: `{ iceServers }` first and again at half their lifetime: what `iceServers()` of the +// facet `record` on /devices/ answers (STUN, and TURN when the mind has it); without that +// facet the app keeps STUN alone. Then each `device/live-signal` whose `to` is "device", as +// `{ signal: { from, want | description | candidate | bye }, who }`: `who` is the event's +// source, as an email or a short name. `{ failed }` lines are for the app's log. +// - stdin: `{ signal: { to, … } }` becomes an ephemeral `device/live-signal { from: "device", to, … }`, +// appended in order (an offer before its candidates). `{ started }` and `{ ended }` become the +// kept events `device/live-watch-started { watcher, who, tracks }` and +// `device/live-watch-ended { watcher, who, seconds, why }`. +import { dirname, join } from "node:path"; +import { createInterface } from "node:readline"; +import { pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; + +const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { iterate: { type: "string" }, mind: { type: "string" } }, +}); +const name = positionals[0]; +if (!name || !values.iterate || !values.mind) + throw new Error("node device-live.mjs --iterate --mind
"); +const say = (line) => process.stdout.write(`${JSON.stringify(line)}\n`); +const failed = (what, error) => say({ failed: `${what}: ${String(error?.message ?? error).slice(0, 300)}` }); +const SIGNAL = "device/live-signal"; + +const { addressOf, connectMind } = await import( + pathToFileURL(join(dirname(dirname(values.iterate)), "dist/node.mjs")).href +); +const { itx } = await connectMind({ address: addressOf(values.mind), signIn: false }); +const device = itx.cd(`/devices/${name}`); + +// The addresses a connection may use, before any watcher; again at half their lifetime. +async function iceServers() { + let ttl = 3600; + try { + const answer = await Promise.race([ + device.facets.get("record").iceServers(), + new Promise((_, reject) => setTimeout(() => reject(new Error("no answer in 5 s")), 5000)), + ]); + const servers = JSON.parse(JSON.stringify(answer?.iceServers ?? [])); + if (servers.length) say({ iceServers: servers }); + ttl = answer?.ttlSeconds ?? ttl; + } catch (error) { + failed("ICE servers, STUN alone", error); + } + setTimeout(iceServers, Math.max(60, ttl / 2) * 1000).unref(); +} +await iceServers(); + +/** Who asked: the principal's email, else its actor, else the origin. */ +function whoOf(source) { + const principal = source?.principal ?? source?.onBehalfOf?.principal; + return String(principal?.email ?? principal?.actor ?? source?.origin ?? "someone").slice(0, 100); +} + +// In: what the watchers say to the device. +const subscription = await device.subscribe({ + name: `device-live-${process.pid}-${Date.now().toString(36)}`, + consumes: [SIGNAL], + target: (batch) => { + for (const event of JSON.parse(JSON.stringify(batch))) { + const payload = event?.payload; + if (event?.type !== SIGNAL || payload?.to !== "device" || !payload.from) continue; + const { to, at, ...signal } = payload; + say({ signal, who: whoOf(event.source) }); + } + }, +}); + +// Out: what the device says, one append at a time, in order. +let chain = Promise.resolve(); +const append = (event) => { + chain = chain.then(() => device.append(event)).catch((error) => failed(event.type, error)); +}; +for await (const text of createInterface({ input: process.stdin })) { + if (!text) continue; + const line = JSON.parse(text); + if (line.signal) append({ type: SIGNAL, ephemeral: true, payload: { from: "device", ...line.signal } }); + for (const kind of ["started", "ended"]) { + if (!line[kind]) continue; + const type = `device/live-watch-${kind}`; + append({ type, idempotencyKey: `${type}/${line[kind].watcher}`, payload: line[kind] }); + } +} +// stdin closed: the app is gone, or the Mac is no longer lent. The last byes go out first. +await chain; +subscription?.[Symbol.dispose]?.(); +process.exit(0); diff --git a/bridge/mac-endpoint.ts b/bridge/mac-endpoint.ts index 4d91e0a..a2d62e0 100644 --- a/bridge/mac-endpoint.ts +++ b/bridge/mac-endpoint.ts @@ -21,7 +21,7 @@ import { join } from "node:path"; * characters ("a rewrite rule's description is one line: a string of at most 500 chars"), and the * lend then never goes live: keep this well under it. The detail is in `instructions` below. */ export const description = - "A person's Mac as an endpoint device, through its iterate menu-bar app. Inputs: screen, camera, microphone, the front window as text. Outputs: speakers, and banners on the screen with buttons (banner.show; call.ring looks like a call). Compute: commands. A record into the mind: record.state, record.set. A person is often near it: state().presence says if one is likely there now. The owner sets each thing to off, ask or shared, and sees each use in the menu bar."; + "A person's Mac as an endpoint device, through its iterate menu-bar app. Inputs: screen, camera, microphone, the front window as text. Outputs: speakers, and banners on the screen with buttons (banner.show; call.ring looks like a call). Compute: commands. A record into the mind: record.state, record.set. Live: live.state, live.end. A person is often near it: state().presence says if one is likely there now. The owner sets each thing to off, ask or shared, and sees each use in the menu bar."; const socketPath = join(homedir(), "Library/Application Support/voice-meet-mac/endpoint.sock"); @@ -201,6 +201,12 @@ type RecordState = { device?: string; }; +/** Who watches this Mac live now (`live.state`), as DeviceLive.swift gives it. */ +type LiveState = { + watchers: { watcher: string; who: string; tracks: string[]; since: number }[]; + signalling: boolean; +}; + /** THE PROVIDED OBJECT: what `iterate provide` lends. Each call fails with the app's own words * when the owner has the thing off, does not approve the use, or macOS has not given the app the * permission. */ @@ -280,6 +286,15 @@ export default async function macEndpoint() { * sources turned off stay off when a source turned on is then refused. */ set: (input: RecordSwitches) => ask("record.set", input) as Promise, }, + live: { + /** Who watches this Mac live now: each watcher's id, who it is (an email or a name), its + * tracks (`screen-`, `camera-0`, `mic-0`) and since when (Unix ms). A watch starts + * from a member's page (record.), never from here; the owner's choices for the + * screen, the camera and the microphone decide what it gets. Reading it is no use. */ + state: () => ask("live.state") as Promise, + /** Ends one watch now: the page hears `bye`. Always allowed. */ + end: (input: { watcher: string }) => ask("live.end", input) as Promise, + }, call: { /** A banner that looks like a call: `from` (who calls; the agent's path when none) and * `reason`, Decline and Accept as round phone buttons, the ring sound, 30 s. Accept opens @@ -292,7 +307,7 @@ export default async function macEndpoint() { return { instructions: `${description} Say what you looked at or did, and why.`, types: - "state(): Promise<{ things: { thing: string; kind: 'screen' | 'camera' | 'microphone' | 'speaker' | 'banners' | 'accessibility' | 'commands'; name: string; sharing: 'off' | 'ask' | 'shared'; permitted: boolean; uses: number; display?: number }[]; microphone: string; presence: { idleSeconds: number; screenLocked: boolean; displayAsleep: boolean }; record: { recording: boolean; paused: boolean; line: string } }>; screen.snapshot(input?: { display?: number; maxWidth?: number }): Promise<{ jpeg: Uint8Array; pixelWidth: number; pixelHeight: number }>; camera.snapshot(input?: { camera?: string; maxWidth?: number }): Promise<{ jpeg: Uint8Array; pixelWidth: number; pixelHeight: number }>; microphone.listen(input?: { seconds?: number }): Promise<{ wav: Uint8Array; seconds: number }>; speaker.play(sound: Uint8Array): Promise<{ seconds: number }>; speaker.say(words: string): Promise<{ characters: number }>; accessibility.frontWindow(input?: { maxDepth?: number; maxNodes?: number }): Promise<{ application: string; bundleIdentifier: string; window?: object; truncated: boolean }>; run(argv: string[], options?: { stdin?: string | Uint8Array; timeoutSeconds?: number }): Promise<{ exitCode: number; timedOut: boolean; stdout: Uint8Array; stderr: Uint8Array }>; appleScript(source: string, options?: { timeoutSeconds?: number }): Promise<{ exitCode: number; timedOut: boolean; stdout: string; stderr: string }>; swift(source: string, options?: { timeoutSeconds?: number }): Promise<{ exitCode: number; timedOut: boolean; stdout: string; stderr: string }>; banner.show(input: { title: string; body?: string; buttons?: { label: string; style?: 'accept' | 'decline' | 'plain'; symbol?: string; open?: string }[]; sound?: 'ring' | 'chime' | 'none'; seconds?: number }): Promise<{ pressed: string } | { pressed: null; why: 'no answer' | 'dismissed' | 'replaced' | 'off'; words: string }>; call.ring(input: { agent: string; from?: string; reason?: string; url?: string }): Promise<{ accepted: true } | { accepted: false; why: 'declined' | 'no answer' | 'replaced' | 'off'; words: string }>; record.state(): Promise; record.set(input: { screens?: boolean; cameras?: boolean; mics?: boolean; speakers?: boolean; windowText?: boolean }): Promise (a source turned on is a use under its power's choice; off is always allowed); type RecordState = { sources: { screens: boolean; cameras: boolean; mics: boolean; speakers: boolean }; windowText: boolean; recording: boolean; paused: boolean; line: string; tracks: string[]; waitingSlots: number; uploadFailure?: string; mind?: string; device?: string };", + "state(): Promise<{ things: { thing: string; kind: 'screen' | 'camera' | 'microphone' | 'speaker' | 'banners' | 'accessibility' | 'commands'; name: string; sharing: 'off' | 'ask' | 'shared'; permitted: boolean; uses: number; display?: number }[]; microphone: string; presence: { idleSeconds: number; screenLocked: boolean; displayAsleep: boolean }; record: { recording: boolean; paused: boolean; line: string }; live: { watchers: string[] } }>; screen.snapshot(input?: { display?: number; maxWidth?: number }): Promise<{ jpeg: Uint8Array; pixelWidth: number; pixelHeight: number }>; camera.snapshot(input?: { camera?: string; maxWidth?: number }): Promise<{ jpeg: Uint8Array; pixelWidth: number; pixelHeight: number }>; microphone.listen(input?: { seconds?: number }): Promise<{ wav: Uint8Array; seconds: number }>; speaker.play(sound: Uint8Array): Promise<{ seconds: number }>; speaker.say(words: string): Promise<{ characters: number }>; accessibility.frontWindow(input?: { maxDepth?: number; maxNodes?: number }): Promise<{ application: string; bundleIdentifier: string; window?: object; truncated: boolean }>; run(argv: string[], options?: { stdin?: string | Uint8Array; timeoutSeconds?: number }): Promise<{ exitCode: number; timedOut: boolean; stdout: Uint8Array; stderr: Uint8Array }>; appleScript(source: string, options?: { timeoutSeconds?: number }): Promise<{ exitCode: number; timedOut: boolean; stdout: string; stderr: string }>; swift(source: string, options?: { timeoutSeconds?: number }): Promise<{ exitCode: number; timedOut: boolean; stdout: string; stderr: string }>; banner.show(input: { title: string; body?: string; buttons?: { label: string; style?: 'accept' | 'decline' | 'plain'; symbol?: string; open?: string }[]; sound?: 'ring' | 'chime' | 'none'; seconds?: number }): Promise<{ pressed: string } | { pressed: null; why: 'no answer' | 'dismissed' | 'replaced' | 'off'; words: string }>; call.ring(input: { agent: string; from?: string; reason?: string; url?: string }): Promise<{ accepted: true } | { accepted: false; why: 'declined' | 'no answer' | 'replaced' | 'off'; words: string }>; record.state(): Promise; record.set(input: { screens?: boolean; cameras?: boolean; mics?: boolean; speakers?: boolean; windowText?: boolean }): Promise (a source turned on is a use under its power's choice; off is always allowed); live.state(): Promise; live.end(input: { watcher: string }): Promise (ends a watch; always allowed); type LiveState = { watchers: { watcher: string; who: string; tracks: string[]; since: number }[]; signalling: boolean }; type RecordState = { sources: { screens: boolean; cameras: boolean; mics: boolean; speakers: boolean }; windowText: boolean; recording: boolean; paused: boolean; line: string; tracks: string[]; waitingSlots: number; uploadFailure?: string; mind?: string; device?: string };", }; }, }; diff --git a/run-on-mac.sh b/run-on-mac.sh index d838f8c..316f9e9 100755 --- a/run-on-mac.sh +++ b/run-on-mac.sh @@ -131,6 +131,7 @@ sign_with_local_identity() { list_signing_keychain add || return 1 trap 'list_signing_keychain remove' EXIT signed=0 + codesign --force --sign "$signing_name" "$app/Contents/Frameworks/LiveKitWebRTC.framework" 2>/dev/null || signed=1 codesign --force --sign "$signing_name" "$app" 2>/dev/null || signed=1 list_signing_keychain remove trap - EXIT @@ -142,6 +143,7 @@ sign_app() { echo "Signed with the local certificate \"$signing_name\": permissions stay through rebuilds." return 0 fi + codesign --force --sign - "$app/Contents/Frameworks/LiveKitWebRTC.framework" codesign --force --sign - "$app" echo "Signed ad hoc: macOS asks for the microphone again after each rebuild." } @@ -239,6 +241,11 @@ if [ -n "${ITERATE_MAC_APP_VERSION:-}" ]; then fi mkdir -p "$app/Contents/Resources" cp Support/AppIcon.icns "$app/Contents/Resources/AppIcon.icns" +# Live's WebRTC library (Package.swift, LiveKitWebRTC): a framework beside the executable, found +# through an @rpath. ditto keeps the framework's symbolic links. +mkdir -p "$app/Contents/Frameworks" +ditto "$bin_path/LiveKitWebRTC.framework" "$app/Contents/Frameworks/LiveKitWebRTC.framework" +install_name_tool -add_rpath "@executable_path/../Frameworks" "$app/Contents/MacOS/iterate" sign_app echo "Built $(pwd)/$app"