diff --git a/README.md b/README.md index ee024e7..568e3ba 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,29 @@ session.startRunning() // frames flow through your existing AVCaptureVideoDataOutput delegates ``` +## iOS Simulator apps (Xcode 26/27) + +The iOS Simulator's own capture stack has no path to Mac cameras: in iOS 26.5 +and iOS 27 simulators `AVCaptureDevice` lists nothing, not even the Mac's +built-in camera. SimulatorCamera therefore ships a small library that you load +into your app **in the Simulator only**. Your app still imports nothing. + +1. Run the SimulatorCamera Mac app and pick a source (or use + `simcamctl feed --qr "TEXT"` / `--image PATH` / `--pattern` / `--camera`). +2. In Xcode: *Edit Scheme › Run › Arguments › Environment Variables*, add + `DYLD_INSERT_LIBRARIES` = `/Applications/SimulatorCamera.app/Contents/Resources/libSimCamInject.dylib`. + From the command line: `SIMCTL_CHILD_DYLD_INSERT_LIBRARIES= xcrun simctl launch booted `. +3. Run on a Simulator. `AVCaptureDevice.default(for: .video)` returns + "SimulatorCamera"; frames reach `AVCaptureVideoDataOutput`, + `AVCaptureVideoPreviewLayer`, QR codes reach `AVCaptureMetadataOutput`, and + `AVCapturePhotoOutput` returns JPEGs. Without the Mac app running you see a + built-in test pattern. + +Frames travel through `/private/tmp/SimulatorCamera/frame.bin` +(`SimulatorInjection/SCFrameStore.h`). Video frames are BGRA; apps that force a +420 YpCbCr format get BGRA anyway. Not supported yet: barcode types other than +QR, depth, multi-cam, movie file output. + ## Picking a source Open the SimulatorCamera Mac app: diff --git a/Shared/FrameStore.swift b/Shared/FrameStore.swift new file mode 100644 index 0000000..5c31593 --- /dev/null +++ b/Shared/FrameStore.swift @@ -0,0 +1,83 @@ +// +// FrameStore.swift +// SimulatorCamera +// +// Publishes the newest frame to /private/tmp/SimulatorCamera/frame.bin, where +// the simulator injection library (SimulatorInjection/SimCamInject.m) reads +// it. This is how frames reach iOS Simulator apps: the Simulator's own +// capture stack has no host-camera path (verified on Xcode 27, iOS 26.5/27), +// so a CMIO virtual camera alone is invisible to simulator apps. +// Layout must match SimulatorInjection/SCFrameStore.h. +// + +import CoreVideo +import Foundation + +final class FrameStore: @unchecked Sendable { + static let directory = "/private/tmp/SimulatorCamera" + static let path = directory + "/frame.bin" + static let magic: UInt32 = 0x3146_4353 + static let headerSize = 64 + static let maxWidth = 1920, maxHeight = 1080 + static let fileSize = headerSize + maxWidth * maxHeight * 4 + + private var base: UnsafeMutableRawPointer? + private var seq: UInt64 = 0 + private let lock = NSLock() + + init() {} + + deinit { + if let base { munmap(base, Self.fileSize) } + } + + private func mapIfNeeded() -> UnsafeMutableRawPointer? { + if let base { return base } + try? FileManager.default.createDirectory(atPath: Self.directory, withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o755]) + let fd = open(Self.path, O_RDWR | O_CREAT, 0o644) + guard fd >= 0 else { return nil } + defer { close(fd) } + guard ftruncate(fd, off_t(Self.fileSize)) == 0 else { return nil } + let p = mmap(nil, Self.fileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) + guard let p, p != MAP_FAILED else { return nil } + base = p + // Continue from whatever sequence a previous writer left, kept even. + let old = p.load(fromByteOffset: 24, as: UInt64.self) + seq = old &+ (old & 1) + return p + } + + /// Write one BGRA frame. Frames larger than 1920x1080 are skipped. + func write(width: Int, height: Int, bytesPerRow: Int, bgra: UnsafeRawPointer, sourceKind: UInt32, time: Double) { + guard width > 0, height > 0, width <= Self.maxWidth, height <= Self.maxHeight, + bytesPerRow >= width * 4, bytesPerRow * height <= Self.maxWidth * Self.maxHeight * 4 else { return } + lock.lock(); defer { lock.unlock() } + guard let p = mapIfNeeded() else { return } + seq &+= 1 // odd: write in progress + p.storeBytes(of: seq, toByteOffset: 24, as: UInt64.self) + OSMemoryBarrier() + p.storeBytes(of: Self.magic, toByteOffset: 0, as: UInt32.self) + p.storeBytes(of: UInt32(1), toByteOffset: 4, as: UInt32.self) + p.storeBytes(of: UInt32(width), toByteOffset: 8, as: UInt32.self) + p.storeBytes(of: UInt32(height), toByteOffset: 12, as: UInt32.self) + p.storeBytes(of: UInt32(bytesPerRow), toByteOffset: 16, as: UInt32.self) + p.storeBytes(of: sourceKind, toByteOffset: 20, as: UInt32.self) + p.storeBytes(of: time, toByteOffset: 32, as: Double.self) + memcpy(p + Self.headerSize, bgra, bytesPerRow * height) + OSMemoryBarrier() + seq &+= 1 // even: complete + p.storeBytes(of: seq, toByteOffset: 24, as: UInt64.self) + } + + func write(_ pixelBuffer: CVPixelBuffer, sourceKind: UInt32) { + guard CVPixelBufferGetPixelFormatType(pixelBuffer) == kCVPixelFormatType_32BGRA else { return } + CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) } + guard let base = CVPixelBufferGetBaseAddress(pixelBuffer) else { return } + // Wall clock: readers drop frames more than 3 s old. + write(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer), + bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), bgra: base, + sourceKind: sourceKind, time: Date().timeIntervalSince1970) + } +} diff --git a/SimulatorCamera.xcodeproj/project.pbxproj b/SimulatorCamera.xcodeproj/project.pbxproj index 362eaaf..3e0b3fc 100644 --- a/SimulatorCamera.xcodeproj/project.pbxproj +++ b/SimulatorCamera.xcodeproj/project.pbxproj @@ -9,9 +9,11 @@ /* Begin PBXBuildFile section */ 07A2A4651AA575FEBF94D191 /* SimulatorCameraProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 06AC788CDB1F5546EF813A18 /* SimulatorCameraProvider.swift */; }; 0B0F268D9B7EE26F8904395C /* SourceManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5009C01A5443FF9242C25F78 /* SourceManager.swift */; }; + 11CF13DB224856F4BD57F8C1 /* FrameStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 591A73ED75C2C97CCA46305C /* FrameStore.swift */; }; 123734FDBD4B874EC803EC0B /* CameraContract.swift in Sources */ = {isa = PBXBuildFile; fileRef = EED1C6F161452746D401A561 /* CameraContract.swift */; }; 15791A539F2708C098E31D62 /* ImageSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = EA6DB0DCE94EADD660FAB81A /* ImageSource.swift */; }; 179FDC3A8B0E644F699454C4 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79EB81F6F0FA26FC8C0D512B /* main.swift */; }; + 1DC705930BDB7DCA604A0DB9 /* TestPatternSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 46D46520BA4BCC428A6C0794 /* TestPatternSource.swift */; }; 2AB97A0BCC2C375207D8AFD6 /* FrameRaster.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7F22D3D2109AC909C586B1F9 /* FrameRaster.swift */; }; 30DCA34043EAA4DDA7E43A57 /* CMIOSinkClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = B2627A8C5CB8687217B22436 /* CMIOSinkClient.swift */; }; 313CD0876911B414C8C71B91 /* FrameRaster.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7F22D3D2109AC909C586B1F9 /* FrameRaster.swift */; }; @@ -28,6 +30,7 @@ A03B198646B946D161279AE4 /* ExtensionController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9F7FE73205CB332230563010 /* ExtensionController.swift */; }; A17CABFD712A0F8A6992E430 /* CameraContractTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 208AA3693AAB57C41B83F610 /* CameraContractTests.swift */; }; A86A15DD7CB53CE6EB144F28 /* CMIOSinkClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = B2627A8C5CB8687217B22436 /* CMIOSinkClient.swift */; }; + AD45CBDEBF2591469C97616B /* FrameStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 591A73ED75C2C97CCA46305C /* FrameStore.swift */; }; B99D036AA1563999AD109EBB /* CameraContract.swift in Sources */ = {isa = PBXBuildFile; fileRef = EED1C6F161452746D401A561 /* CameraContract.swift */; }; BE593A54205871224550F5FF /* CameraContract.swift in Sources */ = {isa = PBXBuildFile; fileRef = EED1C6F161452746D401A561 /* CameraContract.swift */; }; BFBE13333CF067BDB88CB825 /* QRRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 382C35DD80E3AE7C84200CC3 /* QRRenderer.swift */; }; @@ -36,9 +39,12 @@ CA28817E141686080C6B6B62 /* FrameRaster.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7F22D3D2109AC909C586B1F9 /* FrameRaster.swift */; }; D28E5DD221A2F8ACAEFD1D9A /* QRRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 382C35DD80E3AE7C84200CC3 /* QRRenderer.swift */; }; D3B54EFD8EC9634626E4351F /* MacCameraSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F36CFC13EBA8752E4B0A6C2 /* MacCameraSource.swift */; }; + D7400DD5957DCED5E222054E /* FeedCommand.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6104DACFC118A2330797BEC2 /* FeedCommand.swift */; }; D97A5DF273D525682867329C /* CMIOSinkClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = B2627A8C5CB8687217B22436 /* CMIOSinkClient.swift */; }; + E3A5553E615353760AFA72C9 /* FrameStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 591A73ED75C2C97CCA46305C /* FrameStore.swift */; }; EA24C3B90C006BB89A1CDC6B /* CameraContract.swift in Sources */ = {isa = PBXBuildFile; fileRef = EED1C6F161452746D401A561 /* CameraContract.swift */; }; F6D3FCD3CDF09C4E6AB84FB0 /* VideoFileSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 197CD7BA6EB552D5C96D7431 /* VideoFileSource.swift */; }; + F8617E1FC8ABB421235E3C73 /* FrameStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 591A73ED75C2C97CCA46305C /* FrameStore.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -75,9 +81,12 @@ 34DBFC0A082BB7CE216717D6 /* main.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = main.swift; sourceTree = ""; }; 37304038FE36D2D1035F449C /* CameraLink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraLink.swift; sourceTree = ""; }; 382C35DD80E3AE7C84200CC3 /* QRRenderer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QRRenderer.swift; sourceTree = ""; }; + 46D46520BA4BCC428A6C0794 /* TestPatternSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestPatternSource.swift; sourceTree = ""; }; 4F36CFC13EBA8752E4B0A6C2 /* MacCameraSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MacCameraSource.swift; sourceTree = ""; }; 5009C01A5443FF9242C25F78 /* SourceManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SourceManager.swift; sourceTree = ""; }; 5543F1512AE6614C0DE4861D /* QRSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QRSource.swift; sourceTree = ""; }; + 591A73ED75C2C97CCA46305C /* FrameStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrameStore.swift; sourceTree = ""; }; + 6104DACFC118A2330797BEC2 /* FeedCommand.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeedCommand.swift; sourceTree = ""; }; 79EB81F6F0FA26FC8C0D512B /* main.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = main.swift; sourceTree = ""; }; 7F22D3D2109AC909C586B1F9 /* FrameRaster.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrameRaster.swift; sourceTree = ""; }; 9F7FE73205CB332230563010 /* ExtensionController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExtensionController.swift; sourceTree = ""; }; @@ -103,6 +112,7 @@ 272BD53EC752678E900DC201 /* simcamctl */ = { isa = PBXGroup; children = ( + 6104DACFC118A2330797BEC2 /* FeedCommand.swift */, 34DBFC0A082BB7CE216717D6 /* main.swift */, 17E2492228A96A265315275C /* SimCamCLIClient.swift */, ); @@ -124,6 +134,7 @@ EED1C6F161452746D401A561 /* CameraContract.swift */, B2627A8C5CB8687217B22436 /* CMIOSinkClient.swift */, 7F22D3D2109AC909C586B1F9 /* FrameRaster.swift */, + 591A73ED75C2C97CCA46305C /* FrameStore.swift */, 382C35DD80E3AE7C84200CC3 /* QRRenderer.swift */, ); path = Shared; @@ -164,6 +175,7 @@ F9C0BF6EA2153FF6E4FBC5A8 /* MainView.swift */, 5543F1512AE6614C0DE4861D /* QRSource.swift */, 5009C01A5443FF9242C25F78 /* SourceManager.swift */, + 46D46520BA4BCC428A6C0794 /* TestPatternSource.swift */, 197CD7BA6EB552D5C96D7431 /* VideoFileSource.swift */, ); path = SimulatorCamera; @@ -229,6 +241,7 @@ buildPhases = ( 1096B2376032B3F8640D7973 /* Sources */, 4AACD8F0F143DE3CA8F7D170 /* Embed Dependencies */, + 98537C620109260D815B0C49 /* Bundle SimulatorInjection */, ); buildRules = ( ); @@ -291,6 +304,28 @@ }; /* End PBXProject section */ +/* Begin PBXShellScriptBuildPhase section */ + 98537C620109260D815B0C49 /* Bundle SimulatorInjection */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Bundle SimulatorInjection"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "# The library simulator apps load to see SimulatorCamera (see SimulatorInjection/).\n\"$SRCROOT/SimulatorInjection/build.sh\" \"$TARGET_BUILD_DIR/$UNLOCALIZED_RESOURCES_FOLDER_PATH\"\n"; + }; +/* End PBXShellScriptBuildPhase section */ + /* Begin PBXSourcesBuildPhase section */ 1096B2376032B3F8640D7973 /* Sources */ = { isa = PBXSourcesBuildPhase; @@ -303,12 +338,14 @@ A03B198646B946D161279AE4 /* ExtensionController.swift in Sources */, CA28817E141686080C6B6B62 /* FrameRaster.swift in Sources */, 501C657F795A0DC953369029 /* FrameSource.swift in Sources */, + AD45CBDEBF2591469C97616B /* FrameStore.swift in Sources */, 15791A539F2708C098E31D62 /* ImageSource.swift in Sources */, D3B54EFD8EC9634626E4351F /* MacCameraSource.swift in Sources */, 386FA16D47CF0C291E857D00 /* MainView.swift in Sources */, 4FC626F7D38B26A92C8F017F /* QRRenderer.swift in Sources */, 407FDEB20AF617B242C43F43 /* QRSource.swift in Sources */, 0B0F268D9B7EE26F8904395C /* SourceManager.swift in Sources */, + 1DC705930BDB7DCA604A0DB9 /* TestPatternSource.swift in Sources */, F6D3FCD3CDF09C4E6AB84FB0 /* VideoFileSource.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -321,6 +358,7 @@ BE593A54205871224550F5FF /* CameraContract.swift in Sources */, A17CABFD712A0F8A6992E430 /* CameraContractTests.swift in Sources */, 5E029E026C17668F22241296 /* FrameRaster.swift in Sources */, + 11CF13DB224856F4BD57F8C1 /* FrameStore.swift in Sources */, D28E5DD221A2F8ACAEFD1D9A /* QRRenderer.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -332,6 +370,7 @@ D97A5DF273D525682867329C /* CMIOSinkClient.swift in Sources */, EA24C3B90C006BB89A1CDC6B /* CameraContract.swift in Sources */, 2AB97A0BCC2C375207D8AFD6 /* FrameRaster.swift in Sources */, + F8617E1FC8ABB421235E3C73 /* FrameStore.swift in Sources */, 704954DE3FE7D5F29F53A9EF /* QRRenderer.swift in Sources */, 07A2A4651AA575FEBF94D191 /* SimulatorCameraProvider.swift in Sources */, 179FDC3A8B0E644F699454C4 /* main.swift in Sources */, @@ -344,7 +383,9 @@ files = ( A86A15DD7CB53CE6EB144F28 /* CMIOSinkClient.swift in Sources */, 123734FDBD4B874EC803EC0B /* CameraContract.swift in Sources */, + D7400DD5957DCED5E222054E /* FeedCommand.swift in Sources */, 313CD0876911B414C8C71B91 /* FrameRaster.swift in Sources */, + E3A5553E615353760AFA72C9 /* FrameStore.swift in Sources */, BFBE13333CF067BDB88CB825 /* QRRenderer.swift in Sources */, 9A9423F300616A5876893390 /* SimCamCLIClient.swift in Sources */, 9F46B480FDD94B5D2359EB61 /* main.swift in Sources */, @@ -451,6 +492,7 @@ CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; ENABLE_HARDENED_RUNTIME = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; GENERATE_INFOPLIST_FILE = NO; INFOPLIST_FILE = SimulatorCamera/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( @@ -613,6 +655,7 @@ CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; ENABLE_HARDENED_RUNTIME = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; GENERATE_INFOPLIST_FILE = NO; INFOPLIST_FILE = SimulatorCamera/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( diff --git a/SimulatorCamera/SimulatorCamera.entitlements b/SimulatorCamera/SimulatorCamera.entitlements index 70938f3..467ba9b 100644 --- a/SimulatorCamera/SimulatorCamera.entitlements +++ b/SimulatorCamera/SimulatorCamera.entitlements @@ -2,17 +2,21 @@ + com.apple.developer.system-extension.install + com.apple.security.app-sandbox + com.apple.security.application-groups + + $(TeamIdentifierPrefix)com.dautov.SimulatorCamera + com.apple.security.device.camera com.apple.security.files.user-selected.read-only - com.apple.developer.system-extension.install - - com.apple.security.application-groups + com.apple.security.temporary-exception.files.absolute-path.read-write - $(TeamIdentifierPrefix)com.dautov.SimulatorCamera + /private/tmp/SimulatorCamera/ diff --git a/SimulatorCamera/SourceManager.swift b/SimulatorCamera/SourceManager.swift index d316de6..f47980b 100644 --- a/SimulatorCamera/SourceManager.swift +++ b/SimulatorCamera/SourceManager.swift @@ -47,10 +47,16 @@ final class SourceManager { private var activeSource: FrameSource? private var switchGeneration = 0 private let link: CameraLink + /// Frames for iOS Simulator apps (SimulatorInjection reads this file). The + /// Simulator cannot see CMIO cameras, so this is how they reach it. + private let frameStore = FrameStore() + private var testPattern: TestPatternSource? private let log = Logger(subsystem: "com.dautov.SimulatorCamera", category: "source-manager") init(link: CameraLink) { self.link = link + // Simulator apps get the test pattern as soon as the app is open. + Task { @MainActor [weak self] in try? await self?.switchTo(.testPattern) } } /// Switch to the given source kind. Throws on user-correctable issues @@ -65,8 +71,16 @@ final class SourceManager { tearDownActiveSource() lastError = nil - // testPattern needs no container-side source. + // The extension draws its own test pattern for macOS apps; simulator + // apps get this container-side one through the frame file. No sink + // and no camera access needed. if kind == .testPattern { + let pattern = TestPatternSource() + let store = frameStore + pattern.onFrame = { frame in store.write(frame, sourceKind: UInt32(SimCamSourceKind.testPattern.rawValue)) } + try? await pattern.start() + guard switchGeneration == generation else { pattern.stop(); return } + testPattern = pattern declare(.testPattern) return } @@ -75,7 +89,7 @@ final class SourceManager { let source: FrameSource switch kind { case .testPattern: - return + return // handled above case .macCamera: source = MacCameraSource() case .videoFile: @@ -93,28 +107,26 @@ final class SourceManager { isSwitching = true defer { if switchGeneration == generation { isSwitching = false } } - // The sink needs camera access (TCC gates CMIO device streams). - guard await link.requestCameraAccessIfNeeded() else { - lastError = FrameSourceError.permissionDenied.localizedDescription - throw SwitchError.startFailed(lastError ?? "camera access denied") - } - - // Open the sink before the producer starts so its first frames have - // somewhere to go. The extension keeps showing the test pattern - // until we declare the new kind below. - do { - try link.openSink() - } catch { - lastError = "Extension not reachable: \(error.localizedDescription)" - throw SwitchError.startFailed(lastError ?? error.localizedDescription) + // The extension's sink feeds macOS apps. It needs camera access (TCC + // gates CMIO device streams) and an approved extension. Simulator apps + // don't need either, so a missing sink is reported but not fatal. + var sinkOpen = false + if await link.requestCameraAccessIfNeeded() { + do { try link.openSink(); sinkOpen = true } + catch { lastError = "Extension not reachable (simulator apps still get frames): \(error.localizedDescription)" } + } else { + lastError = "Camera access denied: macOS apps won't see this source; simulator apps still do." } - // Frames go straight to the sink from the producer's thread; only - // the counter hops to the main actor, and only every few frames. + // Frames go straight to the sink and the frame file from the + // producer's thread; only the counter hops to the main actor. let link = self.link + let store = frameStore + let kindRaw = UInt32(kind.rawValue) var localCount = 0 source.onFrame = { [weak self] frame in - link.pushFrame(frame) + store.write(frame, sourceKind: kindRaw) + if sinkOpen { link.pushFrame(frame) } localCount += 1 if localCount % 10 == 0 { let count = localCount @@ -172,6 +184,8 @@ final class SourceManager { } private func tearDownActiveSource() { + testPattern?.stop() + testPattern = nil activeSource?.onFrame = nil activeSource?.stop() activeSource = nil diff --git a/SimulatorCamera/TestPatternSource.swift b/SimulatorCamera/TestPatternSource.swift new file mode 100644 index 0000000..68173e3 --- /dev/null +++ b/SimulatorCamera/TestPatternSource.swift @@ -0,0 +1,57 @@ +// +// TestPatternSource.swift +// SimulatorCamera +// +// Container-side test pattern. The camera extension draws its own pattern for +// macOS apps; this one feeds iOS Simulator apps (via FrameStore) so they get +// frames as soon as the Mac app is open, before any source is picked. +// + +import CoreGraphics +import CoreVideo +import Foundation + +final class TestPatternSource: FrameSource, @unchecked Sendable { + let kind: SimCamSourceKind = .testPattern + var onFrame: ((CVPixelBuffer) -> Void)? + + private let width = kSimCamFrameWidth, height = kSimCamFrameHeight + private var timer: DispatchSourceTimer? + private var tick = 0 + + func start() async throws { + stop() + let t = DispatchSource.makeTimerSource(queue: DispatchQueue(label: "com.dautov.SimulatorCamera.pattern")) + t.schedule(deadline: .now(), repeating: .milliseconds(33)) + t.setEventHandler { [weak self] in self?.emit() } + t.resume() + timer = t + } + + func stop() { + timer?.cancel() + timer = nil + } + + private func emit() { + guard let pb = FrameRaster.makePixelBuffer() else { return } + let t = tick + tick += 1 + CVPixelBufferLockBaseAddress(pb, []) + if let ctx = CGContext(data: CVPixelBufferGetBaseAddress(pb), width: width, height: height, bitsPerComponent: 8, + bytesPerRow: CVPixelBufferGetBytesPerRow(pb), space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) { + let bars: [(CGFloat, CGFloat, CGFloat)] = [(0.75, 0.75, 0.75), (0.75, 0.75, 0), (0, 0.75, 0.75), (0, 0.75, 0), + (0.75, 0, 0.75), (0.75, 0, 0), (0, 0, 0.75)] + let w = CGFloat(width) / 7 + for (i, c) in bars.enumerated() { + ctx.setFillColor(red: c.0, green: c.1, blue: c.2, alpha: 1) + ctx.fill(CGRect(x: CGFloat(i) * w, y: 0, width: w + 1, height: CGFloat(height))) + } + ctx.setFillColor(red: 1, green: 1, blue: 1, alpha: 0.9) + ctx.fill(CGRect(x: CGFloat((t * 8) % width), y: 0, width: 24, height: CGFloat(height))) + } + CVPixelBufferUnlockBaseAddress(pb, []) + onFrame?(pb) + } +} diff --git a/SimulatorInjection/SCFrameStore.h b/SimulatorInjection/SCFrameStore.h new file mode 100644 index 0000000..8934a03 --- /dev/null +++ b/SimulatorInjection/SCFrameStore.h @@ -0,0 +1,34 @@ +// SCFrameStore.h — the frame file shared by the Mac app (writer) and the +// simulator injection library (reader). +// +// iOS Simulator processes run on the Mac and can read host paths, so the Mac +// app publishes the newest camera frame as a memory-mapped file. Layout: +// a 64-byte header followed by BGRA pixels. A sequence lock keeps readers from +// copying a half-written frame: `seq` is odd while the writer copies pixels. +// Keep in sync with Shared/FrameStore.swift. +#pragma once +#include + +#define SC_FRAME_STORE_DIR "/private/tmp/SimulatorCamera" +#define SC_FRAME_STORE_PATH SC_FRAME_STORE_DIR "/frame.bin" +#define SC_FRAME_MAGIC 0x31464353u /* "SCF1" little-endian */ +#define SC_FRAME_VERSION 1u +#define SC_FRAME_HEADER_SIZE 64u +#define SC_FRAME_MAX_WIDTH 1920u +#define SC_FRAME_MAX_HEIGHT 1080u +#define SC_FRAME_MAX_BYTES (SC_FRAME_MAX_WIDTH * SC_FRAME_MAX_HEIGHT * 4u) +#define SC_FRAME_FILE_SIZE (SC_FRAME_HEADER_SIZE + SC_FRAME_MAX_BYTES) + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t width; + uint32_t height; + uint32_t bytesPerRow; + uint32_t sourceKind; /* SimCamSourceKind raw value, informational */ + uint64_t seq; /* odd = write in progress */ + double hostTime; /* seconds, writer's clock, for staleness checks */ + uint8_t reserved[24]; +} SCFrameHeader; + +_Static_assert(sizeof(SCFrameHeader) == SC_FRAME_HEADER_SIZE, "header must be 64 bytes"); diff --git a/SimulatorInjection/SimCamInject.m b/SimulatorInjection/SimCamInject.m new file mode 100644 index 0000000..2d21c92 --- /dev/null +++ b/SimulatorInjection/SimCamInject.m @@ -0,0 +1,758 @@ +// SimCamInject.m — SimulatorCamera for iOS Simulator apps. +// +// The iOS Simulator's capture stack (FigCaptureSourceSimulator) is a stub: it +// never produces frames and AVCaptureDevice lists no cameras, not even the +// Mac's own. This library is loaded into a simulator app at launch +// (DYLD_INSERT_LIBRARIES) and supplies a camera through the normal +// AVFoundation API, with frames read from the SimulatorCamera Mac app +// (see SCFrameStore.h). Your app imports nothing. +// +// Covered: device discovery (default/devices/DiscoverySession/uniqueID), +// AVCaptureDeviceInput, AVCaptureSession running state + notifications, +// AVCaptureVideoDataOutput (BGRA sample buffers), AVCaptureVideoPreviewLayer, +// AVCaptureMetadataOutput (QR codes), AVCapturePhotoOutput (JPEG/HEIF data). +// Without the Mac app running, a built-in animated test pattern is shown. +// +// Simulator only. Build: SimulatorInjection/build.sh + +#import +#import +#import +#import +#import +#import +#import +#import "SCFrameStore.h" + +#if !TARGET_OS_SIMULATOR +#error "SimCamInject is for the iOS Simulator only" +#endif + +static NSString *const kSCDeviceID = @"com.dautov.SimulatorCamera.injected"; +static void SCLog(NSString *fmt, ...) NS_FORMAT_FUNCTION(1, 2); +static void SCLog(NSString *fmt, ...) { + va_list ap; va_start(ap, fmt); + NSString *s = [[NSString alloc] initWithFormat:fmt arguments:ap]; + va_end(ap); + NSLog(@"[SimulatorCamera] %@", s); +} + +#pragma mark - Swizzling helpers + +static void SCSwizzleInstance(Class cls, SEL sel, IMP newImp, IMP *orig) { + Method m = class_getInstanceMethod(cls, sel); + if (!m) { SCLog(@"missing -[%@ %@]", cls, NSStringFromSelector(sel)); return; } + // Add on this class first so we never replace a superclass implementation. + if (class_addMethod(cls, sel, newImp, method_getTypeEncoding(m))) { + if (orig) *orig = method_getImplementation(m); + } else { + IMP old = method_setImplementation(m, newImp); + if (orig) *orig = old; + } +} + +static void SCSwizzleClass(Class cls, SEL sel, IMP newImp, IMP *orig) { + SCSwizzleInstance(object_getClass(cls), sel, newImp, orig); +} + +#pragma mark - Fake objects +// AVFoundation classes have no public initializers for these objects, so the +// fakes are subclasses created without -init; every accessor apps use is +// overridden so the (uninitialised) superclass state is never read. + +@interface SCDevice : AVCaptureDevice @end +@interface SCDeviceInput : AVCaptureDeviceInput @end +@interface SCConnection : AVCaptureConnection @end +@interface SCQRObject : AVMetadataMachineReadableCodeObject @end +@interface SCPhoto : AVCapturePhoto @end + +static SCDevice *gDevice; +static SCDevice *SCSharedDevice(void) { + static dispatch_once_t once; + dispatch_once(&once, ^{ gDevice = class_createInstance([SCDevice class], 0); }); + return gDevice; +} + +@implementation SCDevice +- (NSString *)uniqueID { return kSCDeviceID; } +- (NSString *)modelID { return @"SimulatorCamera"; } +- (NSString *)localizedName { return @"SimulatorCamera"; } +- (NSString *)manufacturer { return @"SimulatorCamera"; } +- (BOOL)hasMediaType:(AVMediaType)t { return [t isEqualToString:AVMediaTypeVideo]; } +- (AVCaptureDevicePosition)position { return AVCaptureDevicePositionBack; } +- (AVCaptureDeviceType)deviceType { return AVCaptureDeviceTypeBuiltInWideAngleCamera; } +- (BOOL)isConnected { return YES; } +- (BOOL)isSuspended { return NO; } +- (BOOL)isInUseByAnotherApplication { return NO; } +- (NSArray *)formats { return @[]; } +- (AVCaptureDeviceFormat *)activeFormat { return nil; } +- (void)setActiveFormat:(AVCaptureDeviceFormat *)f {} +- (BOOL)lockForConfiguration:(NSError **)e { return YES; } +- (void)unlockForConfiguration {} +- (BOOL)supportsAVCaptureSessionPreset:(AVCaptureSessionPreset)p { return YES; } +- (BOOL)hasTorch { return NO; } +- (BOOL)hasFlash { return NO; } +- (BOOL)isTorchAvailable { return NO; } +- (BOOL)isTorchModeSupported:(AVCaptureTorchMode)m { return m == AVCaptureTorchModeOff; } +- (AVCaptureTorchMode)torchMode { return AVCaptureTorchModeOff; } +- (void)setTorchMode:(AVCaptureTorchMode)m {} +- (BOOL)isFocusModeSupported:(AVCaptureFocusMode)m { return YES; } +- (AVCaptureFocusMode)focusMode { return AVCaptureFocusModeContinuousAutoFocus; } +- (void)setFocusMode:(AVCaptureFocusMode)m {} +- (BOOL)isFocusPointOfInterestSupported { return YES; } +- (CGPoint)focusPointOfInterest { return CGPointMake(0.5, 0.5); } +- (void)setFocusPointOfInterest:(CGPoint)p {} +- (BOOL)isAutoFocusRangeRestrictionSupported { return NO; } +- (BOOL)isSmoothAutoFocusSupported { return NO; } +- (BOOL)isExposureModeSupported:(AVCaptureExposureMode)m { return YES; } +- (AVCaptureExposureMode)exposureMode { return AVCaptureExposureModeContinuousAutoExposure; } +- (void)setExposureMode:(AVCaptureExposureMode)m {} +- (BOOL)isExposurePointOfInterestSupported { return YES; } +- (CGPoint)exposurePointOfInterest { return CGPointMake(0.5, 0.5); } +- (void)setExposurePointOfInterest:(CGPoint)p {} +- (BOOL)isWhiteBalanceModeSupported:(AVCaptureWhiteBalanceMode)m { return YES; } +- (AVCaptureWhiteBalanceMode)whiteBalanceMode { return AVCaptureWhiteBalanceModeContinuousAutoWhiteBalance; } +- (void)setWhiteBalanceMode:(AVCaptureWhiteBalanceMode)m {} +- (CGFloat)videoZoomFactor { return 1; } +- (void)setVideoZoomFactor:(CGFloat)f {} +- (void)rampToVideoZoomFactor:(CGFloat)f withRate:(float)r {} +- (void)cancelVideoZoomRamp {} +- (CGFloat)minAvailableVideoZoomFactor { return 1; } +- (CGFloat)maxAvailableVideoZoomFactor { return 1; } +- (BOOL)isSubjectAreaChangeMonitoringEnabled { return NO; } +- (void)setSubjectAreaChangeMonitoringEnabled:(BOOL)e {} +- (BOOL)isLowLightBoostSupported { return NO; } +- (CMTime)activeVideoMinFrameDuration { return CMTimeMake(1, 30); } +- (void)setActiveVideoMinFrameDuration:(CMTime)t {} +- (CMTime)activeVideoMaxFrameDuration { return CMTimeMake(1, 30); } +- (void)setActiveVideoMaxFrameDuration:(CMTime)t {} +- (NSArray *)constituentDevices { return @[]; } +- (BOOL)isVirtualDevice { return NO; } +- (float)lensAperture { return 1.8f; } +- (float)lensPosition { return 0.5f; } +- (NSString *)description { return @""; } +- (NSUInteger)hash { return kSCDeviceID.hash; } +- (BOOL)isEqual:(id)o { return o == self; } +@end + +@implementation SCDeviceInput { + @public SCDevice *_scDevice; +} +- (AVCaptureDevice *)device { return _scDevice; } +- (NSArray *)ports { return @[]; } +- (NSString *)description { return @""; } +@end + +@implementation SCConnection { + @public AVCaptureVideoOrientation _orientation; + @public CGFloat _rotation; + @public BOOL _mirrored; +} +- (NSArray *)inputPorts { return @[]; } +- (AVMediaType)mediaType { return AVMediaTypeVideo; } // private, read by AVCaptureSession internals +- (AVCaptureOutput *)output { return nil; } +- (AVCaptureVideoPreviewLayer *)videoPreviewLayer { return nil; } +- (BOOL)isEnabled { return YES; } +- (void)setEnabled:(BOOL)e {} +- (BOOL)isActive { return YES; } +- (BOOL)isVideoOrientationSupported { return YES; } +- (AVCaptureVideoOrientation)videoOrientation { return _orientation ?: AVCaptureVideoOrientationPortrait; } +- (void)setVideoOrientation:(AVCaptureVideoOrientation)o { _orientation = o; } +- (CGFloat)videoRotationAngle { return _rotation; } +- (void)setVideoRotationAngle:(CGFloat)a { _rotation = a; } +- (BOOL)isVideoRotationAngleSupported:(CGFloat)a { return YES; } +- (BOOL)isVideoMirroringSupported { return YES; } +- (BOOL)isVideoMirrored { return _mirrored; } +- (void)setVideoMirrored:(BOOL)m { _mirrored = m; } +- (BOOL)automaticallyAdjustsVideoMirroring { return NO; } +- (void)setAutomaticallyAdjustsVideoMirroring:(BOOL)a {} +- (BOOL)isVideoStabilizationSupported { return NO; } +- (AVCaptureVideoStabilizationMode)preferredVideoStabilizationMode { return AVCaptureVideoStabilizationModeOff; } +- (void)setPreferredVideoStabilizationMode:(AVCaptureVideoStabilizationMode)m {} +- (AVCaptureVideoStabilizationMode)activeVideoStabilizationMode { return AVCaptureVideoStabilizationModeOff; } +- (BOOL)isCameraIntrinsicMatrixDeliverySupported { return NO; } +- (void)setCameraIntrinsicMatrixDeliveryEnabled:(BOOL)e {} +- (CMTime)videoMinFrameDuration { return CMTimeMake(1, 30); } +- (void)setVideoMinFrameDuration:(CMTime)t {} +@end + +static SCConnection *SCNewConnection(void) { + SCConnection *c = class_createInstance([SCConnection class], 0); + c->_orientation = AVCaptureVideoOrientationPortrait; + c->_rotation = 90; + return c; +} + +@implementation SCQRObject { + @public NSString *_value; + @public CGRect _bounds; + @public NSArray *_corners; + @public CMTime _time; +} +- (AVMetadataObjectType)type { return AVMetadataObjectTypeQRCode; } +- (NSString *)stringValue { return _value; } +- (CGRect)bounds { return _bounds; } +- (NSArray *)corners { return _corners; } +- (CMTime)time { return _time; } +- (CMTime)duration { return kCMTimeInvalid; } +- (CIBarcodeDescriptor *)descriptor { return nil; } +- (NSString *)description { return [NSString stringWithFormat:@"", _value]; } +@end + +@implementation SCPhoto { + @public CVPixelBufferRef _pixels; + @public NSData *_data; + @public CMTime _time; +} +- (void)dealloc { if (_pixels) CVPixelBufferRelease(_pixels); } +- (CMTime)timestamp { return _time; } +- (BOOL)isRawPhoto { return NO; } +- (CVPixelBufferRef)pixelBuffer { return _pixels; } +- (CVPixelBufferRef)previewPixelBuffer { return _pixels; } +- (NSDictionary *)metadata { return @{}; } +- (NSData *)fileDataRepresentation { return _data; } +- (CGImageRef)CGImageRepresentation { + CIImage *ci = [CIImage imageWithCVPixelBuffer:_pixels]; + CGImageRef img = [[CIContext context] createCGImage:ci fromRect:ci.extent]; + return (CGImageRef)CFAutorelease(img); +} +- (CGImageRef)previewCGImageRepresentation { return [self CGImageRepresentation]; } +- (NSInteger)photoCount { return 1; } +- (AVCaptureResolvedPhotoSettings *)resolvedSettings { return nil; } +@end + +#pragma mark - Frame source (Mac app frame file, or built-in test pattern) + +static uint8_t *gStore; // mmap of SC_FRAME_STORE_PATH, read-only +static uint64_t gLastSeq; +static CVPixelBufferRef gLatest; // newest delivered frame (Mac or pattern), owned +static CVPixelBufferRef gStoreLatest; // newest complete frame from the Mac app, owned +static double gStoreLatestAt; // when gStoreLatest was read (seconds since 1970) +static uint64_t gFrameCount; +static dispatch_queue_t gFrameQueue; + +static void SCMapStore(void) { + if (gStore) return; + int fd = open(SC_FRAME_STORE_PATH, O_RDONLY); + if (fd < 0) return; + void *p = mmap(NULL, SC_FRAME_FILE_SIZE, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + if (p != MAP_FAILED) gStore = p; +} + +static CVPixelBufferRef SCNewBuffer(size_t w, size_t h) { + NSDictionary *attrs = @{ (id)kCVPixelBufferIOSurfacePropertiesKey: @{}, + (id)kCVPixelBufferCGImageCompatibilityKey: @YES, + (id)kCVPixelBufferCGBitmapContextCompatibilityKey: @YES }; + CVPixelBufferRef pb = NULL; + CVPixelBufferCreate(kCFAllocatorDefault, w, h, kCVPixelFormatType_32BGRA, (__bridge CFDictionaryRef)attrs, &pb); + return pb; +} + +/// Copy the Mac app's newest frame if there is a fresh, complete one. +static CVPixelBufferRef SCReadStoreOnce(void) { + SCMapStore(); + if (!gStore) return NULL; + const SCFrameHeader *h = (const SCFrameHeader *)gStore; + if (h->magic != SC_FRAME_MAGIC) return NULL; + uint64_t s1 = atomic_load_explicit((_Atomic uint64_t *)&h->seq, memory_order_acquire); + if (s1 & 1) return NULL; + // Treat a writer that stopped more than 3 s ago as gone. + if ([NSDate date].timeIntervalSince1970 - h->hostTime > 3 && h->hostTime > 1e9) return NULL; + uint32_t w = h->width, ht = h->height, bpr = h->bytesPerRow; + if (!w || !ht || w > SC_FRAME_MAX_WIDTH || ht > SC_FRAME_MAX_HEIGHT || bpr < w * 4 || (uint64_t)bpr * ht > SC_FRAME_MAX_BYTES) return NULL; + if (s1 == gLastSeq && gStoreLatest && CVPixelBufferGetWidth(gStoreLatest) == w) return CVPixelBufferRetain(gStoreLatest); + CVPixelBufferRef pb = SCNewBuffer(w, ht); + if (!pb) return NULL; + CVPixelBufferLockBaseAddress(pb, 0); + uint8_t *dst = CVPixelBufferGetBaseAddress(pb); + size_t dbpr = CVPixelBufferGetBytesPerRow(pb); + const uint8_t *src = gStore + SC_FRAME_HEADER_SIZE; + for (uint32_t y = 0; y < ht; y++) memcpy(dst + y * dbpr, src + (size_t)y * bpr, w * 4); + CVPixelBufferUnlockBaseAddress(pb, 0); + uint64_t s2 = atomic_load_explicit((_Atomic uint64_t *)&h->seq, memory_order_acquire); + if (s1 != s2) { CVPixelBufferRelease(pb); return NULL; } // torn; keep previous + gLastSeq = s1; + return pb; +} + +/// The Mac app's frame, or its last good frame if we caught a write in progress. +/// NULL once the Mac app has been quiet for 1 s (then the test pattern shows). +static CVPixelBufferRef SCReadStore(void) { + double now = [NSDate date].timeIntervalSince1970; + CVPixelBufferRef pb = SCReadStoreOnce(); + if (pb) { + if (pb != gStoreLatest) { + if (gStoreLatest) CVPixelBufferRelease(gStoreLatest); + gStoreLatest = CVPixelBufferRetain(pb); + } + gStoreLatestAt = now; + return pb; + } + if (gStoreLatest && now - gStoreLatestAt < 1.0) return CVPixelBufferRetain(gStoreLatest); + return NULL; +} + +/// Animated colour bars with a moving stripe, so a stuck pipeline is visible. +static CVPixelBufferRef SCTestPattern(uint64_t n) { + const size_t w = 1280, h = 720; + CVPixelBufferRef pb = SCNewBuffer(w, h); + if (!pb) return NULL; + CVPixelBufferLockBaseAddress(pb, 0); + CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); + CGContextRef ctx = CGBitmapContextCreate(CVPixelBufferGetBaseAddress(pb), w, h, 8, CVPixelBufferGetBytesPerRow(pb), cs, + kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Little); + const CGFloat bars[7][3] = {{.75,.75,.75},{.75,.75,0},{0,.75,.75},{0,.75,0},{.75,0,.75},{.75,0,0},{0,0,.75}}; + for (int i = 0; i < 7; i++) { + CGContextSetRGBFillColor(ctx, bars[i][0], bars[i][1], bars[i][2], 1); + CGContextFillRect(ctx, CGRectMake(i * w / 7.0, 0, w / 7.0 + 1, h)); + } + CGContextSetRGBFillColor(ctx, 1, 1, 1, 0.85); + CGContextFillRect(ctx, CGRectMake((n * 8) % w, 0, 24, h)); + CGContextSetRGBFillColor(ctx, 0, 0, 0, 0.6); + CGContextFillRect(ctx, CGRectMake(0, h / 2 - 60, w, 120)); + UIGraphicsPushContext(ctx); + CGContextTranslateCTM(ctx, 0, h); CGContextScaleCTM(ctx, 1, -1); + NSDictionary *a = @{ NSFontAttributeName: [UIFont boldSystemFontOfSize:44], NSForegroundColorAttributeName: UIColor.whiteColor }; + [@"SimulatorCamera — open the Mac app for live frames" drawAtPoint:CGPointMake(40, h / 2 - 28) withAttributes:a]; + UIGraphicsPopContext(); + CGContextRelease(ctx); CGColorSpaceRelease(cs); + CVPixelBufferUnlockBaseAddress(pb, 0); + return pb; +} + +#pragma mark - Session registry + +@interface SCSessionState : NSObject +@property (nonatomic) BOOL usesFake; +@property (nonatomic) BOOL running; +@property (nonatomic, strong) NSMutableArray *inputs; +@property (nonatomic, strong) NSHashTable *outputs; // weak +@end +@implementation SCSessionState +- (instancetype)init { if ((self = [super init])) { _inputs = [NSMutableArray array]; _outputs = [NSHashTable weakObjectsHashTable]; } return self; } +@end + +static const void *kSCStateKey = &kSCStateKey; +static const void *kSCSessionKey = &kSCSessionKey; // output/layer -> session (weak box) +static const void *kSCConnKey = &kSCConnKey; // output/layer -> SCConnection +static const void *kSCMetaTypesKey = &kSCMetaTypesKey; +static const void *kSCRectKey = &kSCRectKey; +static NSHashTable *gRunning; // running fake sessions (weak) +static NSHashTable *gLayers; // preview layers (weak) +static dispatch_source_t gTimer; +static CIDetector *gQR; +static NSLock *gLock; + +static SCSessionState *SCState(AVCaptureSession *s) { + SCSessionState *st = objc_getAssociatedObject(s, kSCStateKey); + if (!st) { st = [SCSessionState new]; objc_setAssociatedObject(s, kSCStateKey, st, OBJC_ASSOCIATION_RETAIN); } + return st; +} + +static SCConnection *SCConnectionFor(id obj) { + SCConnection *c = objc_getAssociatedObject(obj, kSCConnKey); + if (!c) { c = SCNewConnection(); objc_setAssociatedObject(obj, kSCConnKey, c, OBJC_ASSOCIATION_RETAIN); } + return c; +} + +static CMSampleBufferRef SCSampleBuffer(CVPixelBufferRef pb, CMTime pts) { + CMVideoFormatDescriptionRef fmt = NULL; + if (CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pb, &fmt) != noErr) return NULL; + CMSampleTimingInfo t = { CMTimeMake(1, 30), pts, kCMTimeInvalid }; + CMSampleBufferRef sb = NULL; + CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pb, fmt, &t, &sb); + CFRelease(fmt); + return sb; +} + +static void SCDeliverMetadata(AVCaptureMetadataOutput *out, AVCaptureSession *session, CVPixelBufferRef pb, CMTime pts) { + NSArray *wanted = objc_getAssociatedObject(out, kSCMetaTypesKey); + if (![wanted containsObject:AVMetadataObjectTypeQRCode]) return; + id d = out.metadataObjectsDelegate; + dispatch_queue_t q = out.metadataObjectsCallbackQueue; + if (!d || !q || ![d respondsToSelector:@selector(captureOutput:didOutputMetadataObjects:fromConnection:)]) return; + CIImage *img = [CIImage imageWithCVPixelBuffer:pb]; + NSArray *found = [gQR featuresInImage:img]; + if (!found.count) return; + CGFloat W = CVPixelBufferGetWidth(pb), H = CVPixelBufferGetHeight(pb); + NSMutableArray *objs = [NSMutableArray array]; + for (CIQRCodeFeature *f in found) { + if (![f isKindOfClass:[CIQRCodeFeature class]] || !f.messageString) continue; + SCQRObject *o = class_createInstance([SCQRObject class], 0); + // Metadata coordinates are normalised, origin top-left (CI is bottom-left). + CGRect b = f.bounds; + o->_bounds = CGRectMake(b.origin.x / W, 1 - (b.origin.y + b.size.height) / H, b.size.width / W, b.size.height / H); + NSDictionary *(^pt)(CGPoint) = ^(CGPoint p) { return (__bridge_transfer NSDictionary *)CGPointCreateDictionaryRepresentation(CGPointMake(p.x / W, 1 - p.y / H)); }; + o->_corners = @[pt(f.topLeft), pt(f.topRight), pt(f.bottomRight), pt(f.bottomLeft)]; + o->_value = f.messageString; + o->_time = pts; + [objs addObject:o]; + } + if (!objs.count) return; + AVCaptureConnection *c = SCConnectionFor(out); + dispatch_async(q, ^{ [d captureOutput:out didOutputMetadataObjects:objs fromConnection:c]; }); +} + +static void SCTick(void) { + gFrameCount++; + CVPixelBufferRef pb = SCReadStore(); + if (!pb) pb = SCTestPattern(gFrameCount); + if (!pb) return; + if (gLatest) CVPixelBufferRelease(gLatest); + gLatest = CVPixelBufferRetain(pb); + CMTime pts = CMClockGetTime(CMClockGetHostTimeClock()); + + [gLock lock]; + NSArray *sessions = gRunning.allObjects; + NSArray *layers = gLayers.allObjects; + [gLock unlock]; + + for (AVCaptureSession *s in sessions) { + for (AVCaptureOutput *out in SCState(s).outputs.allObjects) { + if ([out isKindOfClass:[AVCaptureVideoDataOutput class]]) { + AVCaptureVideoDataOutput *vo = (AVCaptureVideoDataOutput *)out; + id d = vo.sampleBufferDelegate; + dispatch_queue_t q = vo.sampleBufferCallbackQueue; + if (!d || !q || ![d respondsToSelector:@selector(captureOutput:didOutputSampleBuffer:fromConnection:)]) continue; + CMSampleBufferRef sb = SCSampleBuffer(pb, pts); + if (!sb) continue; + AVCaptureConnection *c = SCConnectionFor(vo); + dispatch_async(q, ^{ [d captureOutput:vo didOutputSampleBuffer:sb fromConnection:c]; CFRelease(sb); }); + } else if ([out isKindOfClass:[AVCaptureMetadataOutput class]] && gFrameCount % 6 == 0) { + SCDeliverMetadata((AVCaptureMetadataOutput *)out, s, pb, pts); + } + } + } + + if (layers.count && gFrameCount % 2 == 0) { // preview at 15 fps is plenty + CIImage *ci = [CIImage imageWithCVPixelBuffer:pb]; + static CIContext *ctx; if (!ctx) ctx = [CIContext context]; + CGImageRef img = [ctx createCGImage:ci fromRect:ci.extent]; + dispatch_async(dispatch_get_main_queue(), ^{ + [CATransaction begin]; + [CATransaction setDisableActions:YES]; // no cross-fade between frames + for (AVCaptureVideoPreviewLayer *l in layers) { + AVCaptureSession *s = l.session; + if (!s || !SCState(s).running) continue; + NSString *g = l.videoGravity; + l.contentsGravity = [g isEqualToString:AVLayerVideoGravityResize] ? kCAGravityResize + : [g isEqualToString:AVLayerVideoGravityResizeAspect] ? kCAGravityResizeAspect + : kCAGravityResizeAspectFill; + l.contents = (__bridge id)img; + } + [CATransaction commit]; + CGImageRelease(img); + }); + } + CVPixelBufferRelease(pb); +} + +static void SCUpdateTimer(void) { + [gLock lock]; + BOOL any = gRunning.count > 0; + [gLock unlock]; + if (any && !gTimer) { + gTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, gFrameQueue); + dispatch_source_set_timer(gTimer, dispatch_time(DISPATCH_TIME_NOW, 0), NSEC_PER_SEC / 30, NSEC_PER_MSEC); + dispatch_source_set_event_handler(gTimer, ^{ @autoreleasepool { SCTick(); } }); + dispatch_resume(gTimer); + } else if (!any && gTimer) { + dispatch_source_cancel(gTimer); + gTimer = nil; + } +} + +#pragma mark - AVCaptureDevice discovery + +static IMP o_defaultDevice, o_defaultDeviceType, o_devicesWithMediaType, o_devices, o_deviceWithUniqueID, o_discoveryDevices; + +static AVCaptureDevice *sc_defaultDevice(id self, SEL _cmd, AVMediaType t) { + if ([t isEqualToString:AVMediaTypeVideo]) return SCSharedDevice(); + return ((AVCaptureDevice *(*)(id, SEL, AVMediaType))o_defaultDevice)(self, _cmd, t); +} +static AVCaptureDevice *sc_defaultDeviceType(id self, SEL _cmd, AVCaptureDeviceType dt, AVMediaType t, AVCaptureDevicePosition p) { + if (!t || [t isEqualToString:AVMediaTypeVideo]) { + BOOL frontOnly = p == AVCaptureDevicePositionFront; + if (!frontOnly || getenv("SIMCAM_ANY_POSITION")) return SCSharedDevice(); + } + return ((AVCaptureDevice *(*)(id, SEL, AVCaptureDeviceType, AVMediaType, AVCaptureDevicePosition))o_defaultDeviceType)(self, _cmd, dt, t, p); +} +static NSArray *sc_devicesWithMediaType(id self, SEL _cmd, AVMediaType t) { + NSArray *r = ((NSArray *(*)(id, SEL, AVMediaType))o_devicesWithMediaType)(self, _cmd, t); + return [t isEqualToString:AVMediaTypeVideo] ? [@[SCSharedDevice()] arrayByAddingObjectsFromArray:r] : r; +} +static NSArray *sc_devices(id self, SEL _cmd) { + NSArray *r = ((NSArray *(*)(id, SEL))o_devices)(self, _cmd); + return [@[SCSharedDevice()] arrayByAddingObjectsFromArray:r]; +} +static AVCaptureDevice *sc_deviceWithUniqueID(id self, SEL _cmd, NSString *uid) { + if ([uid isEqualToString:kSCDeviceID]) return SCSharedDevice(); + return ((AVCaptureDevice *(*)(id, SEL, NSString *))o_deviceWithUniqueID)(self, _cmd, uid); +} +static NSArray *sc_discoveryDevices(AVCaptureDeviceDiscoverySession *self, SEL _cmd) { + NSArray *r = ((NSArray *(*)(id, SEL))o_discoveryDevices)(self, _cmd); + // The discovery session stores its media type privately; only video sessions reach here in practice. + return [@[SCSharedDevice()] arrayByAddingObjectsFromArray:r ?: @[]]; +} + +#pragma mark - AVCaptureDeviceInput + +static IMP o_inputInit, o_inputFactory; + +static SCDeviceInput *SCMakeInput(AVCaptureDevice *d) { + SCDeviceInput *in = class_createInstance([SCDeviceInput class], 0); + in->_scDevice = (SCDevice *)d; + return in; +} +static id sc_inputInit(id self, SEL _cmd, AVCaptureDevice *d, NSError **e) { + if ([d isKindOfClass:[SCDevice class]]) return SCMakeInput(d); + return ((id (*)(id, SEL, AVCaptureDevice *, NSError **))o_inputInit)(self, _cmd, d, e); +} +static id sc_inputFactory(id self, SEL _cmd, AVCaptureDevice *d, NSError **e) { + if ([d isKindOfClass:[SCDevice class]]) return SCMakeInput(d); + return ((id (*)(id, SEL, AVCaptureDevice *, NSError **))o_inputFactory)(self, _cmd, d, e); +} + +#pragma mark - AVCaptureSession + +static IMP o_canAddInput, o_addInput, o_removeInput, o_inputs, o_addOutput, o_removeOutput, + o_startRunning, o_stopRunning, o_isRunning, o_canAddOutput, o_addInputNoConn; + +static BOOL sc_canAddInput(AVCaptureSession *self, SEL _cmd, AVCaptureInput *in) { + if ([in isKindOfClass:[SCDeviceInput class]]) return YES; + return ((BOOL (*)(id, SEL, id))o_canAddInput)(self, _cmd, in); +} +static void sc_addInput(AVCaptureSession *self, SEL _cmd, AVCaptureInput *in) { + if ([in isKindOfClass:[SCDeviceInput class]]) { + SCSessionState *st = SCState(self); + if (![st.inputs containsObject:in]) [st.inputs addObject:in]; + st.usesFake = YES; + return; + } + ((void (*)(id, SEL, id))o_addInput)(self, _cmd, in); +} +static void sc_removeInput(AVCaptureSession *self, SEL _cmd, AVCaptureInput *in) { + if ([in isKindOfClass:[SCDeviceInput class]]) { + SCSessionState *st = SCState(self); + [st.inputs removeObject:in]; + st.usesFake = st.inputs.count > 0; + return; + } + ((void (*)(id, SEL, id))o_removeInput)(self, _cmd, in); +} +static NSArray *sc_inputs(AVCaptureSession *self, SEL _cmd) { + NSArray *r = ((NSArray *(*)(id, SEL))o_inputs)(self, _cmd); + SCSessionState *st = objc_getAssociatedObject(self, kSCStateKey); + return st.inputs.count ? [st.inputs arrayByAddingObjectsFromArray:r] : r; +} +static BOOL sc_canAddOutput(AVCaptureSession *self, SEL _cmd, AVCaptureOutput *o) { + BOOL ok = ((BOOL (*)(id, SEL, id))o_canAddOutput)(self, _cmd, o); + return ok || SCState(self).usesFake; +} +static void sc_addOutput(AVCaptureSession *self, SEL _cmd, AVCaptureOutput *o) { + [SCState(self).outputs addObject:o]; + objc_setAssociatedObject(o, kSCSessionKey, self, OBJC_ASSOCIATION_ASSIGN); + @try { ((void (*)(id, SEL, id))o_addOutput)(self, _cmd, o); } + @catch (NSException *ex) { SCLog(@"addOutput fallback (%@)", ex.reason); } +} +static void sc_removeOutput(AVCaptureSession *self, SEL _cmd, AVCaptureOutput *o) { + [SCState(self).outputs removeObject:o]; + @try { ((void (*)(id, SEL, id))o_removeOutput)(self, _cmd, o); } @catch (NSException *ex) {} +} +static void sc_startRunning(AVCaptureSession *self, SEL _cmd) { + SCSessionState *st = SCState(self); + if (!st.usesFake) { ((void (*)(id, SEL))o_startRunning)(self, _cmd); return; } + if (st.running) return; + st.running = YES; + [gLock lock]; [gRunning addObject:self]; [gLock unlock]; + SCUpdateTimer(); + SCLog(@"session running with the injected camera"); + [[NSNotificationCenter defaultCenter] postNotificationName:AVCaptureSessionDidStartRunningNotification object:self]; +} +static void sc_stopRunning(AVCaptureSession *self, SEL _cmd) { + SCSessionState *st = SCState(self); + if (!st.usesFake) { ((void (*)(id, SEL))o_stopRunning)(self, _cmd); return; } + if (!st.running) return; + st.running = NO; + [gLock lock]; [gRunning removeObject:self]; [gLock unlock]; + SCUpdateTimer(); + [[NSNotificationCenter defaultCenter] postNotificationName:AVCaptureSessionDidStopRunningNotification object:self]; +} +static IMP o_beginConfig, o_commitConfig; +// A session on the injected camera has no real graph to configure; the stock +// commit walks output connections and would hit the fake ones. +static void sc_beginConfig(AVCaptureSession *self, SEL _cmd) { + SCSessionState *st = objc_getAssociatedObject(self, kSCStateKey); + if (st.usesFake) return; + ((void (*)(id, SEL))o_beginConfig)(self, _cmd); +} +static void sc_commitConfig(AVCaptureSession *self, SEL _cmd) { + SCSessionState *st = objc_getAssociatedObject(self, kSCStateKey); + if (st.usesFake) return; + ((void (*)(id, SEL))o_commitConfig)(self, _cmd); +} +static BOOL sc_isRunning(AVCaptureSession *self, SEL _cmd) { + SCSessionState *st = objc_getAssociatedObject(self, kSCStateKey); + if (st.usesFake) return st.running; + return ((BOOL (*)(id, SEL))o_isRunning)(self, _cmd); +} + +#pragma mark - Outputs and preview layer + +static IMP o_connectionWithMediaType, o_connections, o_setMetaTypes, o_metaTypes, o_availableMetaTypes, + o_layerConnection, o_layerSetSession, o_layerInitSession, o_layerInitNoConn, o_transformMeta, + o_capturePhoto, o_setRectOfInterest, o_rectOfInterest; + +static BOOL SCOutputIsFake(id out) { + AVCaptureSession *s = objc_getAssociatedObject(out, kSCSessionKey); + SCSessionState *st = s ? objc_getAssociatedObject(s, kSCStateKey) : nil; + return st.usesFake; +} +static AVCaptureConnection *sc_connectionWithMediaType(AVCaptureOutput *self, SEL _cmd, AVMediaType t) { + if (SCOutputIsFake(self)) return SCConnectionFor(self); + return ((id (*)(id, SEL, id))o_connectionWithMediaType)(self, _cmd, t); +} +static NSArray *sc_connections(AVCaptureOutput *self, SEL _cmd) { + if (SCOutputIsFake(self)) return @[SCConnectionFor(self)]; + return ((id (*)(id, SEL))o_connections)(self, _cmd); +} +static void sc_setMetaTypes(AVCaptureMetadataOutput *self, SEL _cmd, NSArray *types) { + objc_setAssociatedObject(self, kSCMetaTypesKey, [types copy], OBJC_ASSOCIATION_RETAIN); + if (SCOutputIsFake(self)) return; // the stock call throws: no real input to validate against + ((void (*)(id, SEL, id))o_setMetaTypes)(self, _cmd, types); +} +static NSArray *sc_metaTypes(AVCaptureMetadataOutput *self, SEL _cmd) { + if (SCOutputIsFake(self)) return objc_getAssociatedObject(self, kSCMetaTypesKey) ?: @[]; + return ((id (*)(id, SEL))o_metaTypes)(self, _cmd); +} +static NSArray *sc_availableMetaTypes(AVCaptureMetadataOutput *self, SEL _cmd) { + if (SCOutputIsFake(self)) return @[AVMetadataObjectTypeQRCode]; + return ((id (*)(id, SEL))o_availableMetaTypes)(self, _cmd); +} +static void sc_setRectOfInterest(AVCaptureMetadataOutput *self, SEL _cmd, CGRect r) { + objc_setAssociatedObject(self, kSCRectKey, [NSValue valueWithCGRect:r], OBJC_ASSOCIATION_RETAIN); + if (SCOutputIsFake(self)) return; + ((void (*)(id, SEL, CGRect))o_setRectOfInterest)(self, _cmd, r); +} +static CGRect sc_rectOfInterest(AVCaptureMetadataOutput *self, SEL _cmd) { + if (SCOutputIsFake(self)) { NSValue *v = objc_getAssociatedObject(self, kSCRectKey); return v ? v.CGRectValue : CGRectMake(0, 0, 1, 1); } + return ((CGRect (*)(id, SEL))o_rectOfInterest)(self, _cmd); +} + +static void SCTrackLayer(AVCaptureVideoPreviewLayer *l) { + [gLock lock]; [gLayers addObject:l]; [gLock unlock]; +} +static AVCaptureConnection *sc_layerConnection(AVCaptureVideoPreviewLayer *self, SEL _cmd) { + AVCaptureSession *s = self.session; + SCSessionState *st = s ? objc_getAssociatedObject(s, kSCStateKey) : nil; + if (st.usesFake) return SCConnectionFor(self); + return ((id (*)(id, SEL))o_layerConnection)(self, _cmd); +} +static void sc_layerSetSession(AVCaptureVideoPreviewLayer *self, SEL _cmd, AVCaptureSession *s) { + ((void (*)(id, SEL, id))o_layerSetSession)(self, _cmd, s); + SCTrackLayer(self); +} +static id sc_layerInitSession(AVCaptureVideoPreviewLayer *self, SEL _cmd, AVCaptureSession *s) { + id r = ((id (*)(id, SEL, id))o_layerInitSession)(self, _cmd, s); + if (r) SCTrackLayer(r); + return r; +} +static id sc_layerInitNoConn(AVCaptureVideoPreviewLayer *self, SEL _cmd, AVCaptureSession *s) { + id r = ((id (*)(id, SEL, id))o_layerInitNoConn)(self, _cmd, s); + if (r) SCTrackLayer(r); + return r; +} +static AVMetadataObject *sc_transformMeta(AVCaptureVideoPreviewLayer *self, SEL _cmd, AVMetadataObject *o) { + if (![o isKindOfClass:[SCQRObject class]]) return ((id (*)(id, SEL, id))o_transformMeta)(self, _cmd, o); + SCQRObject *q = (SCQRObject *)o; + SCQRObject *t = class_createInstance([SCQRObject class], 0); + CGRect b = self.bounds; + t->_value = q->_value; t->_time = q->_time; + t->_bounds = CGRectMake(b.origin.x + q->_bounds.origin.x * b.size.width, b.origin.y + q->_bounds.origin.y * b.size.height, + q->_bounds.size.width * b.size.width, q->_bounds.size.height * b.size.height); + NSMutableArray *cs = [NSMutableArray array]; + for (NSDictionary *d in q->_corners) { + CGPoint p; CGPointMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)d, &p); + [cs addObject:(__bridge_transfer NSDictionary *)CGPointCreateDictionaryRepresentation( + CGPointMake(b.origin.x + p.x * b.size.width, b.origin.y + p.y * b.size.height))]; + } + t->_corners = cs; + return t; +} + +static void sc_capturePhoto(AVCapturePhotoOutput *self, SEL _cmd, AVCapturePhotoSettings *settings, id d) { + if (!SCOutputIsFake(self)) { ((void (*)(id, SEL, id, id))o_capturePhoto)(self, _cmd, settings, d); return; } + dispatch_async(gFrameQueue, ^{ + CVPixelBufferRef pb = gLatest ? CVPixelBufferRetain(gLatest) : SCTestPattern(gFrameCount); + if (!pb) return; + SCPhoto *p = class_createInstance([SCPhoto class], 0); + p->_pixels = pb; // owned + p->_time = CMClockGetTime(CMClockGetHostTimeClock()); + CIImage *ci = [CIImage imageWithCVPixelBuffer:pb]; + CGColorSpaceRef cs = CGColorSpaceCreateWithName(kCGColorSpaceSRGB); + p->_data = [[CIContext context] JPEGRepresentationOfImage:ci colorSpace:cs options:@{}]; + CGColorSpaceRelease(cs); + dispatch_async(dispatch_get_main_queue(), ^{ + if ([d respondsToSelector:@selector(captureOutput:didFinishProcessingPhoto:error:)]) + [d captureOutput:self didFinishProcessingPhoto:p error:nil]; + }); + }); +} + +#pragma mark - Install + +__attribute__((constructor)) static void SCInstall(void) { + if (getenv("SIMCAM_DISABLE")) return; + gLock = [NSLock new]; + gRunning = [NSHashTable weakObjectsHashTable]; + gLayers = [NSHashTable weakObjectsHashTable]; + gFrameQueue = dispatch_queue_create("com.dautov.SimulatorCamera.frames", DISPATCH_QUEUE_SERIAL); + gQR = [CIDetector detectorOfType:CIDetectorTypeQRCode context:nil options:@{ CIDetectorAccuracy: CIDetectorAccuracyHigh }]; + + Class D = [AVCaptureDevice class]; + SCSwizzleClass(D, @selector(defaultDeviceWithMediaType:), (IMP)sc_defaultDevice, &o_defaultDevice); + SCSwizzleClass(D, @selector(defaultDeviceWithDeviceType:mediaType:position:), (IMP)sc_defaultDeviceType, &o_defaultDeviceType); + SCSwizzleClass(D, @selector(devicesWithMediaType:), (IMP)sc_devicesWithMediaType, &o_devicesWithMediaType); + SCSwizzleClass(D, @selector(devices), (IMP)sc_devices, &o_devices); + SCSwizzleClass(D, @selector(deviceWithUniqueID:), (IMP)sc_deviceWithUniqueID, &o_deviceWithUniqueID); + SCSwizzleInstance([AVCaptureDeviceDiscoverySession class], @selector(devices), (IMP)sc_discoveryDevices, &o_discoveryDevices); + + Class I = [AVCaptureDeviceInput class]; + SCSwizzleInstance(I, @selector(initWithDevice:error:), (IMP)sc_inputInit, &o_inputInit); + SCSwizzleClass(I, @selector(deviceInputWithDevice:error:), (IMP)sc_inputFactory, &o_inputFactory); + + Class S = [AVCaptureSession class]; + SCSwizzleInstance(S, @selector(canAddInput:), (IMP)sc_canAddInput, &o_canAddInput); + SCSwizzleInstance(S, @selector(addInput:), (IMP)sc_addInput, &o_addInput); + SCSwizzleInstance(S, @selector(removeInput:), (IMP)sc_removeInput, &o_removeInput); + SCSwizzleInstance(S, @selector(inputs), (IMP)sc_inputs, &o_inputs); + SCSwizzleInstance(S, @selector(canAddOutput:), (IMP)sc_canAddOutput, &o_canAddOutput); + SCSwizzleInstance(S, @selector(addOutput:), (IMP)sc_addOutput, &o_addOutput); + SCSwizzleInstance(S, @selector(removeOutput:), (IMP)sc_removeOutput, &o_removeOutput); + SCSwizzleInstance(S, @selector(startRunning), (IMP)sc_startRunning, &o_startRunning); + SCSwizzleInstance(S, @selector(stopRunning), (IMP)sc_stopRunning, &o_stopRunning); + SCSwizzleInstance(S, @selector(isRunning), (IMP)sc_isRunning, &o_isRunning); + SCSwizzleInstance(S, @selector(beginConfiguration), (IMP)sc_beginConfig, &o_beginConfig); + SCSwizzleInstance(S, @selector(commitConfiguration), (IMP)sc_commitConfig, &o_commitConfig); + + Class O = [AVCaptureOutput class]; + SCSwizzleInstance(O, @selector(connectionWithMediaType:), (IMP)sc_connectionWithMediaType, &o_connectionWithMediaType); + SCSwizzleInstance(O, @selector(connections), (IMP)sc_connections, &o_connections); + Class M = [AVCaptureMetadataOutput class]; + SCSwizzleInstance(M, @selector(setMetadataObjectTypes:), (IMP)sc_setMetaTypes, &o_setMetaTypes); + SCSwizzleInstance(M, @selector(metadataObjectTypes), (IMP)sc_metaTypes, &o_metaTypes); + SCSwizzleInstance(M, @selector(availableMetadataObjectTypes), (IMP)sc_availableMetaTypes, &o_availableMetaTypes); + SCSwizzleInstance(M, @selector(setRectOfInterest:), (IMP)sc_setRectOfInterest, &o_setRectOfInterest); + SCSwizzleInstance(M, @selector(rectOfInterest), (IMP)sc_rectOfInterest, &o_rectOfInterest); + SCSwizzleInstance([AVCapturePhotoOutput class], @selector(capturePhotoWithSettings:delegate:), (IMP)sc_capturePhoto, &o_capturePhoto); + + Class L = [AVCaptureVideoPreviewLayer class]; + SCSwizzleInstance(L, @selector(connection), (IMP)sc_layerConnection, &o_layerConnection); + SCSwizzleInstance(L, @selector(setSession:), (IMP)sc_layerSetSession, &o_layerSetSession); + SCSwizzleInstance(L, @selector(initWithSession:), (IMP)sc_layerInitSession, &o_layerInitSession); + SCSwizzleInstance(L, @selector(initWithSessionWithNoConnection:), (IMP)sc_layerInitNoConn, &o_layerInitNoConn); + SCSwizzleInstance(L, @selector(transformedMetadataObjectForMetadataObject:), (IMP)sc_transformMeta, &o_transformMeta); + + SCLog(@"camera injected (frames: %s, or the built-in test pattern)", SC_FRAME_STORE_PATH); +} diff --git a/SimulatorInjection/build.sh b/SimulatorInjection/build.sh new file mode 100755 index 0000000..7846f9a --- /dev/null +++ b/SimulatorInjection/build.sh @@ -0,0 +1,18 @@ +#!/bin/zsh +# Builds libSimCamInject.dylib for the iOS Simulator (arm64 + x86_64). +# Usage: SimulatorInjection/build.sh [output-dir] (default: build/SimulatorInjection) +set -euo pipefail +# When run from the Mac app's build phase, drop the macOS build settings. +unset SDKROOT MACOSX_DEPLOYMENT_TARGET IPHONEOS_DEPLOYMENT_TARGET ARCHS 2>/dev/null || true +HERE=${0:A:h} +OUT=${1:-$HERE/../build/SimulatorInjection} +mkdir -p "$OUT" +SDK=$(xcrun --sdk iphonesimulator --show-sdk-path) +xcrun clang -dynamiclib -fobjc-arc -fmodules -O2 -Wall -Wno-unused-variable \ + -isysroot "$SDK" -arch arm64 -arch x86_64 -mios-simulator-version-min=16.0 \ + -framework Foundation -framework AVFoundation -framework CoreImage -framework CoreMedia \ + -framework CoreVideo -framework UIKit -framework QuartzCore \ + -install_name @rpath/libSimCamInject.dylib \ + -o "$OUT/libSimCamInject.dylib" "$HERE/SimCamInject.m" +codesign -s - --force "$OUT/libSimCamInject.dylib" >/dev/null +echo "$OUT/libSimCamInject.dylib" diff --git a/project.yml b/project.yml index 9d4da84..33ab8d9 100644 --- a/project.yml +++ b/project.yml @@ -62,6 +62,13 @@ targets: CODE_SIGN_STYLE: Automatic ENABLE_HARDENED_RUNTIME: YES GENERATE_INFOPLIST_FILE: NO + ENABLE_USER_SCRIPT_SANDBOXING: NO # the bundling script reads SimulatorInjection/ + postBuildScripts: + - name: Bundle SimulatorInjection + basedOnDependencyAnalysis: false + script: | + # The library simulator apps load to see SimulatorCamera (see SimulatorInjection/). + "$SRCROOT/SimulatorInjection/build.sh" "$TARGET_BUILD_DIR/$UNLOCALIZED_RESOURCES_FOLDER_PATH" dependencies: - target: SimulatorCameraExtension copy: diff --git a/simcamctl/FeedCommand.swift b/simcamctl/FeedCommand.swift new file mode 100644 index 0000000..f570e78 --- /dev/null +++ b/simcamctl/FeedCommand.swift @@ -0,0 +1,168 @@ +// +// FeedCommand.swift +// simcamctl +// +// `simcamctl feed …` publishes frames straight to the simulator frame file +// (FrameStore) at 30 fps, without the Mac app or the camera extension. +// Simulator apps launched with SimulatorInjection's library show them. +// Runs until interrupted (Ctrl-C) or for --seconds N. +// + +import AVFoundation +import CoreImage +import Foundation + +final class FeedCommand: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, @unchecked Sendable { + enum Source { + case pattern + case qr(String) + case image(String) + case camera + } + + private let store = FrameStore() + private let width = 1280, height = 720 + private var still: CVPixelBuffer? + private var tick = 0 + private var session: AVCaptureSession? + private var written = 0 + + func run(_ source: Source, seconds: Double?) -> Never { + switch source { + case .pattern: break + case .qr(let text): still = render(qr: text) + case .image(let path): still = render(imageAt: path) + case .camera: startCamera() + } + if case .qr = source, still == nil { fail("could not render the QR code") } + if case .image = source, still == nil { fail("could not read the image") } + + let started = Date() + let timer = DispatchSource.makeTimerSource(queue: DispatchQueue(label: "feed")) + timer.schedule(deadline: .now(), repeating: .milliseconds(33)) + timer.setEventHandler { [self] in + if let seconds, Date().timeIntervalSince(started) > seconds { finish() } + switch source { + case .camera: break // frames arrive from the capture delegate + case .pattern: write(pattern()) + case .qr, .image: if let still { write(still) } + } + if tick % 150 == 0 { print("feeding \(store.description(written: written))"); fflush(stdout) } + tick += 1 + } + timer.resume() + signal(SIGINT) { _ in exit(0) } + print("simcamctl: feeding \(FrameStore.path) — launch simulator apps with SimulatorInjection") + dispatchMain() + } + + private func finish() -> Never { + print("done: \(written) frames") + exit(0) + } + + private func fail(_ message: String) -> Never { + FileHandle.standardError.write(Data("error: \(message)\n".utf8)) + exit(3) + } + + private func write(_ pb: CVPixelBuffer) { + CVPixelBufferLockBaseAddress(pb, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(pb, .readOnly) } + guard let base = CVPixelBufferGetBaseAddress(pb) else { return } + store.write(width: CVPixelBufferGetWidth(pb), height: CVPixelBufferGetHeight(pb), + bytesPerRow: CVPixelBufferGetBytesPerRow(pb), bgra: base, sourceKind: 0, + time: Date().timeIntervalSince1970) + written += 1 + } + + // MARK: Sources + + private func newBuffer() -> CVPixelBuffer? { + var pb: CVPixelBuffer? + CVPixelBufferCreate(nil, width, height, kCVPixelFormatType_32BGRA, + [kCVPixelBufferCGBitmapContextCompatibilityKey: true] as CFDictionary, &pb) + return pb + } + + private func withContext(_ pb: CVPixelBuffer, _ draw: (CGContext) -> Void) { + CVPixelBufferLockBaseAddress(pb, []) + defer { CVPixelBufferUnlockBaseAddress(pb, []) } + guard let ctx = CGContext(data: CVPixelBufferGetBaseAddress(pb), width: width, height: height, + bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pb), + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) + else { return } + draw(ctx) + } + + private func pattern() -> CVPixelBuffer { + let pb = newBuffer()! + let t = tick + withContext(pb) { ctx in + let bars: [(CGFloat, CGFloat, CGFloat)] = [(0.75, 0.75, 0.75), (0.75, 0.75, 0), (0, 0.75, 0.75), (0, 0.75, 0), + (0.75, 0, 0.75), (0.75, 0, 0), (0, 0, 0.75)] + let w = CGFloat(width) / 7 + for (i, c) in bars.enumerated() { + ctx.setFillColor(red: c.0, green: c.1, blue: c.2, alpha: 1) + ctx.fill(CGRect(x: CGFloat(i) * w, y: 0, width: w + 1, height: CGFloat(height))) + } + ctx.setFillColor(red: 1, green: 1, blue: 1, alpha: 0.9) + ctx.fill(CGRect(x: CGFloat((t * 8) % width), y: 0, width: 24, height: CGFloat(height))) + } + return pb + } + + private func render(qr text: String) -> CVPixelBuffer? { + let filter = CIFilter(name: "CIQRCodeGenerator") + filter?.setValue(Data(text.utf8), forKey: "inputMessage") + filter?.setValue("M", forKey: "inputCorrectionLevel") + guard let qr = filter?.outputImage, let pb = newBuffer() else { return nil } + let side = 600.0 + let scaled = qr.transformed(by: CGAffineTransform(scaleX: side / qr.extent.width, y: side / qr.extent.height)) + guard let cg = CIContext().createCGImage(scaled, from: scaled.extent) else { return nil } + withContext(pb) { ctx in + ctx.setFillColor(red: 1, green: 1, blue: 1, alpha: 1) + ctx.fill(CGRect(x: 0, y: 0, width: width, height: height)) + ctx.interpolationQuality = .none + ctx.draw(cg, in: CGRect(x: (Double(width) - side) / 2, y: (Double(height) - side) / 2, width: side, height: side)) + } + return pb + } + + private func render(imageAt path: String) -> CVPixelBuffer? { + guard let src = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil), + let img = CGImageSourceCreateImageAtIndex(src, 0, nil), let pb = newBuffer() else { return nil } + withContext(pb) { ctx in + ctx.setFillColor(red: 0, green: 0, blue: 0, alpha: 1) + ctx.fill(CGRect(x: 0, y: 0, width: width, height: height)) + let s = min(Double(width) / Double(img.width), Double(height) / Double(img.height)) + let w = Double(img.width) * s, h = Double(img.height) * s + ctx.draw(img, in: CGRect(x: (Double(width) - w) / 2, y: (Double(height) - h) / 2, width: w, height: h)) + } + return pb + } + + private func startCamera() { + guard let device = AVCaptureDevice.default(for: .video), + let input = try? AVCaptureDeviceInput(device: device) else { fail("no Mac camera available (or camera access denied)") } + let s = AVCaptureSession() + s.sessionPreset = .hd1280x720 + s.addInput(input) + let out = AVCaptureVideoDataOutput() + out.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA] + out.setSampleBufferDelegate(self, queue: DispatchQueue(label: "camera")) + s.addOutput(out) + s.startRunning() + session = s + print("using Mac camera: \(device.localizedName)") + } + + func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { + if let pb = CMSampleBufferGetImageBuffer(sampleBuffer) { write(pb) } + } +} + +extension FrameStore { + func description(written: Int) -> String { "\(written) frames written" } +} diff --git a/simcamctl/main.swift b/simcamctl/main.swift index 9b396df..086ca40 100644 --- a/simcamctl/main.swift +++ b/simcamctl/main.swift @@ -32,6 +32,9 @@ func usage() -> Never { set-source --pattern Use the built-in test pattern set-source --qr "PAYLOAD" Generate + stream a QR code set-source --image PATH Stream a static PNG/JPG file + feed --pattern|--qr TEXT|--image PATH|--camera [--seconds N] + Stream frames to iOS Simulator apps launched + with SimulatorInjection (no extension needed) help Show this help EXIT CODES: @@ -54,6 +57,23 @@ func usage() -> Never { if args.count < 2 { usage() } let subcommand = args[1] + +if subcommand == "feed" { + guard args.count >= 3 else { usage() } + var seconds: Double? + if let i = args.firstIndex(of: "--seconds"), i + 1 < args.count { seconds = Double(args[i + 1]) } + let value = args.count >= 4 ? args[3] : "" + let source: FeedCommand.Source + switch args[2] { + case "--pattern": source = .pattern + case "--camera": source = .camera + case "--qr" where !value.isEmpty: source = .qr(value) + case "--image" where !value.isEmpty: source = .image(value) + default: usage() + } + FeedCommand().run(source, seconds: seconds) +} + let cli = SimCamCLIClient() switch subcommand {