Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,29 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
- Extension state is queried at launch (`propertiesRequest`) so a relaunched
app shows "Active" instead of "Not checked". Leftover activation diagnostics
that wrote files into `~/Library/Logs` are gone.
- **Injected camera no longer crashes apps.** The `-init` swizzles use
init-family ownership (callers of `-[AVCaptureDeviceInput
initWithDevice:error:]` over-released the fake input), the fake objects skip
AVFoundation's `-dealloc` (it crashed on their never-initialised state), and
adding a second output to a fake session no longer crashes in
`-canAddOutput:`. Found with CommunityToolkit.Maui.Camera.
- **Injected photos have a camera's shape.** `capturePhoto` returns a centre
4:3 crop of the frame, portrait when the app captures in portrait, instead
of the raw landscape frame.
- **Incremental builds keep a valid signature.** The dylib bundling phase
declares its inputs and output, so Xcode re-signs the app after the dylib
changes.
- **Extension activates.** Its bundle is named after its bundle identifier
(`com.dautov.SimulatorCamera.Extension.systemextension`), as sysextd
requires.
- **Sink stream opens.** `CMIOStreamCopyBufferQueue` gets a no-op queue-altered
callback; with nil it returned `noErr` but no queue.
- **Camera link attaches reliably.** The app requests camera access before
attaching (without it the CMIO device list is empty) and re-resolves device
and stream IDs on every attach (stale IDs failed with `'!str'`).
- **Sink producers are recognised** when CoreMediaIO reports their signing ID
as nil or "unknown": the extension reads it from the kernel for that pid and
accepts only a valid, non-ad-hoc signature from its own team.

### Added
- `SimulatorCameraTests`: 20 logic tests for the contract, status encoding,
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ We ship `patch-package`-compatible patches for known libraries:
SimulatorCamera.app (container)
↓ CoreMediaIO sink stream (IOSurface frames) + custom properties
.app/Contents/Library/SystemExtensions/
SimulatorCameraExtension.systemextension
com.dautov.SimulatorCamera.Extension.systemextension
↓ CMIOExtensionStream.send
macOS CoreMediaIO subsystem
↓ host AVFoundation
Expand Down
7 changes: 4 additions & 3 deletions Shared/CMIOSinkClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -216,10 +216,11 @@ public final class CMIOSinkClient: @unchecked Sendable {
guard device != 0, sink != 0 else { throw CMIOSinkClientError.deviceNotFound }

var queueRef: Unmanaged<CMSimpleQueue>?
// No "queue altered" callback: we size our own pacing by frame rate
// and use the queue count as backpressure instead.
// We pace by frame rate and use the queue count as backpressure, so the
// "queue altered" callback is a no-op. It must still be non-nil:
// with a nil proc CoreMediaIO returns noErr but no queue.
let copyStatus = withUnsafeMutablePointer(to: &queueRef) { ptr in
CMIOStreamCopyBufferQueue(sink, nil, nil, ptr)
CMIOStreamCopyBufferQueue(sink, { _, _, _ in }, nil, ptr)
}
guard copyStatus == noErr, let queue = queueRef?.takeRetainedValue() else {
throw CMIOSinkClientError.osStatus("CMIOStreamCopyBufferQueue", copyStatus)
Expand Down
9 changes: 6 additions & 3 deletions SimulatorCamera.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -307,18 +307,21 @@
/* Begin PBXShellScriptBuildPhase section */
98537C620109260D815B0C49 /* Bundle SimulatorInjection */ = {
isa = PBXShellScriptBuildPhase;
alwaysOutOfDate = 1;
buildActionMask = 2147483647;
files = (
);
inputFileListPaths = (
);
inputPaths = (
"$(SRCROOT)/SimulatorInjection/SimCamInject.m",
"$(SRCROOT)/SimulatorInjection/SCFrameStore.h",
"$(SRCROOT)/SimulatorInjection/build.sh",
);
name = "Bundle SimulatorInjection";
outputFileListPaths = (
);
outputPaths = (
"$(TARGET_BUILD_DIR)/$(UNLOCALIZED_RESOURCES_FOLDER_PATH)/libSimCamInject.dylib",
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
Expand Down Expand Up @@ -641,7 +644,7 @@
"@executable_path/../Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = com.dautov.SimulatorCamera.Extension;
PRODUCT_NAME = SimulatorCameraExtension;
PRODUCT_NAME = "$(PRODUCT_BUNDLE_IDENTIFIER)";
SDKROOT = macosx;
SKIP_INSTALL = YES;
};
Expand Down Expand Up @@ -682,7 +685,7 @@
"@executable_path/../Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = com.dautov.SimulatorCamera.Extension;
PRODUCT_NAME = SimulatorCameraExtension;
PRODUCT_NAME = "$(PRODUCT_BUNDLE_IDENTIFIER)";
SDKROOT = macosx;
SKIP_INSTALL = YES;
};
Expand Down
13 changes: 10 additions & 3 deletions SimulatorCamera/CameraLink.swift
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,12 @@ final class CameraLink {
Task { @MainActor [weak self] in self?.attachNow() }
}
}
attachNow()
// A sandboxed app without camera (TCC) access gets an empty CMIO
// device list, so the virtual camera looks "not registered".
Task { @MainActor [weak self] in
_ = await self?.requestCameraAccessIfNeeded()
self?.attachNow()
}
}

func disconnect() {
Expand Down Expand Up @@ -144,15 +149,17 @@ final class CameraLink {
/// frames pushed while the extension still shows the test pattern are
/// consumed and ignored, so there is no harm in opening early.
func openSink() throws {
if !client.isAttached { try client.attach() }
// Always re-resolve: device and stream IDs change whenever the
// extension restarts or is replaced, and stale IDs fail with '!str'.
try client.attach()
try client.openSink()
framesRejected = 0
}

/// Declare the active source to the extension. Selecting the test
/// pattern also closes the sink.
func setSource(_ kind: SimCamSourceKind) throws {
if !client.isAttached { try client.attach() }
try client.attach()
if kind == .testPattern {
client.closeSink()
} else {
Expand Down
75 changes: 73 additions & 2 deletions SimulatorCameraExtension/SimulatorCameraProvider.swift
Original file line number Diff line number Diff line change
Expand Up @@ -614,14 +614,50 @@ final class SimulatorCameraSinkStream: NSObject, CMIOExtensionStreamSource {
/// so forks that keep the naming pattern need no changes.
func authorizedToStartStream(for client: CMIOExtensionClient) -> Bool {
let allowed = Self.allowedProducerSigningIDs()
guard let signingID = client.signingID, allowed.contains(signingID) else {
log.error("refusing sink producer pid \(client.pid) signingID=\(client.signingID ?? "nil", privacy: .public)")
// CoreMediaIO cannot always resolve the client process and then
// reports its signing ID as nil or "unknown". Ask the code signing
// machinery about the pid directly in that case.
var signingID = client.signingID
if signingID == nil || signingID == "unknown" {
signingID = Self.sameTeamSigningID(ofPID: client.pid)
}
guard let signingID, allowed.contains(signingID) else {
log.error("refusing sink producer pid \(client.pid) signingID=\(client.signingID ?? "nil", privacy: .public) resolved=\(signingID ?? "nil", privacy: .public)")
return false
}
_client = client
return true
}

/// Signing identifier of the process `pid`, or nil unless the kernel
/// reports a valid, certificate-backed (not ad-hoc) signature from the
/// same team as this extension. The team check matters: anyone can
/// ad-hoc sign a binary with our identifier.
///
/// This asks the kernel (csops) rather than Security.framework because
/// SecCodeCheckValidity re-reads the client's executable from disk,
/// which the extension sandbox denies, and a file-read exception
/// entitlement makes the extension crash at sandbox init.
static func sameTeamSigningID(ofPID pid: pid_t) -> String? {
guard let team = CodeSigning.teamID(of: getpid()) else {
log.error("signing check: could not read own team ID")
return nil
}
guard let flags = CodeSigning.status(of: pid) else {
log.error("signing check: no code signing status for pid \(pid) (errno \(errno))")
return nil
}
guard flags & CodeSigning.valid != 0, flags & CodeSigning.adhoc == 0 else {
log.error("signing check: pid \(pid) signature not valid or ad-hoc (flags 0x\(String(flags, radix: 16), privacy: .public))")
return nil
}
guard CodeSigning.teamID(of: pid) == team else {
log.error("signing check: pid \(pid) team \(CodeSigning.teamID(of: pid) ?? "nil", privacy: .public) != \(team, privacy: .public)")
return nil
}
return CodeSigning.identity(of: pid)
}

static func allowedProducerSigningIDs(extensionBundleID: String? = Bundle.main.bundleIdentifier) -> Set<String> {
guard let ext = extensionBundleID else { return [] }
let app = ext.hasSuffix(".Extension") ? String(ext.dropLast(".Extension".count)) : ext
Expand Down Expand Up @@ -702,3 +738,38 @@ final class SimulatorCameraProviderSource: NSObject, CMIOExtensionProviderSource
// No settable properties.
}
}

/// Minimal wrapper over the kernel's code signing queries. `csops` is not in
/// the public headers but is exported by libsystem_kernel and is what
/// Security.framework itself uses for dynamic (running process) checks.
private enum CodeSigning {
static let valid: UInt32 = 0x1 // CS_VALID
static let adhoc: UInt32 = 0x2 // CS_ADHOC

private static let opsStatus: UInt32 = 0 // CS_OPS_STATUS
private static let opsIdentity: UInt32 = 11 // CS_OPS_IDENTITY
private static let opsTeamID: UInt32 = 14 // CS_OPS_TEAMID

static func status(of pid: pid_t) -> UInt32? {
var flags: UInt32 = 0
return csops(pid, opsStatus, &flags, MemoryLayout<UInt32>.size) == 0 ? flags : nil
}

static func identity(of pid: pid_t) -> String? { blobString(pid, opsIdentity) }
static func teamID(of pid: pid_t) -> String? { blobString(pid, opsTeamID) }

/// Identity and team ID come back as a blob: 4-byte magic, 4-byte
/// big-endian total length, then a NUL-terminated string.
private static func blobString(_ pid: pid_t, _ op: UInt32) -> String? {
var buffer = [UInt8](repeating: 0, count: 512)
let result = buffer.withUnsafeMutableBytes { csops(pid, op, $0.baseAddress, $0.count) }
guard result == 0 else { return nil }
let length = Int(buffer[4]) << 24 | Int(buffer[5]) << 16 | Int(buffer[6]) << 8 | Int(buffer[7])
guard length > 8 else { return nil }
let bytes = buffer[8..<min(length, buffer.count)].prefix { $0 != 0 }
return bytes.isEmpty ? nil : String(decoding: bytes, as: UTF8.self)
}
}

@_silgen_name("csops")
private func csops(_ pid: pid_t, _ ops: UInt32, _ useraddr: UnsafeMutableRawPointer?, _ usersize: Int) -> Int32
93 changes: 81 additions & 12 deletions SimulatorInjection/SimCamInject.m
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#import <CoreImage/CoreImage.h>
#import <UIKit/UIKit.h>
#import <objc/runtime.h>
#import <objc/message.h>
#import <sys/mman.h>
#import <fcntl.h>
#import <stdatomic.h>
Expand All @@ -39,6 +40,14 @@ static void SCLog(NSString *fmt, ...) {

#pragma mark - Swizzling helpers

// -init replacements are plain C functions, so ARC would treat them as +0
// returns with an unowned self. Declare init-family ownership explicitly:
// self is consumed and the result is +1, matching what callers of -init expect.
#define SC_INIT_RESULT __attribute__((ns_returns_retained)) id
#define SC_INIT_SELF __attribute__((ns_consumed)) id
typedef SC_INIT_RESULT (*SCInitIMP1)(SC_INIT_SELF, SEL, id);
typedef SC_INIT_RESULT (*SCInitIMP2)(SC_INIT_SELF, SEL, id, NSError **);

static void SCSwizzleInstance(Class cls, SEL sel, IMP newImp, IMP *orig) {
Method m = class_getInstanceMethod(cls, sel);
if (!m) { SCLog(@"missing -[%@ %@]", cls, NSStringFromSelector(sel)); return; }
Expand Down Expand Up @@ -150,6 +159,7 @@ @implementation SCConnection {
}
- (NSArray *)inputPorts { return @[]; }
- (AVMediaType)mediaType { return AVMediaTypeVideo; } // private, read by AVCaptureSession internals
- (AVCaptureDevice *)sourceDevice { return SCSharedDevice(); } // private, read by -canAddOutput:/-addOutput:
- (AVCaptureOutput *)output { return nil; }
- (AVCaptureVideoPreviewLayer *)videoPreviewLayer { return nil; }
- (BOOL)isEnabled { return YES; }
Expand Down Expand Up @@ -204,7 +214,6 @@ @implementation SCPhoto {
@public NSData *_data;
@public CMTime _time;
}
- (void)dealloc { if (_pixels) CVPixelBufferRelease(_pixels); }
- (CMTime)timestamp { return _time; }
- (BOOL)isRawPhoto { return NO; }
- (CVPixelBufferRef)pixelBuffer { return _pixels; }
Expand All @@ -221,6 +230,26 @@ - (NSInteger)photoCount { return 1; }
- (AVCaptureResolvedPhotoSettings *)resolvedSettings { return nil; }
@end

// The fakes never ran their AVFoundation -init, so the AVFoundation -dealloc
// would tear down internals that were never set up (it crashes in
// -[AVCaptureConnection dealloc]). Go straight to NSObject's -dealloc, which
// still releases our ivars (.cxx_destruct) and associated objects.
static void SCFakeDealloc(__unsafe_unretained id self, SEL _cmd) {
if ([self isKindOfClass:[SCPhoto class]]) {
__unsafe_unretained SCPhoto *p = self;
if (p->_pixels) CVPixelBufferRelease(p->_pixels);
p->_pixels = NULL;
}
struct objc_super sup = { self, [NSObject class] };
((void (*)(struct objc_super *, SEL))objc_msgSendSuper)(&sup, _cmd);
}

static void SCInstallFakeDealloc(void) {
SEL dealloc = sel_registerName("dealloc"); // ARC disallows @selector(dealloc)
for (Class c in @[[SCDevice class], [SCDeviceInput class], [SCConnection class], [SCQRObject class], [SCPhoto class]])
class_replaceMethod(c, dealloc, (IMP)SCFakeDealloc, "v@:");
}

#pragma mark - Frame source (Mac app frame file, or built-in test pattern)

static uint8_t *gStore; // mmap of SC_FRAME_STORE_PATH, read-only
Expand Down Expand Up @@ -505,9 +534,14 @@ static void SCUpdateTimer(void) {
in->_scDevice = (SCDevice *)d;
return in;
}
static id sc_inputInit(id self, SEL _cmd, AVCaptureDevice *d, NSError **e) {
if ([d isKindOfClass:[SCDevice class]]) return SCMakeInput(d);
return ((id (*)(id, SEL, AVCaptureDevice *, NSError **))o_inputInit)(self, _cmd, d, e);
static SC_INIT_RESULT sc_inputInit(SC_INIT_SELF self, SEL _cmd, AVCaptureDevice *d, NSError **e) {
if ([d isKindOfClass:[SCDevice class]]) {
// Keep the alloc'd-but-uninitialised receiver alive rather than
// running AVCaptureDeviceInput's -dealloc on it.
(void)CFBridgingRetain(self);
return SCMakeInput(d);
}
return ((SCInitIMP2)o_inputInit)(self, _cmd, d, e);
}
static id sc_inputFactory(id self, SEL _cmd, AVCaptureDevice *d, NSError **e) {
if ([d isKindOfClass:[SCDevice class]]) return SCMakeInput(d);
Expand Down Expand Up @@ -547,8 +581,9 @@ static void sc_removeInput(AVCaptureSession *self, SEL _cmd, AVCaptureInput *in)
return st.inputs.count ? [st.inputs arrayByAddingObjectsFromArray:r] : r;
}
static BOOL sc_canAddOutput(AVCaptureSession *self, SEL _cmd, AVCaptureOutput *o) {
BOOL ok = ((BOOL (*)(id, SEL, id))o_canAddOutput)(self, _cmd, o);
return ok || SCState(self).usesFake;
// The stock check walks existing outputs' (fake) connections; skip it.
if (SCState(self).usesFake) return ![SCState(self).outputs containsObject:o];
return ((BOOL (*)(id, SEL, id))o_canAddOutput)(self, _cmd, o);
}
static void sc_addOutput(AVCaptureSession *self, SEL _cmd, AVCaptureOutput *o) {
[SCState(self).outputs addObject:o];
Expand Down Expand Up @@ -653,13 +688,13 @@ static void sc_layerSetSession(AVCaptureVideoPreviewLayer *self, SEL _cmd, AVCap
((void (*)(id, SEL, id))o_layerSetSession)(self, _cmd, s);
SCTrackLayer(self);
}
static id sc_layerInitSession(AVCaptureVideoPreviewLayer *self, SEL _cmd, AVCaptureSession *s) {
id r = ((id (*)(id, SEL, id))o_layerInitSession)(self, _cmd, s);
static SC_INIT_RESULT sc_layerInitSession(SC_INIT_SELF self, SEL _cmd, AVCaptureSession *s) {
id r = ((SCInitIMP1)o_layerInitSession)(self, _cmd, s);
if (r) SCTrackLayer(r);
return r;
}
static id sc_layerInitNoConn(AVCaptureVideoPreviewLayer *self, SEL _cmd, AVCaptureSession *s) {
id r = ((id (*)(id, SEL, id))o_layerInitNoConn)(self, _cmd, s);
static SC_INIT_RESULT sc_layerInitNoConn(SC_INIT_SELF self, SEL _cmd, AVCaptureSession *s) {
id r = ((SCInitIMP1)o_layerInitNoConn)(self, _cmd, s);
if (r) SCTrackLayer(r);
return r;
}
Expand All @@ -681,11 +716,44 @@ static id sc_layerInitNoConn(AVCaptureVideoPreviewLayer *self, SEL _cmd, AVCaptu
return t;
}

/// Whether the app is capturing in portrait: its preview layer is taller than
/// wide, or (no layer) the connection rotates by 90/270 degrees.
static BOOL SCIsPortraitCapture(AVCaptureOutput *out) {
AVCaptureSession *s = objc_getAssociatedObject(out, kSCSessionKey);
[gLock lock]; NSArray *layers = gLayers.allObjects; [gLock unlock];
for (AVCaptureVideoPreviewLayer *l in layers) {
CGSize b = l.bounds.size;
if (l.session == s && b.width > 0 && b.height > 0) return b.height > b.width;
}
return fmod(SCConnectionFor(out)->_rotation, 180) != 0;
}

/// Centre crop to a width/height ratio. Frames are upright as the preview shows
/// them, so this is the photo a real camera takes around the previewed area.
static CVPixelBufferRef SCCropToAspect(CVPixelBufferRef pb, CGFloat aspect) {
size_t W = CVPixelBufferGetWidth(pb), H = CVPixelBufferGetHeight(pb);
size_t w = W, h = H;
if ((CGFloat)W / H > aspect) w = (size_t)llround(H * aspect); else h = (size_t)llround(W / aspect);
if (w == W && h == H) return CVPixelBufferRetain(pb);
CVPixelBufferRef out = SCNewBuffer(w, h);
if (!out) return CVPixelBufferRetain(pb);
CGFloat x = (W - w) / 2.0, y = (H - h) / 2.0;
CIImage *ci = [[[CIImage imageWithCVPixelBuffer:pb] imageByCroppingToRect:CGRectMake(x, y, w, h)]
imageByApplyingTransform:CGAffineTransformMakeTranslation(-x, -y)];
static CIContext *ctx; if (!ctx) ctx = [CIContext context];
[ctx render:ci toCVPixelBuffer:out];
return out;
}

static void sc_capturePhoto(AVCapturePhotoOutput *self, SEL _cmd, AVCapturePhotoSettings *settings, id<AVCapturePhotoCaptureDelegate> d) {
if (!SCOutputIsFake(self)) { ((void (*)(id, SEL, id, id))o_capturePhoto)(self, _cmd, settings, d); return; }
// iPhone photos are 4:3, portrait when the app captures in portrait.
CGFloat aspect = SCIsPortraitCapture(self) ? 3.0 / 4.0 : 4.0 / 3.0;
dispatch_async(gFrameQueue, ^{
CVPixelBufferRef pb = gLatest ? CVPixelBufferRetain(gLatest) : SCTestPattern(gFrameCount);
if (!pb) return;
CVPixelBufferRef frame = gLatest ? CVPixelBufferRetain(gLatest) : SCTestPattern(gFrameCount);
if (!frame) return;
CVPixelBufferRef pb = SCCropToAspect(frame, aspect);
CVPixelBufferRelease(frame);
SCPhoto *p = class_createInstance([SCPhoto class], 0);
p->_pixels = pb; // owned
p->_time = CMClockGetTime(CMClockGetHostTimeClock());
Expand All @@ -709,6 +777,7 @@ static void sc_capturePhoto(AVCapturePhotoOutput *self, SEL _cmd, AVCapturePhoto
gLayers = [NSHashTable weakObjectsHashTable];
gFrameQueue = dispatch_queue_create("com.dautov.SimulatorCamera.frames", DISPATCH_QUEUE_SERIAL);
gQR = [CIDetector detectorOfType:CIDetectorTypeQRCode context:nil options:@{ CIDetectorAccuracy: CIDetectorAccuracyHigh }];
SCInstallFakeDealloc();

Class D = [AVCaptureDevice class];
SCSwizzleClass(D, @selector(defaultDeviceWithMediaType:), (IMP)sc_defaultDevice, &o_defaultDevice);
Expand Down
Loading