Verify Swift OTLP delivery across XCUITest runner replacement

.buildkite/pipeline.yml

@@ -203,10 +203,11 @@ steps:
     steps:
       - label: ":swift: Swift: XCTest (test-collector-swift)"
         commands:
+          - git clone --branch swift-otel-executions --depth 1 https://github.com/buildkite/bktest .bktest
           - cd swift-xctest
           - swift package resolve
           # Run XCTest tests (which includes the BuildkiteTestCollector).
-          # The collector auto-loads via XCTest observation and uploads results
+          # The collector auto-loads via XCTest observation and exports results
           # when BUILDKITE_ANALYTICS_TOKEN is set.
           # Use --disable-swift-testing to only run XCTest (not Swift Testing).
           - swift test --disable-swift-testing
@@ -229,19 +230,26 @@ steps:
                 - BUILDKITE_ANALYTICS_DEBUG_ENABLED
 
       - label: ":swift: Swift: XCUITest (test-collector-swift)"
-        command: swift-xcuitest/bin/test
+        commands:
+          - git clone --branch swift-otel-executions --depth 1 https://github.com/buildkite/bktest .bktest
+          - bktec run
         agents:
           queue: macos
         artifact_paths:
           - swift-xcuitest/artifacts/SwiftXCUITest.xcresult/**/*
+          - swift-xcuitest/artifacts/xcodebuild.log
         env:
+          BUILDKITE_ANALYTICS_DEBUG_ENABLED: "true"
           BUILDKITE_ANALYTICS_TAGS: '{"test.framework.name":"xcuitest","language.name":"swift","custom.tag.from":"upload"}'
+          BUILDKITE_TESTS_OTLP_RELAY: "true"
         plugins:
-          - tests#v1.0.0:
+          - tests#v1.0.1:
               test-runner: custom
+              test-cmd: swift-xcuitest/bin/test
+              test-file-pattern: swift-xcuitest/ExampleAppUITests/**/*.swift
               suite-slug: test-engine-client-examples
               upload-results: false
-              install-client: false
+              client-version: v3.1.0-rc.1
 
   - group: ":android: Android"
     steps:

.gitignore

@@ -1,6 +1,7 @@
 node_modules/
 tmp/
 bktec
+.bktest/
 
 # pytest
 *.egg*

swift-xctest/Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version: 5.9
+// swift-tools-version: 5.10
 import PackageDescription
 
 let package = Package(
@@ -10,7 +10,7 @@ let package = Package(
         .library(name: "ExampleLib", targets: ["ExampleLib"])
     ],
     dependencies: [
-        .package(url: "https://github.com/buildkite/test-collector-swift", from: "0.6.0")
+        .package(path: "../.bktest/test-collector-swift")
     ],
     targets: [
         .target(

swift-xcuitest/ExampleApp/ExampleApp.swift

@@ -1,12 +1,44 @@
+import Foundation
 import SwiftUI
 
 @main
 struct ExampleApp: App {
     var body: some Scene {
         WindowGroup {
+            ContentView()
+        }
+    }
+}
+
+private struct ContentView: View {
+    private let runID: String
+    @State private var runnerCrashRecorded: Bool
+
+    init() {
+        let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local"
+        self.runID = runID
+        self._runnerCrashRecorded = State(
+            initialValue: UserDefaults.standard.string(forKey: "runner-crash-run-id") == runID
+        )
+    }
+
+    var body: some View {
+        VStack {
             Text("Swift XCUITest example")
                 .accessibilityIdentifier("status")
-                .padding()
+
+            if runnerCrashRecorded {
+                Text("Runner crash recorded")
+                    .accessibilityIdentifier("runner-crash-recorded")
+            } else {
+                Button("Record runner crash") {
+                    UserDefaults.standard.set(runID, forKey: "runner-crash-run-id")
+                    UserDefaults.standard.synchronize()
+                    runnerCrashRecorded = true
+                }
+                .accessibilityIdentifier("record-runner-crash")
+            }
         }
+        .padding()
     }
 }

swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift

@@ -1,4 +1,5 @@
 import Core
+import Darwin
 import XCTest
 
 final class ExampleAppUITests: XCTestCase {
@@ -7,18 +8,52 @@ final class ExampleAppUITests: XCTestCase {
     }
 
     func test01AppLaunches() {
-        let app = XCUIApplication()
-        app.launch()
+        let app = launchApp()
 
         XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5))
     }
 
     func test02ExecutionTag() {
         tagExecution("custom.tag.from", "execution")
 
-        let app = XCUIApplication()
-        app.launch()
+        let app = launchApp()
 
         XCTAssertEqual(app.staticTexts["status"].label, "Swift XCUITest example")
     }
+
+    func test03RunnerCrashesOnce() {
+        let app = launchApp()
+
+        if app.buttons["record-runner-crash"].waitForExistence(timeout: 2) {
+            app.buttons["record-runner-crash"].tap()
+            XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2))
+
+            _ = kill(getpid(), SIGKILL)
+            XCTFail("The runner process should have terminated")
+        }
+
+        XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2))
+    }
+
+    func test04RunsInReplacementRunner() {
+        let app = launchApp()
+
+        XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 5))
+    }
+
+    func test05ExecutionTagAfterRestart() {
+        tagExecution("runner.replacement", "true")
+
+        let app = launchApp()
+        XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5))
+        XCTAssertEqual(app.staticTexts["status"].label, "Swift XCUITest example")
+    }
+
+    private func launchApp() -> XCUIApplication {
+        let app = XCUIApplication()
+        let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local"
+        app.launchEnvironment["XCUITEST_RUN_ID"] = runID
+        app.launch()
+        return app
+    }
 }