Reproduce Swift collector loss after an XCUITest runner restart

.buildkite/pipeline.yml

@@ -234,7 +234,10 @@ steps:
           queue: macos
         artifact_paths:
           - swift-xcuitest/artifacts/SwiftXCUITest.xcresult/**/*
+          - swift-xcuitest/artifacts/xcodebuild.log
+        soft_fail: true
         env:
+          BUILDKITE_ANALYTICS_DEBUG_ENABLED: "true"
           BUILDKITE_ANALYTICS_TAGS: '{"test.framework.name":"xcuitest","language.name":"swift","custom.tag.from":"upload"}'
         plugins:
           - tests#v1.0.0:

swift-xcuitest/ExampleApp/ExampleApp.swift

@@ -1,12 +1,44 @@
+import Foundation
 import SwiftUI
 
 @main
 struct ExampleApp: App {
     var body: some Scene {
         WindowGroup {
+            ContentView()
+        }
+    }
+}
+
+private struct ContentView: View {
+    private let runID: String
+    @State private var runnerCrashRecorded: Bool
+
+    init() {
+        let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local"
+        self.runID = runID
+        self._runnerCrashRecorded = State(
+            initialValue: UserDefaults.standard.string(forKey: "runner-crash-run-id") == runID
+        )
+    }
+
+    var body: some View {
+        VStack {
             Text("Swift XCUITest example")
                 .accessibilityIdentifier("status")
-                .padding()
+
+            if runnerCrashRecorded {
+                Text("Runner crash recorded")
+                    .accessibilityIdentifier("runner-crash-recorded")
+            } else {
+                Button("Record runner crash") {
+                    UserDefaults.standard.set(runID, forKey: "runner-crash-run-id")
+                    UserDefaults.standard.synchronize()
+                    runnerCrashRecorded = true
+                }
+                .accessibilityIdentifier("record-runner-crash")
+            }
         }
+        .padding()
     }
 }

swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift

@@ -1,4 +1,5 @@
 import Core
+import Darwin
 import XCTest
 
 final class ExampleAppUITests: XCTestCase {
@@ -7,18 +8,56 @@ final class ExampleAppUITests: XCTestCase {
     }
 
     func test01AppLaunches() {
-        let app = XCUIApplication()
-        app.launch()
+        let app = launchApp()
 
         XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5))
     }
 
     func test02ExecutionTag() {
         tagExecution("custom.tag.from", "execution")
 
-        let app = XCUIApplication()
-        app.launch()
+        let app = launchApp()
 
         XCTAssertEqual(app.staticTexts["status"].label, "Swift XCUITest example")
     }
+
+    func test03RunnerCrashesOnce() {
+        let app = launchApp()
+
+        if app.buttons["record-runner-crash"].waitForExistence(timeout: 2) {
+            app.buttons["record-runner-crash"].tap()
+            XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2))
+
+            _ = kill(getpid(), SIGKILL)
+            XCTFail("The runner process should have terminated")
+        }
+
+        XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2))
+    }
+
+    func test04RunsInReplacementRunner() {
+        let app = launchApp()
+
+        XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 5))
+    }
+
+    func test05FailureIncludesDiagnostics() {
+        tagExecution("runner.replacement", "true")
+
+        let app = launchApp()
+        XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5))
+        XCTAssertEqual(
+            app.staticTexts["status"].label,
+            "Intentional failure",
+            "This failure demonstrates native XCTest diagnostics"
+        )
+    }
+
+    private func launchApp() -> XCUIApplication {
+        let app = XCUIApplication()
+        let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local"
+        app.launchEnvironment["XCUITEST_RUN_ID"] = runID
+        app.launch()
+        return app
+    }
 }

swift-xcuitest/README.md

@@ -1,9 +1,16 @@
 # Swift XCUITest example
 
-This example runs passing UI tests against a minimal iOS app and uploads the
-results with [`test-collector-swift`](https://github.com/buildkite/test-collector-swift).
-It uses a shared scheme, a lexical test plan, and an explicit `.xcresult`
-bundle so the same command works locally and on a macOS Buildkite agent.
+This example reproduces the result loss that occurs when XCTest replaces an
+XCUITest runner. Two tests finish before a third test terminates the runner.
+Xcode continues the remaining tests in a replacement process and keeps all
+five tests in the `.xcresult`, but `test-collector-swift` loses the completed
+executions buffered by the terminated process. The final test also fails with
+an execution tag, backtrace, and source location to exercise the richer native
+collector metadata.
+
+The verification script compares the `.xcresult` test count with the unique
+test names logged when the collector uploads. It intentionally fails while
+the counts differ. The Buildkite step is therefore soft-failed.
 
 Run it with:
 

swift-xcuitest/bin/test

@@ -5,6 +5,7 @@ set -euo pipefail
 root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 artifacts_dir="${XCUITEST_ARTIFACTS_DIR:-$root/artifacts}"
 result_bundle="$artifacts_dir/SwiftXCUITest.xcresult"
+test_log="$artifacts_dir/xcodebuild.log"
 
 for name in \
   BUILDKITE_ANALYTICS_BASE_URL \
@@ -34,6 +35,9 @@ do
   fi
 done
 
+run_id="${XCUITEST_RUN_ID:-${BUILDKITE_BUILD_ID:-local}-${BUILDKITE_JOB_ID:-$(date +%s)}}"
+export "TEST_RUNNER_XCUITEST_RUN_ID=$run_id"
+
 destination="${XCUITEST_DESTINATION:-}"
 if [[ -z "$destination" ]]; then
   device_id="$({
@@ -57,11 +61,25 @@ xcodebuild \
   -scheme SwiftXCUITest \
   -resolvePackageDependencies
 
+set +e
 xcodebuild \
   -project "$root/SwiftXCUITest.xcodeproj" \
   -scheme SwiftXCUITest \
   -testPlan SwiftXCUITest \
   -destination "$destination" \
   -derivedDataPath "$artifacts_dir/DerivedData" \
   -resultBundlePath "$result_bundle" \
-  test
+  -retry-tests-on-failure \
+  -test-iterations 2 \
+  test 2>&1 | tee "$test_log"
+xcodebuild_status="${PIPESTATUS[0]}"
+set -e
+
+verification_status=0
+"$root/bin/verify-results" "$result_bundle" "$test_log" || verification_status="$?"
+
+if [[ "$xcodebuild_status" -ne 0 ]]; then
+  exit "$xcodebuild_status"
+fi
+
+exit "$verification_status"

swift-xcuitest/bin/verify-results

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -euo pipefail
+
+result_bundle="${1:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}"
+test_log="${2:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}"
+expected_count=5
+
+result_count="$({
+  xcrun xcresulttool get test-results summary --path "$result_bundle"
+} | plutil -extract totalTestCount raw -o - -)"
+uploaded_count="$(awk '
+  {
+    line = $0
+    while (match(line, /name: Optional\("test[^"]*"\)/)) {
+      names[substr(line, RSTART, RLENGTH)] = 1
+      line = substr(line, RSTART + RLENGTH)
+    }
+  }
+  END {
+    for (name in names) count++
+    print count + 0
+  }
+' "$test_log")"
+
+echo "Expected unique tests: $expected_count"
+echo ".xcresult unique tests: $result_count"
+echo "Collector unique tests recorded for upload: $uploaded_count"
+
+status=0
+
+if [[ "$result_count" -ne "$expected_count" ]]; then
+  echo "Expected .xcresult to retain all $expected_count tests" >&2
+  status=1
+fi
+
+if grep -Fq 'location: Optional("ExampleAppUITests.swift:' "$test_log" \
+  && grep -Fq 'failureExpanded: [Core.Trace.FailureExpanded' "$test_log" \
+  && grep -Fq 'tags: Optional(["runner.replacement": "true"])' "$test_log"
+then
+  echo "Collector retained the failure location, expanded backtrace, and execution tag"
+else
+  echo "Collector did not log all expected native failure metadata" >&2
+  status=1
+fi
+
+if [[ "$uploaded_count" -ne "$result_count" ]]; then
+  echo "Collector lost executions when the XCTest runner was replaced" >&2
+  status=1
+fi
+
+exit "$status"