From 5be5c0708c173f157ced714e07b4ec4c0d19b770 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 13:18:30 +0930 Subject: [PATCH 1/6] Reproduce Swift collector loss after runner restart Extend the green XCUITest fixture with a crash-once marker persisted by the target app. Two tests finish before the UI test runner kills itself; Xcode then replaces the runner and continues the remaining tests. Capture collector debug output and compare its unique uploaded test names with the completed xcresult. The local reproduction retains all five tests in xcresult but records only the two tests run after replacement. Keep an intentional post-restart assertion failure to prove execution tags, source location, and expanded backtrace remain available on delivered traces. The pipeline step is soft-failed because this branch is intentionally a failing regression fixture until the collector gains durable delivery. Amp-Thread-ID: https://ampcode.com/threads/T-01a036f0-4e44-73ab-a085-68af2e00b5cb Co-authored-by: Amp --- .buildkite/pipeline.yml | 3 ++ swift-xcuitest/ExampleApp/ExampleApp.swift | 34 +++++++++++- .../ExampleAppUITests/ExampleAppUITests.swift | 47 +++++++++++++++-- swift-xcuitest/README.md | 15 ++++-- swift-xcuitest/bin/test | 20 ++++++- swift-xcuitest/bin/verify-results | 52 +++++++++++++++++++ 6 files changed, 161 insertions(+), 10 deletions(-) create mode 100755 swift-xcuitest/bin/verify-results diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 15c9d94..ed3e67e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -234,7 +234,10 @@ steps: queue: macos artifact_paths: - swift-xcuitest/artifacts/SwiftXCUITest.xcresult/**/* + - swift-xcuitest/artifacts/xcodebuild.log + soft_fail: true env: + BUILDKITE_ANALYTICS_DEBUG_ENABLED: "true" BUILDKITE_ANALYTICS_TAGS: '{"test.framework.name":"xcuitest","language.name":"swift","custom.tag.from":"upload"}' plugins: - tests#v1.0.0: diff --git a/swift-xcuitest/ExampleApp/ExampleApp.swift b/swift-xcuitest/ExampleApp/ExampleApp.swift index 9d0a04a..d8dc18f 100644 --- a/swift-xcuitest/ExampleApp/ExampleApp.swift +++ b/swift-xcuitest/ExampleApp/ExampleApp.swift @@ -1,12 +1,44 @@ +import Foundation import SwiftUI @main struct ExampleApp: App { var body: some Scene { WindowGroup { + ContentView() + } + } +} + +private struct ContentView: View { + private let runID: String + @State private var runnerCrashRecorded: Bool + + init() { + let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local" + self.runID = runID + self._runnerCrashRecorded = State( + initialValue: UserDefaults.standard.string(forKey: "runner-crash-run-id") == runID + ) + } + + var body: some View { + VStack { Text("Swift XCUITest example") .accessibilityIdentifier("status") - .padding() + + if runnerCrashRecorded { + Text("Runner crash recorded") + .accessibilityIdentifier("runner-crash-recorded") + } else { + Button("Record runner crash") { + UserDefaults.standard.set(runID, forKey: "runner-crash-run-id") + UserDefaults.standard.synchronize() + runnerCrashRecorded = true + } + .accessibilityIdentifier("record-runner-crash") + } } + .padding() } } diff --git a/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift b/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift index 500ebcf..68285d6 100644 --- a/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift +++ b/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift @@ -1,4 +1,5 @@ import Core +import Darwin import XCTest final class ExampleAppUITests: XCTestCase { @@ -7,8 +8,7 @@ final class ExampleAppUITests: XCTestCase { } func test01AppLaunches() { - let app = XCUIApplication() - app.launch() + let app = launchApp() XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5)) } @@ -16,9 +16,48 @@ final class ExampleAppUITests: XCTestCase { func test02ExecutionTag() { tagExecution("custom.tag.from", "execution") - let app = XCUIApplication() - app.launch() + let app = launchApp() XCTAssertEqual(app.staticTexts["status"].label, "Swift XCUITest example") } + + func test03RunnerCrashesOnce() { + let app = launchApp() + + if app.buttons["record-runner-crash"].waitForExistence(timeout: 2) { + app.buttons["record-runner-crash"].tap() + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2)) + + _ = kill(getpid(), SIGKILL) + XCTFail("The runner process should have terminated") + } + + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2)) + } + + func test04RunsInReplacementRunner() { + let app = launchApp() + + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 5)) + } + + func test05FailureIncludesDiagnostics() { + tagExecution("runner.replacement", "true") + + let app = launchApp() + XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5)) + XCTAssertEqual( + app.staticTexts["status"].label, + "Intentional failure", + "This failure demonstrates native XCTest diagnostics" + ) + } + + private func launchApp() -> XCUIApplication { + let app = XCUIApplication() + let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local" + app.launchEnvironment["XCUITEST_RUN_ID"] = runID + app.launch() + return app + } } diff --git a/swift-xcuitest/README.md b/swift-xcuitest/README.md index be7ffb4..6e176e6 100644 --- a/swift-xcuitest/README.md +++ b/swift-xcuitest/README.md @@ -1,9 +1,16 @@ # Swift XCUITest example -This example runs passing UI tests against a minimal iOS app and uploads the -results with [`test-collector-swift`](https://github.com/buildkite/test-collector-swift). -It uses a shared scheme, a lexical test plan, and an explicit `.xcresult` -bundle so the same command works locally and on a macOS Buildkite agent. +This example reproduces the result loss that occurs when XCTest replaces an +XCUITest runner. Two tests finish before a third test terminates the runner. +Xcode continues the remaining tests in a replacement process and keeps all +five tests in the `.xcresult`, but `test-collector-swift` loses the completed +executions buffered by the terminated process. The final test also fails with +an execution tag, backtrace, and source location to exercise the richer native +collector metadata. + +The verification script compares the `.xcresult` test count with the unique +test names logged when the collector uploads. It intentionally fails while +the counts differ. The Buildkite step is therefore soft-failed. Run it with: diff --git a/swift-xcuitest/bin/test b/swift-xcuitest/bin/test index c33db23..180f2ee 100755 --- a/swift-xcuitest/bin/test +++ b/swift-xcuitest/bin/test @@ -5,6 +5,7 @@ set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" artifacts_dir="${XCUITEST_ARTIFACTS_DIR:-$root/artifacts}" result_bundle="$artifacts_dir/SwiftXCUITest.xcresult" +test_log="$artifacts_dir/xcodebuild.log" for name in \ BUILDKITE_ANALYTICS_BASE_URL \ @@ -34,6 +35,9 @@ do fi done +run_id="${XCUITEST_RUN_ID:-${BUILDKITE_BUILD_ID:-local}-${BUILDKITE_JOB_ID:-$(date +%s)}}" +export "TEST_RUNNER_XCUITEST_RUN_ID=$run_id" + destination="${XCUITEST_DESTINATION:-}" if [[ -z "$destination" ]]; then device_id="$({ @@ -57,6 +61,7 @@ xcodebuild \ -scheme SwiftXCUITest \ -resolvePackageDependencies +set +e xcodebuild \ -project "$root/SwiftXCUITest.xcodeproj" \ -scheme SwiftXCUITest \ @@ -64,4 +69,17 @@ xcodebuild \ -destination "$destination" \ -derivedDataPath "$artifacts_dir/DerivedData" \ -resultBundlePath "$result_bundle" \ - test + -retry-tests-on-failure \ + -test-iterations 2 \ + test 2>&1 | tee "$test_log" +xcodebuild_status="${PIPESTATUS[0]}" +set -e + +verification_status=0 +"$root/bin/verify-results" "$result_bundle" "$test_log" || verification_status="$?" + +if [[ "$xcodebuild_status" -ne 0 ]]; then + exit "$xcodebuild_status" +fi + +exit "$verification_status" diff --git a/swift-xcuitest/bin/verify-results b/swift-xcuitest/bin/verify-results new file mode 100755 index 0000000..39c29ba --- /dev/null +++ b/swift-xcuitest/bin/verify-results @@ -0,0 +1,52 @@ +#!/bin/bash + +set -euo pipefail + +result_bundle="${1:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" +test_log="${2:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" +expected_count=5 + +result_count="$({ + xcrun xcresulttool get test-results summary --path "$result_bundle" +} | plutil -extract totalTestCount raw -o - -)" +uploaded_count="$(awk ' + { + line = $0 + while (match(line, /name: Optional\("test[^"]*"\)/)) { + names[substr(line, RSTART, RLENGTH)] = 1 + line = substr(line, RSTART + RLENGTH) + } + } + END { + for (name in names) count++ + print count + 0 + } +' "$test_log")" + +echo "Expected unique tests: $expected_count" +echo ".xcresult unique tests: $result_count" +echo "Collector unique tests recorded for upload: $uploaded_count" + +status=0 + +if [[ "$result_count" -ne "$expected_count" ]]; then + echo "Expected .xcresult to retain all $expected_count tests" >&2 + status=1 +fi + +if grep -Fq 'location: Optional("ExampleAppUITests.swift:' "$test_log" \ + && grep -Fq 'failureExpanded: [Core.Trace.FailureExpanded' "$test_log" \ + && grep -Fq 'tags: Optional(["runner.replacement": "true"])' "$test_log" +then + echo "Collector retained the failure location, expanded backtrace, and execution tag" +else + echo "Collector did not log all expected native failure metadata" >&2 + status=1 +fi + +if [[ "$uploaded_count" -ne "$result_count" ]]; then + echo "Collector lost executions when the XCTest runner was replaced" >&2 + status=1 +fi + +exit "$status" From 02200cc4b2f7cfe9ed76f8dcb8406e7b7c147789 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 17:56:57 +0930 Subject: [PATCH 2/6] Exercise Swift collector OTLP delivery Run the XCTest example against the OTLP-only collector branch and use the bktec relay for hosted macOS XCUITest. Verify that every completed execution reaches the relay across XCTest runner replacement while documenting the unavoidable active-test SIGKILL gap. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- .buildkite/pipeline.yml | 13 +- .gitignore | 1 + swift-xctest/Package.resolved | 227 +++++++++++++++++- swift-xctest/Package.swift | 4 +- .../ExampleAppUITests/ExampleAppUITests.swift | 8 +- swift-xcuitest/README.md | 26 +- .../SwiftXCUITest.xcodeproj/project.pbxproj | 18 +- .../xcshareddata/swiftpm/Package.resolved | 226 ++++++++++++++++- swift-xcuitest/bin/test | 18 +- swift-xcuitest/bin/verify-results | 57 +++-- 10 files changed, 535 insertions(+), 63 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ed3e67e..4e06541 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -203,10 +203,11 @@ steps: steps: - label: ":swift: Swift: XCTest (test-collector-swift)" commands: + - git clone --branch swift-otel-executions --depth 1 https://github.com/buildkite/bktest .bktest - cd swift-xctest - swift package resolve # Run XCTest tests (which includes the BuildkiteTestCollector). - # The collector auto-loads via XCTest observation and uploads results + # The collector auto-loads via XCTest observation and exports results # when BUILDKITE_ANALYTICS_TOKEN is set. # Use --disable-swift-testing to only run XCTest (not Swift Testing). - swift test --disable-swift-testing @@ -229,22 +230,26 @@ steps: - BUILDKITE_ANALYTICS_DEBUG_ENABLED - label: ":swift: Swift: XCUITest (test-collector-swift)" - command: swift-xcuitest/bin/test + commands: + - git clone --branch swift-otel-executions --depth 1 https://github.com/buildkite/bktest .bktest + - bktec run agents: queue: macos artifact_paths: - swift-xcuitest/artifacts/SwiftXCUITest.xcresult/**/* - swift-xcuitest/artifacts/xcodebuild.log - soft_fail: true env: BUILDKITE_ANALYTICS_DEBUG_ENABLED: "true" BUILDKITE_ANALYTICS_TAGS: '{"test.framework.name":"xcuitest","language.name":"swift","custom.tag.from":"upload"}' + BUILDKITE_TESTS_OTLP_RELAY: "true" plugins: - tests#v1.0.0: test-runner: custom + test-cmd: swift-xcuitest/bin/test + test-file-pattern: swift-xcuitest/ExampleAppUITests/**/*.swift suite-slug: test-engine-client-examples upload-results: false - install-client: false + client-version: v3.1.0-rc.1 - group: ":android: Android" steps: diff --git a/.gitignore b/.gitignore index 24520cf..4f17ba4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules/ tmp/ bktec +.bktest/ # pytest *.egg* diff --git a/swift-xctest/Package.resolved b/swift-xctest/Package.resolved index 5aba5dd..ed12425 100644 --- a/swift-xctest/Package.resolved +++ b/swift-xctest/Package.resolved @@ -1,14 +1,231 @@ { + "originHash" : "8ada01ac22eb9fda6ae0e01cc963353741d89b9a0d1afbe96ca04a62c54e12eb", "pins" : [ { - "identity" : "test-collector-swift", + "identity" : "grpc-swift", "kind" : "remoteSourceControl", - "location" : "https://github.com/buildkite/test-collector-swift", + "location" : "https://github.com/grpc/grpc-swift.git", "state" : { - "revision" : "dbb0f13987c3f9f54611aaf0ac0dc1085ebb5ea6", - "version" : "0.6.0" + "revision" : "f857994e146f5146d702e9c31ac6f3c27d55d18a", + "version" : "1.27.0" + } + }, + { + "identity" : "opentelemetry-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/open-telemetry/opentelemetry-swift", + "state" : { + "revision" : "df57cc4fd0b4cf4f9d715253c6aaba4b6912f7e0", + "version" : "2.3.0" + } + }, + { + "identity" : "opentelemetry-swift-core", + "kind" : "remoteSourceControl", + "location" : "https://github.com/open-telemetry/opentelemetry-swift-core", + "state" : { + "revision" : "240c8d5e36c3c7b774ed961325369f0b1f2c965f", + "version" : "2.3.0" + } + }, + { + "identity" : "opentracing-objc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/undefinedlabs/opentracing-objc", + "state" : { + "revision" : "18c1a35ca966236cee0c5a714a51a73ff33384c1", + "version" : "0.5.2" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", + "version" : "1.1.5" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-certificates", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-certificates.git", + "state" : { + "revision" : "449dbbecd0f31e82b510ada227ca152caa8b5e98", + "version" : "1.19.4" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", + "version" : "4.5.1" + } + }, + { + "identity" : "swift-http-structured-headers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-structured-headers.git", + "state" : { + "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", + "version" : "1.7.0" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types.git", + "state" : { + "revision" : "db774a277f60063a32d854f2980299caf06da041", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", + "version" : "1.15.0" + } + }, + { + "identity" : "swift-metrics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-metrics.git", + "state" : { + "revision" : "f17c111cec972c2a4922cef38cf64f76f7e87886", + "version" : "2.8.0" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", + "version" : "2.101.3" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras.git", + "state" : { + "revision" : "88a51340f59cf181ebde888bd1b749296b3ec029", + "version" : "1.34.3" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "45bdf670248be5f16ec0340e125dca285536f0fb", + "version" : "1.45.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "d930168b86f46ca51a4bc09c5ca45c1833db8067", + "version" : "2.37.2" + } + }, + { + "identity" : "swift-nio-transport-services", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-transport-services.git", + "state" : { + "revision" : "67787bb645a5e67d2edcdfbe48a216cc549222d5", + "version" : "1.28.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "55d7a1cc5666b85c13464aea1c4b4a90feccb4c8", + "version" : "1.38.1" + } + }, + { + "identity" : "swift-service-lifecycle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-service-lifecycle.git", + "state" : { + "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", + "version" : "2.12.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", + "version" : "1.8.1" + } + }, + { + "identity" : "thrift-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/undefinedlabs/Thrift-Swift", + "state" : { + "revision" : "18ff09e6b30e589ed38f90a1af23e193b8ecef8e", + "version" : "1.1.2" } } ], - "version" : 2 + "version" : 3 } diff --git a/swift-xctest/Package.swift b/swift-xctest/Package.swift index bd3cf9c..0e0aa27 100644 --- a/swift-xctest/Package.swift +++ b/swift-xctest/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.9 +// swift-tools-version: 6.0 import PackageDescription let package = Package( @@ -10,7 +10,7 @@ let package = Package( .library(name: "ExampleLib", targets: ["ExampleLib"]) ], dependencies: [ - .package(url: "https://github.com/buildkite/test-collector-swift", from: "0.6.0") + .package(path: "../.bktest/test-collector-swift") ], targets: [ .target( diff --git a/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift b/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift index 68285d6..270732a 100644 --- a/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift +++ b/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift @@ -41,16 +41,12 @@ final class ExampleAppUITests: XCTestCase { XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 5)) } - func test05FailureIncludesDiagnostics() { + func test05ExecutionTagAfterRestart() { tagExecution("runner.replacement", "true") let app = launchApp() XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5)) - XCTAssertEqual( - app.staticTexts["status"].label, - "Intentional failure", - "This failure demonstrates native XCTest diagnostics" - ) + XCTAssertEqual(app.staticTexts["status"].label, "Swift XCUITest example") } private func launchApp() -> XCUIApplication { diff --git a/swift-xcuitest/README.md b/swift-xcuitest/README.md index 6e176e6..a714882 100644 --- a/swift-xcuitest/README.md +++ b/swift-xcuitest/README.md @@ -1,16 +1,26 @@ # Swift XCUITest example -This example reproduces the result loss that occurs when XCTest replaces an +This example verifies OpenTelemetry execution delivery when XCTest replaces an XCUITest runner. Two tests finish before a third test terminates the runner. Xcode continues the remaining tests in a replacement process and keeps all -five tests in the `.xcresult`, but `test-collector-swift` loses the completed -executions buffered by the terminated process. The final test also fails with -an execution tag, backtrace, and source location to exercise the richer native -collector metadata. +five tests in the `.xcresult`: four passes and the deliberately killed active +test. Each of the four tests that receives an XCTest completion callback is +handed to the bktec OTLP relay before its runner can exit. -The verification script compares the `.xcresult` test count with the unique -test names logged when the collector uploads. It intentionally fails while -the counts differ. The Buildkite step is therefore soft-failed. +The verification script asserts the expected runner crash and replacement, +then checks that all four completed tests were accepted over OTLP. The killed +test cannot be emitted by an in-process collector because `SIGKILL` prevents +XCTest's completion callback; recovering that execution would require durable +start-state reconciliation outside the runner. + +This branch uses the Swift collector from the `swift-otel-executions` branch of +[`buildkite/bktest`](https://github.com/buildkite/bktest). Check it out before +running locally: + +```sh +git clone --branch swift-otel-executions --depth 1 \ + https://github.com/buildkite/bktest ../.bktest +``` Run it with: diff --git a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.pbxproj b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.pbxproj index 65a3cd5..8abff26 100644 --- a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.pbxproj +++ b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.pbxproj @@ -145,7 +145,7 @@ mainGroup = 15771E3CC944F418353D1657; minimizedProjectReferenceProxies = 0; packageReferences = ( - 0F260402842AAB649211B9B2 /* XCRemoteSwiftPackageReference "test-collector-swift" */, + 0F260402842AAB649211B9B2 /* XCLocalSwiftPackageReference "../.bktest/test-collector-swift" */, ); preferredProjectObjectVersion = 100; productRefGroup = 55AE842CDC491C8FB4A68112 /* Products */; @@ -415,21 +415,17 @@ }; /* End XCConfigurationList section */ -/* Begin XCRemoteSwiftPackageReference section */ - 0F260402842AAB649211B9B2 /* XCRemoteSwiftPackageReference "test-collector-swift" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/buildkite/test-collector-swift"; - requirement = { - kind = upToNextMajorVersion; - minimumVersion = 0.6.0; - }; +/* Begin XCLocalSwiftPackageReference section */ + 0F260402842AAB649211B9B2 /* XCLocalSwiftPackageReference "../.bktest/test-collector-swift" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = "../.bktest/test-collector-swift"; }; -/* End XCRemoteSwiftPackageReference section */ +/* End XCLocalSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ DB07F6513929E915435499BA /* BuildkiteTestCollector */ = { isa = XCSwiftPackageProductDependency; - package = 0F260402842AAB649211B9B2 /* XCRemoteSwiftPackageReference "test-collector-swift" */; + package = 0F260402842AAB649211B9B2 /* XCLocalSwiftPackageReference "../.bktest/test-collector-swift" */; productName = BuildkiteTestCollector; }; /* End XCSwiftPackageProductDependency section */ diff --git a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index cef9935..779a1c1 100644 --- a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,13 +1,229 @@ { - "originHash" : "f28a54e288718485642adbe7b780a05abf05f95f0f435b3f87b6d3684b95d268", + "originHash" : "70638e11aba3a1472dfd0d75666b7caef58756d280301136e8d429ecf34ce1b3", "pins" : [ { - "identity" : "test-collector-swift", + "identity" : "grpc-swift", "kind" : "remoteSourceControl", - "location" : "https://github.com/buildkite/test-collector-swift", + "location" : "https://github.com/grpc/grpc-swift.git", "state" : { - "revision" : "dbb0f13987c3f9f54611aaf0ac0dc1085ebb5ea6", - "version" : "0.6.0" + "revision" : "f857994e146f5146d702e9c31ac6f3c27d55d18a", + "version" : "1.27.0" + } + }, + { + "identity" : "opentelemetry-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/open-telemetry/opentelemetry-swift", + "state" : { + "revision" : "df57cc4fd0b4cf4f9d715253c6aaba4b6912f7e0", + "version" : "2.3.0" + } + }, + { + "identity" : "opentelemetry-swift-core", + "kind" : "remoteSourceControl", + "location" : "https://github.com/open-telemetry/opentelemetry-swift-core", + "state" : { + "revision" : "240c8d5e36c3c7b774ed961325369f0b1f2c965f", + "version" : "2.3.0" + } + }, + { + "identity" : "opentracing-objc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/undefinedlabs/opentracing-objc", + "state" : { + "revision" : "18c1a35ca966236cee0c5a714a51a73ff33384c1", + "version" : "0.5.2" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", + "version" : "1.1.5" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-certificates", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-certificates.git", + "state" : { + "revision" : "449dbbecd0f31e82b510ada227ca152caa8b5e98", + "version" : "1.19.4" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", + "version" : "4.5.1" + } + }, + { + "identity" : "swift-http-structured-headers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-structured-headers.git", + "state" : { + "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", + "version" : "1.7.0" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types.git", + "state" : { + "revision" : "db774a277f60063a32d854f2980299caf06da041", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", + "version" : "1.15.0" + } + }, + { + "identity" : "swift-metrics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-metrics.git", + "state" : { + "revision" : "f17c111cec972c2a4922cef38cf64f76f7e87886", + "version" : "2.8.0" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", + "version" : "2.101.3" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras.git", + "state" : { + "revision" : "88a51340f59cf181ebde888bd1b749296b3ec029", + "version" : "1.34.3" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "45bdf670248be5f16ec0340e125dca285536f0fb", + "version" : "1.45.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "d930168b86f46ca51a4bc09c5ca45c1833db8067", + "version" : "2.37.2" + } + }, + { + "identity" : "swift-nio-transport-services", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-transport-services.git", + "state" : { + "revision" : "67787bb645a5e67d2edcdfbe48a216cc549222d5", + "version" : "1.28.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "55d7a1cc5666b85c13464aea1c4b4a90feccb4c8", + "version" : "1.38.1" + } + }, + { + "identity" : "swift-service-lifecycle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-service-lifecycle.git", + "state" : { + "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", + "version" : "2.12.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", + "version" : "1.8.1" + } + }, + { + "identity" : "thrift-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/undefinedlabs/Thrift-Swift", + "state" : { + "revision" : "18ff09e6b30e589ed38f90a1af23e193b8ecef8e", + "version" : "1.1.2" } } ], diff --git a/swift-xcuitest/bin/test b/swift-xcuitest/bin/test index 180f2ee..fcb4bcb 100755 --- a/swift-xcuitest/bin/test +++ b/swift-xcuitest/bin/test @@ -8,7 +8,7 @@ result_bundle="$artifacts_dir/SwiftXCUITest.xcresult" test_log="$artifacts_dir/xcodebuild.log" for name in \ - BUILDKITE_ANALYTICS_BASE_URL \ + BUILDKITE_ANALYTICS_KEY \ BUILDKITE_ANALYTICS_BRANCH \ BUILDKITE_ANALYTICS_DEBUG_ENABLED \ BUILDKITE_ANALYTICS_ENABLED \ @@ -18,6 +18,7 @@ for name in \ BUILDKITE_ANALYTICS_JOB_ID \ BUILDKITE_ANALYTICS_MESSAGE \ BUILDKITE_ANALYTICS_NUMBER \ + BUILDKITE_ANALYTICS_OTLP_ENDPOINT \ BUILDKITE_ANALYTICS_SHA \ BUILDKITE_ANALYTICS_TAGS \ BUILDKITE_ANALYTICS_TOKEN \ @@ -28,7 +29,18 @@ for name in \ BUILDKITE_BUILD_URL \ BUILDKITE_COMMIT \ BUILDKITE_JOB_ID \ - BUILDKITE_MESSAGE + BUILDKITE_MESSAGE \ + BUILDKITE_ORGANIZATION_SLUG \ + BUILDKITE_STEP_ID \ + BUILDKITE_TAG \ + BUILDKITE_TEST_ENGINE_SUITE_SLUG \ + BUILDKITE_TESTS_OTLP_TOKEN \ + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT \ + OTEL_EXPORTER_OTLP_TRACES_HEADERS \ + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL \ + OTEL_RESOURCE_ATTRIBUTES \ + TRACEPARENT \ + TRACESTATE do if [[ -n "${!name:-}" ]]; then export "TEST_RUNNER_${name}=${!name}" @@ -79,7 +91,7 @@ verification_status=0 "$root/bin/verify-results" "$result_bundle" "$test_log" || verification_status="$?" if [[ "$xcodebuild_status" -ne 0 ]]; then - exit "$xcodebuild_status" + echo "xcodebuild exited $xcodebuild_status after the expected runner termination" fi exit "$verification_status" diff --git a/swift-xcuitest/bin/verify-results b/swift-xcuitest/bin/verify-results index 39c29ba..7e0af28 100755 --- a/swift-xcuitest/bin/verify-results +++ b/swift-xcuitest/bin/verify-results @@ -4,15 +4,19 @@ set -euo pipefail result_bundle="${1:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" test_log="${2:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" -expected_count=5 +expected_result_count=5 +expected_completed_count=4 -result_count="$({ - xcrun xcresulttool get test-results summary --path "$result_bundle" -} | plutil -extract totalTestCount raw -o - -)" -uploaded_count="$(awk ' +summary="$(xcrun xcresulttool get test-results summary --path "$result_bundle")" +result_count="$(plutil -extract totalTestCount raw -o - - <<< "$summary")" +passed_count="$(plutil -extract passedTests raw -o - - <<< "$summary")" +failed_count="$(plutil -extract failedTests raw -o - - <<< "$summary")" +failure_test="$(plutil -extract testFailures.0.testIdentifierString raw -o - - <<< "$summary")" +failure_text="$(plutil -extract testFailures.0.failureText raw -o - - <<< "$summary")" +exported_count="$(awk ' { line = $0 - while (match(line, /name: Optional\("test[^"]*"\)/)) { + while (match(line, /Exported OpenTelemetry test execution: test[^[:space:]]*/)) { names[substr(line, RSTART, RLENGTH)] = 1 line = substr(line, RSTART + RLENGTH) } @@ -23,29 +27,44 @@ uploaded_count="$(awk ' } ' "$test_log")" -echo "Expected unique tests: $expected_count" -echo ".xcresult unique tests: $result_count" -echo "Collector unique tests recorded for upload: $uploaded_count" +echo "Expected .xcresult tests: $expected_result_count" +echo ".xcresult tests: $result_count ($passed_count passed, $failed_count runner crash)" +echo "Expected completed executions: $expected_completed_count" +echo "Collector unique executions accepted over OTLP: $exported_count" status=0 -if [[ "$result_count" -ne "$expected_count" ]]; then - echo "Expected .xcresult to retain all $expected_count tests" >&2 +if [[ "$result_count" -ne "$expected_result_count" || "$passed_count" -ne 4 || "$failed_count" -ne 1 ]]; then + echo "Expected .xcresult to retain four passes and the deliberate runner crash" >&2 status=1 fi -if grep -Fq 'location: Optional("ExampleAppUITests.swift:' "$test_log" \ - && grep -Fq 'failureExpanded: [Core.Trace.FailureExpanded' "$test_log" \ - && grep -Fq 'tags: Optional(["runner.replacement": "true"])' "$test_log" +if [[ "$failure_test" != "ExampleAppUITests/test03RunnerCrashesOnce()" \ + || "$failure_text" != "Test crashed with signal kill." ]] then - echo "Collector retained the failure location, expanded backtrace, and execution tag" -else - echo "Collector did not log all expected native failure metadata" >&2 + echo "Expected test03RunnerCrashesOnce to be the only failed test" >&2 status=1 fi -if [[ "$uploaded_count" -ne "$result_count" ]]; then - echo "Collector lost executions when the XCTest runner was replaced" >&2 +if ! grep -Fq "Restarting after unexpected exit, crash, or test timeout" "$test_log"; then + echo "XCTest did not replace the terminated runner" >&2 + status=1 +fi + +for name in \ + test01AppLaunches \ + test02ExecutionTag \ + test04RunsInReplacementRunner \ + test05ExecutionTagAfterRestart +do + if ! grep -Fq "Exported OpenTelemetry test execution: $name" "$test_log"; then + echo "Collector did not export completed execution $name" >&2 + status=1 + fi +done + +if [[ "$exported_count" -ne "$expected_completed_count" ]]; then + echo "Collector did not hand every completed execution to the OTLP endpoint" >&2 status=1 fi From f3cac40683f52f034fe431a8f3784e4d4915c8cf Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:00:11 +0930 Subject: [PATCH 3/6] Fix bktec release version format Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4e06541..47bb80b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -249,7 +249,7 @@ steps: test-file-pattern: swift-xcuitest/ExampleAppUITests/**/*.swift suite-slug: test-engine-client-examples upload-results: false - client-version: v3.1.0-rc.1 + client-version: 3.1.0-rc.1 - group: ":android: Android" steps: From 2d9bf6bbc95efb771e1d28048e422bda3af20bad Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:01:13 +0930 Subject: [PATCH 4/6] Lock Swift 6.0-compatible OTLP dependencies Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- swift-xctest/Package.resolved | 12 ++++++------ .../xcshareddata/swiftpm/Package.resolved | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/swift-xctest/Package.resolved b/swift-xctest/Package.resolved index ed12425..7063566 100644 --- a/swift-xctest/Package.resolved +++ b/swift-xctest/Package.resolved @@ -6,8 +6,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/grpc/grpc-swift.git", "state" : { - "revision" : "f857994e146f5146d702e9c31ac6f3c27d55d18a", - "version" : "1.27.0" + "revision" : "a56a157218877ef3e9625f7e1f7b2cb7e46ead1b", + "version" : "1.26.1" } }, { @@ -15,8 +15,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/open-telemetry/opentelemetry-swift", "state" : { - "revision" : "df57cc4fd0b4cf4f9d715253c6aaba4b6912f7e0", - "version" : "2.3.0" + "revision" : "0b77b8e2b2121261562b1d304c638a004e1d2f19", + "version" : "2.2.0" } }, { @@ -24,8 +24,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/open-telemetry/opentelemetry-swift-core", "state" : { - "revision" : "240c8d5e36c3c7b774ed961325369f0b1f2c965f", - "version" : "2.3.0" + "revision" : "fd787757decabfa93319cc3c04f03a49e0cf40b6", + "version" : "2.2.0" } }, { diff --git a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 779a1c1..c529f53 100644 --- a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -6,8 +6,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/grpc/grpc-swift.git", "state" : { - "revision" : "f857994e146f5146d702e9c31ac6f3c27d55d18a", - "version" : "1.27.0" + "revision" : "a56a157218877ef3e9625f7e1f7b2cb7e46ead1b", + "version" : "1.26.1" } }, { @@ -15,8 +15,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/open-telemetry/opentelemetry-swift", "state" : { - "revision" : "df57cc4fd0b4cf4f9d715253c6aaba4b6912f7e0", - "version" : "2.3.0" + "revision" : "0b77b8e2b2121261562b1d304c638a004e1d2f19", + "version" : "2.2.0" } }, { @@ -24,8 +24,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/open-telemetry/opentelemetry-swift-core", "state" : { - "revision" : "240c8d5e36c3c7b774ed961325369f0b1f2c965f", - "version" : "2.3.0" + "revision" : "fd787757decabfa93319cc3c04f03a49e0cf40b6", + "version" : "2.2.0" } }, { From 543ef4b987509aadd0a4d91fdab1053073dea1a6 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:07:39 +0930 Subject: [PATCH 5/6] Use prerelease-aware relay plugin Upgrade the Tests plugin so it can install the bktec relay release candidate, and resolve both Swift examples with Swift 6.0-compatible transitive versions. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- .buildkite/pipeline.yml | 4 +- swift-xctest/Package.resolved | 76 +++++++------------ .../xcshareddata/swiftpm/Package.resolved | 56 +++++++------- 3 files changed, 59 insertions(+), 77 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 47bb80b..880a4c5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -243,13 +243,13 @@ steps: BUILDKITE_ANALYTICS_TAGS: '{"test.framework.name":"xcuitest","language.name":"swift","custom.tag.from":"upload"}' BUILDKITE_TESTS_OTLP_RELAY: "true" plugins: - - tests#v1.0.0: + - tests#v1.0.1: test-runner: custom test-cmd: swift-xcuitest/bin/test test-file-pattern: swift-xcuitest/ExampleAppUITests/**/*.swift suite-slug: test-engine-client-examples upload-results: false - client-version: 3.1.0-rc.1 + client-version: v3.1.0-rc.1 - group: ":android: Android" steps: diff --git a/swift-xctest/Package.resolved b/swift-xctest/Package.resolved index 7063566..71e244a 100644 --- a/swift-xctest/Package.resolved +++ b/swift-xctest/Package.resolved @@ -28,15 +28,6 @@ "version" : "2.2.0" } }, - { - "identity" : "opentracing-objc", - "kind" : "remoteSourceControl", - "location" : "https://github.com/undefinedlabs/opentracing-objc", - "state" : { - "revision" : "18c1a35ca966236cee0c5a714a51a73ff33384c1", - "version" : "0.5.2" - } - }, { "identity" : "swift-algorithms", "kind" : "remoteSourceControl", @@ -51,8 +42,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-asn1.git", "state" : { - "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", - "version" : "1.7.1" + "revision" : "9f542610331815e29cc3821d3b6f488db8715517", + "version" : "1.6.0" } }, { @@ -78,8 +69,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "449dbbecd0f31e82b510ada227ca152caa8b5e98", - "version" : "1.19.4" + "revision" : "24ccdeeeed4dfaae7955fcac9dbf5489ed4f1a25", + "version" : "1.18.0" } }, { @@ -96,8 +87,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", - "version" : "4.5.1" + "revision" : "bb4ba815dab96d4edc1e0b86d7b9acf9ff973a84", + "version" : "4.3.1" } }, { @@ -105,8 +96,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-structured-headers.git", "state" : { - "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", - "version" : "1.7.0" + "revision" : "76d7627bd88b47bf5a0f8497dd244885960dde0b", + "version" : "1.6.0" } }, { @@ -114,8 +105,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-types.git", "state" : { - "revision" : "db774a277f60063a32d854f2980299caf06da041", - "version" : "1.6.0" + "revision" : "45eb0224913ea070ec4fba17291b9e7ecf4749ca", + "version" : "1.5.1" } }, { @@ -123,14 +114,14 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", - "version" : "1.15.0" + "revision" : "8c0f217f01000dd30f60d6e536569ad4e74291f9", + "version" : "1.11.0" } }, { "identity" : "swift-metrics", "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-metrics.git", + "location" : "https://github.com/apple/swift-metrics", "state" : { "revision" : "f17c111cec972c2a4922cef38cf64f76f7e87886", "version" : "2.8.0" @@ -141,8 +132,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", - "version" : "2.101.3" + "revision" : "558f24a4647193b5a0e2104031b71c55d31ff83a", + "version" : "2.97.1" } }, { @@ -150,8 +141,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "88a51340f59cf181ebde888bd1b749296b3ec029", - "version" : "1.34.3" + "revision" : "abcf5312eb8ed2fb11916078aef7c46b06f20813", + "version" : "1.33.0" } }, { @@ -159,8 +150,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-http2.git", "state" : { - "revision" : "45bdf670248be5f16ec0340e125dca285536f0fb", - "version" : "1.45.0" + "revision" : "6d8d596f0a9bfebb925733003731fe2d749b7e02", + "version" : "1.42.0" } }, { @@ -168,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "d930168b86f46ca51a4bc09c5ca45c1833db8067", - "version" : "2.37.2" + "revision" : "df9c3406028e3297246e6e7081977a167318b692", + "version" : "2.36.1" } }, { @@ -177,8 +168,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "67787bb645a5e67d2edcdfbe48a216cc549222d5", - "version" : "1.28.0" + "revision" : "60c3e187154421171721c1a38e800b390680fb5d", + "version" : "1.26.0" } }, { @@ -195,8 +186,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-protobuf.git", "state" : { - "revision" : "55d7a1cc5666b85c13464aea1c4b4a90feccb4c8", - "version" : "1.38.1" + "revision" : "81558271e243f8f47dfe8e9fdd55f3c2b5413f68", + "version" : "1.37.0" } }, { @@ -204,8 +195,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/swift-service-lifecycle.git", "state" : { - "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", - "version" : "2.12.0" + "revision" : "9829955b385e5bb88128b73f1b8389e9b9c3191a", + "version" : "2.11.0" } }, { @@ -213,17 +204,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-system.git", "state" : { - "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", - "version" : "1.8.1" - } - }, - { - "identity" : "thrift-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/undefinedlabs/Thrift-Swift", - "state" : { - "revision" : "18ff09e6b30e589ed38f90a1af23e193b8ecef8e", - "version" : "1.1.2" + "revision" : "0b30161977799fd949f7f6848586b82ff6764f73", + "version" : "1.6.6" } } ], diff --git a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index c529f53..a5e1821 100644 --- a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -51,8 +51,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-asn1.git", "state" : { - "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", - "version" : "1.7.1" + "revision" : "9f542610331815e29cc3821d3b6f488db8715517", + "version" : "1.6.0" } }, { @@ -78,8 +78,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "449dbbecd0f31e82b510ada227ca152caa8b5e98", - "version" : "1.19.4" + "revision" : "24ccdeeeed4dfaae7955fcac9dbf5489ed4f1a25", + "version" : "1.18.0" } }, { @@ -96,8 +96,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", - "version" : "4.5.1" + "revision" : "bb4ba815dab96d4edc1e0b86d7b9acf9ff973a84", + "version" : "4.3.1" } }, { @@ -105,8 +105,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-structured-headers.git", "state" : { - "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", - "version" : "1.7.0" + "revision" : "76d7627bd88b47bf5a0f8497dd244885960dde0b", + "version" : "1.6.0" } }, { @@ -114,8 +114,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-types.git", "state" : { - "revision" : "db774a277f60063a32d854f2980299caf06da041", - "version" : "1.6.0" + "revision" : "45eb0224913ea070ec4fba17291b9e7ecf4749ca", + "version" : "1.5.1" } }, { @@ -123,8 +123,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", - "version" : "1.15.0" + "revision" : "8c0f217f01000dd30f60d6e536569ad4e74291f9", + "version" : "1.11.0" } }, { @@ -141,8 +141,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", - "version" : "2.101.3" + "revision" : "558f24a4647193b5a0e2104031b71c55d31ff83a", + "version" : "2.97.1" } }, { @@ -150,8 +150,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "88a51340f59cf181ebde888bd1b749296b3ec029", - "version" : "1.34.3" + "revision" : "abcf5312eb8ed2fb11916078aef7c46b06f20813", + "version" : "1.33.0" } }, { @@ -159,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-http2.git", "state" : { - "revision" : "45bdf670248be5f16ec0340e125dca285536f0fb", - "version" : "1.45.0" + "revision" : "6d8d596f0a9bfebb925733003731fe2d749b7e02", + "version" : "1.42.0" } }, { @@ -168,8 +168,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "d930168b86f46ca51a4bc09c5ca45c1833db8067", - "version" : "2.37.2" + "revision" : "df9c3406028e3297246e6e7081977a167318b692", + "version" : "2.36.1" } }, { @@ -177,8 +177,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "67787bb645a5e67d2edcdfbe48a216cc549222d5", - "version" : "1.28.0" + "revision" : "60c3e187154421171721c1a38e800b390680fb5d", + "version" : "1.26.0" } }, { @@ -195,8 +195,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-protobuf.git", "state" : { - "revision" : "55d7a1cc5666b85c13464aea1c4b4a90feccb4c8", - "version" : "1.38.1" + "revision" : "81558271e243f8f47dfe8e9fdd55f3c2b5413f68", + "version" : "1.37.0" } }, { @@ -204,8 +204,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/swift-service-lifecycle.git", "state" : { - "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", - "version" : "2.12.0" + "revision" : "9829955b385e5bb88128b73f1b8389e9b9c3191a", + "version" : "2.11.0" } }, { @@ -213,8 +213,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-system.git", "state" : { - "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", - "version" : "1.8.1" + "revision" : "0b30161977799fd949f7f6848586b82ff6764f73", + "version" : "1.6.6" } }, { From bf33bbcd3820e68addb527703df25af2cacfc230 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:42:05 +0930 Subject: [PATCH 6/6] Support hosted Xcode 15 Swift examples Lower the Linux example manifest to Swift tools 5.10 and pin the transitive collector graph to versions that compile with Swift 5.10/Xcode 15.4. Teach the XCUITest result verifier to read both the modern test-results summary and the legacy XCResult action schema. This keeps the hosted macOS check strict about all five records, including the deliberately killed runner, without requiring a newer xcresulttool. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- swift-xctest/Package.resolved | 58 +++++++++---------- swift-xctest/Package.swift | 2 +- .../xcshareddata/swiftpm/Package.resolved | 4 +- swift-xcuitest/bin/verify-results | 25 ++++++-- 4 files changed, 51 insertions(+), 38 deletions(-) diff --git a/swift-xctest/Package.resolved b/swift-xctest/Package.resolved index 71e244a..0112e6d 100644 --- a/swift-xctest/Package.resolved +++ b/swift-xctest/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "8ada01ac22eb9fda6ae0e01cc963353741d89b9a0d1afbe96ca04a62c54e12eb", + "originHash" : "160cfae0e90ff235481ca5f06cd3ead4f7d578a3db9d0f43104fc3e07a08a105", "pins" : [ { "identity" : "grpc-swift", @@ -42,8 +42,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-asn1.git", "state" : { - "revision" : "9f542610331815e29cc3821d3b6f488db8715517", - "version" : "1.6.0" + "revision" : "f70225981241859eb4aa1a18a75531d26637c8cc", + "version" : "1.4.0" } }, { @@ -69,8 +69,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "24ccdeeeed4dfaae7955fcac9dbf5489ed4f1a25", - "version" : "1.18.0" + "revision" : "c399f90e7bbe8874f6cbfda1d5f9023d1f5ce122", + "version" : "1.15.1" } }, { @@ -78,8 +78,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", - "version" : "1.6.0" + "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", + "version" : "1.2.1" } }, { @@ -87,8 +87,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "bb4ba815dab96d4edc1e0b86d7b9acf9ff973a84", - "version" : "4.3.1" + "revision" : "95ba0316a9b733e92bb6b071255ff46263bbe7dc", + "version" : "3.15.1" } }, { @@ -96,8 +96,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-structured-headers.git", "state" : { - "revision" : "76d7627bd88b47bf5a0f8497dd244885960dde0b", - "version" : "1.6.0" + "revision" : "1625f271afb04375bf48737a5572613248d0e7a0", + "version" : "1.4.0" } }, { @@ -114,8 +114,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "8c0f217f01000dd30f60d6e536569ad4e74291f9", - "version" : "1.11.0" + "revision" : "ce592ae52f982c847a4efc0dd881cc9eb32d29f2", + "version" : "1.6.4" } }, { @@ -123,8 +123,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-metrics", "state" : { - "revision" : "f17c111cec972c2a4922cef38cf64f76f7e87886", - "version" : "2.8.0" + "revision" : "0743a9364382629da3bf5677b46a2c4b1ce5d2a6", + "version" : "2.7.1" } }, { @@ -132,8 +132,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "558f24a4647193b5a0e2104031b71c55d31ff83a", - "version" : "2.97.1" + "revision" : "a18bddb0acf7a40d982b2f128ce73ce4ee31f352", + "version" : "2.86.2" } }, { @@ -141,8 +141,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "abcf5312eb8ed2fb11916078aef7c46b06f20813", - "version" : "1.33.0" + "revision" : "a55c3dd3a81d035af8a20ce5718889c0dcab073d", + "version" : "1.29.0" } }, { @@ -150,8 +150,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-http2.git", "state" : { - "revision" : "6d8d596f0a9bfebb925733003731fe2d749b7e02", - "version" : "1.42.0" + "revision" : "5e9e99ec96c53bc2c18ddd10c1e25a3cd97c55e5", + "version" : "1.38.0" } }, { @@ -159,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "df9c3406028e3297246e6e7081977a167318b692", - "version" : "2.36.1" + "revision" : "b2b043a8810ab6d51b3ff4df17f057d87ef1ec7c", + "version" : "2.34.1" } }, { @@ -168,8 +168,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "60c3e187154421171721c1a38e800b390680fb5d", - "version" : "1.26.0" + "revision" : "df6c28355051c72c884574a6c858bc54f7311ff9", + "version" : "1.25.2" } }, { @@ -186,8 +186,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-protobuf.git", "state" : { - "revision" : "81558271e243f8f47dfe8e9fdd55f3c2b5413f68", - "version" : "1.37.0" + "revision" : "c5ab62237f21cad094812719a1bbe29443407c5f", + "version" : "1.34.1" } }, { @@ -195,8 +195,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/swift-service-lifecycle.git", "state" : { - "revision" : "9829955b385e5bb88128b73f1b8389e9b9c3191a", - "version" : "2.11.0" + "revision" : "1de37290c0ab3c5a96028e0f02911b672fd42348", + "version" : "2.9.1" } }, { diff --git a/swift-xctest/Package.swift b/swift-xctest/Package.swift index 0e0aa27..4e5b04e 100644 --- a/swift-xctest/Package.swift +++ b/swift-xctest/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.0 +// swift-tools-version: 5.10 import PackageDescription let package = Package( diff --git a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index a5e1821..7cb84d3 100644 --- a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -132,8 +132,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-metrics.git", "state" : { - "revision" : "f17c111cec972c2a4922cef38cf64f76f7e87886", - "version" : "2.8.0" + "revision" : "0743a9364382629da3bf5677b46a2c4b1ce5d2a6", + "version" : "2.7.1" } }, { diff --git a/swift-xcuitest/bin/verify-results b/swift-xcuitest/bin/verify-results index 7e0af28..c3f93cf 100755 --- a/swift-xcuitest/bin/verify-results +++ b/swift-xcuitest/bin/verify-results @@ -7,12 +7,24 @@ test_log="${2:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" expected_result_count=5 expected_completed_count=4 -summary="$(xcrun xcresulttool get test-results summary --path "$result_bundle")" -result_count="$(plutil -extract totalTestCount raw -o - - <<< "$summary")" -passed_count="$(plutil -extract passedTests raw -o - - <<< "$summary")" -failed_count="$(plutil -extract failedTests raw -o - - <<< "$summary")" -failure_test="$(plutil -extract testFailures.0.testIdentifierString raw -o - - <<< "$summary")" -failure_text="$(plutil -extract testFailures.0.failureText raw -o - - <<< "$summary")" +if summary="$(xcrun xcresulttool get test-results summary --path "$result_bundle" 2>/dev/null)"; then + result_count="$(plutil -extract totalTestCount raw -o - - <<< "$summary")" + passed_count="$(plutil -extract passedTests raw -o - - <<< "$summary")" + failed_count="$(plutil -extract failedTests raw -o - - <<< "$summary")" + failure_test="$(plutil -extract testFailures.0.testIdentifierString raw -o - - <<< "$summary")" + failure_text="$(plutil -extract testFailures.0.failureText raw -o - - <<< "$summary")" +else + if ! summary="$(xcrun xcresulttool get object --legacy --path "$result_bundle" --format json 2>/dev/null)"; then + summary="$(xcrun xcresulttool get --path "$result_bundle" --format json)" + fi + + action_result="actions._values.0.actionResult" + result_count="$(plutil -extract "$action_result.metrics.testsCount._value" raw -o - - <<< "$summary")" + failed_count="$(plutil -extract "$action_result.metrics.testsFailedCount._value" raw -o - - <<< "$summary")" + passed_count="$((result_count - failed_count))" + failure_test="$(plutil -extract "$action_result.issues.testFailureSummaries._values.0.testCaseName._value" raw -o - - <<< "$summary")" + failure_text="$(plutil -extract "$action_result.issues.testFailureSummaries._values.0.message._value" raw -o - - <<< "$summary")" +fi exported_count="$(awk ' { line = $0 @@ -40,6 +52,7 @@ if [[ "$result_count" -ne "$expected_result_count" || "$passed_count" -ne 4 || " fi if [[ "$failure_test" != "ExampleAppUITests/test03RunnerCrashesOnce()" \ + && "$failure_test" != "ExampleAppUITests.test03RunnerCrashesOnce()" \ || "$failure_text" != "Test crashed with signal kill." ]] then echo "Expected test03RunnerCrashesOnce to be the only failed test" >&2