diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 15c9d94..880a4c5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -203,10 +203,11 @@ steps: steps: - label: ":swift: Swift: XCTest (test-collector-swift)" commands: + - git clone --branch swift-otel-executions --depth 1 https://github.com/buildkite/bktest .bktest - cd swift-xctest - swift package resolve # Run XCTest tests (which includes the BuildkiteTestCollector). - # The collector auto-loads via XCTest observation and uploads results + # The collector auto-loads via XCTest observation and exports results # when BUILDKITE_ANALYTICS_TOKEN is set. # Use --disable-swift-testing to only run XCTest (not Swift Testing). - swift test --disable-swift-testing @@ -229,19 +230,26 @@ steps: - BUILDKITE_ANALYTICS_DEBUG_ENABLED - label: ":swift: Swift: XCUITest (test-collector-swift)" - command: swift-xcuitest/bin/test + commands: + - git clone --branch swift-otel-executions --depth 1 https://github.com/buildkite/bktest .bktest + - bktec run agents: queue: macos artifact_paths: - swift-xcuitest/artifacts/SwiftXCUITest.xcresult/**/* + - swift-xcuitest/artifacts/xcodebuild.log env: + BUILDKITE_ANALYTICS_DEBUG_ENABLED: "true" BUILDKITE_ANALYTICS_TAGS: '{"test.framework.name":"xcuitest","language.name":"swift","custom.tag.from":"upload"}' + BUILDKITE_TESTS_OTLP_RELAY: "true" plugins: - - tests#v1.0.0: + - tests#v1.0.1: test-runner: custom + test-cmd: swift-xcuitest/bin/test + test-file-pattern: swift-xcuitest/ExampleAppUITests/**/*.swift suite-slug: test-engine-client-examples upload-results: false - install-client: false + client-version: v3.1.0-rc.1 - group: ":android: Android" steps: diff --git a/.gitignore b/.gitignore index 24520cf..4f17ba4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules/ tmp/ bktec +.bktest/ # pytest *.egg* diff --git a/swift-xctest/Package.resolved b/swift-xctest/Package.resolved index 5aba5dd..0112e6d 100644 --- a/swift-xctest/Package.resolved +++ b/swift-xctest/Package.resolved @@ -1,14 +1,213 @@ { + "originHash" : "160cfae0e90ff235481ca5f06cd3ead4f7d578a3db9d0f43104fc3e07a08a105", "pins" : [ { - "identity" : "test-collector-swift", + "identity" : "grpc-swift", "kind" : "remoteSourceControl", - "location" : "https://github.com/buildkite/test-collector-swift", + "location" : "https://github.com/grpc/grpc-swift.git", "state" : { - "revision" : "dbb0f13987c3f9f54611aaf0ac0dc1085ebb5ea6", - "version" : "0.6.0" + "revision" : "a56a157218877ef3e9625f7e1f7b2cb7e46ead1b", + "version" : "1.26.1" + } + }, + { + "identity" : "opentelemetry-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/open-telemetry/opentelemetry-swift", + "state" : { + "revision" : "0b77b8e2b2121261562b1d304c638a004e1d2f19", + "version" : "2.2.0" + } + }, + { + "identity" : "opentelemetry-swift-core", + "kind" : "remoteSourceControl", + "location" : "https://github.com/open-telemetry/opentelemetry-swift-core", + "state" : { + "revision" : "fd787757decabfa93319cc3c04f03a49e0cf40b6", + "version" : "2.2.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "f70225981241859eb4aa1a18a75531d26637c8cc", + "version" : "1.4.0" + } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", + "version" : "1.1.5" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-certificates", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-certificates.git", + "state" : { + "revision" : "c399f90e7bbe8874f6cbfda1d5f9023d1f5ce122", + "version" : "1.15.1" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "95ba0316a9b733e92bb6b071255ff46263bbe7dc", + "version" : "3.15.1" + } + }, + { + "identity" : "swift-http-structured-headers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-structured-headers.git", + "state" : { + "revision" : "1625f271afb04375bf48737a5572613248d0e7a0", + "version" : "1.4.0" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types.git", + "state" : { + "revision" : "45eb0224913ea070ec4fba17291b9e7ecf4749ca", + "version" : "1.5.1" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "ce592ae52f982c847a4efc0dd881cc9eb32d29f2", + "version" : "1.6.4" + } + }, + { + "identity" : "swift-metrics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-metrics", + "state" : { + "revision" : "0743a9364382629da3bf5677b46a2c4b1ce5d2a6", + "version" : "2.7.1" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "a18bddb0acf7a40d982b2f128ce73ce4ee31f352", + "version" : "2.86.2" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras.git", + "state" : { + "revision" : "a55c3dd3a81d035af8a20ce5718889c0dcab073d", + "version" : "1.29.0" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "5e9e99ec96c53bc2c18ddd10c1e25a3cd97c55e5", + "version" : "1.38.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "b2b043a8810ab6d51b3ff4df17f057d87ef1ec7c", + "version" : "2.34.1" + } + }, + { + "identity" : "swift-nio-transport-services", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-transport-services.git", + "state" : { + "revision" : "df6c28355051c72c884574a6c858bc54f7311ff9", + "version" : "1.25.2" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "c5ab62237f21cad094812719a1bbe29443407c5f", + "version" : "1.34.1" + } + }, + { + "identity" : "swift-service-lifecycle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-service-lifecycle.git", + "state" : { + "revision" : "1de37290c0ab3c5a96028e0f02911b672fd42348", + "version" : "2.9.1" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "0b30161977799fd949f7f6848586b82ff6764f73", + "version" : "1.6.6" } } ], - "version" : 2 + "version" : 3 } diff --git a/swift-xctest/Package.swift b/swift-xctest/Package.swift index bd3cf9c..4e5b04e 100644 --- a/swift-xctest/Package.swift +++ b/swift-xctest/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.9 +// swift-tools-version: 5.10 import PackageDescription let package = Package( @@ -10,7 +10,7 @@ let package = Package( .library(name: "ExampleLib", targets: ["ExampleLib"]) ], dependencies: [ - .package(url: "https://github.com/buildkite/test-collector-swift", from: "0.6.0") + .package(path: "../.bktest/test-collector-swift") ], targets: [ .target( diff --git a/swift-xcuitest/ExampleApp/ExampleApp.swift b/swift-xcuitest/ExampleApp/ExampleApp.swift index 9d0a04a..d8dc18f 100644 --- a/swift-xcuitest/ExampleApp/ExampleApp.swift +++ b/swift-xcuitest/ExampleApp/ExampleApp.swift @@ -1,12 +1,44 @@ +import Foundation import SwiftUI @main struct ExampleApp: App { var body: some Scene { WindowGroup { + ContentView() + } + } +} + +private struct ContentView: View { + private let runID: String + @State private var runnerCrashRecorded: Bool + + init() { + let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local" + self.runID = runID + self._runnerCrashRecorded = State( + initialValue: UserDefaults.standard.string(forKey: "runner-crash-run-id") == runID + ) + } + + var body: some View { + VStack { Text("Swift XCUITest example") .accessibilityIdentifier("status") - .padding() + + if runnerCrashRecorded { + Text("Runner crash recorded") + .accessibilityIdentifier("runner-crash-recorded") + } else { + Button("Record runner crash") { + UserDefaults.standard.set(runID, forKey: "runner-crash-run-id") + UserDefaults.standard.synchronize() + runnerCrashRecorded = true + } + .accessibilityIdentifier("record-runner-crash") + } } + .padding() } } diff --git a/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift b/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift index 500ebcf..270732a 100644 --- a/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift +++ b/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift @@ -1,4 +1,5 @@ import Core +import Darwin import XCTest final class ExampleAppUITests: XCTestCase { @@ -7,8 +8,7 @@ final class ExampleAppUITests: XCTestCase { } func test01AppLaunches() { - let app = XCUIApplication() - app.launch() + let app = launchApp() XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5)) } @@ -16,9 +16,44 @@ final class ExampleAppUITests: XCTestCase { func test02ExecutionTag() { tagExecution("custom.tag.from", "execution") - let app = XCUIApplication() - app.launch() + let app = launchApp() XCTAssertEqual(app.staticTexts["status"].label, "Swift XCUITest example") } + + func test03RunnerCrashesOnce() { + let app = launchApp() + + if app.buttons["record-runner-crash"].waitForExistence(timeout: 2) { + app.buttons["record-runner-crash"].tap() + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2)) + + _ = kill(getpid(), SIGKILL) + XCTFail("The runner process should have terminated") + } + + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2)) + } + + func test04RunsInReplacementRunner() { + let app = launchApp() + + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 5)) + } + + func test05ExecutionTagAfterRestart() { + tagExecution("runner.replacement", "true") + + let app = launchApp() + XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5)) + XCTAssertEqual(app.staticTexts["status"].label, "Swift XCUITest example") + } + + private func launchApp() -> XCUIApplication { + let app = XCUIApplication() + let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local" + app.launchEnvironment["XCUITEST_RUN_ID"] = runID + app.launch() + return app + } } diff --git a/swift-xcuitest/README.md b/swift-xcuitest/README.md index be7ffb4..a714882 100644 --- a/swift-xcuitest/README.md +++ b/swift-xcuitest/README.md @@ -1,9 +1,26 @@ # Swift XCUITest example -This example runs passing UI tests against a minimal iOS app and uploads the -results with [`test-collector-swift`](https://github.com/buildkite/test-collector-swift). -It uses a shared scheme, a lexical test plan, and an explicit `.xcresult` -bundle so the same command works locally and on a macOS Buildkite agent. +This example verifies OpenTelemetry execution delivery when XCTest replaces an +XCUITest runner. Two tests finish before a third test terminates the runner. +Xcode continues the remaining tests in a replacement process and keeps all +five tests in the `.xcresult`: four passes and the deliberately killed active +test. Each of the four tests that receives an XCTest completion callback is +handed to the bktec OTLP relay before its runner can exit. + +The verification script asserts the expected runner crash and replacement, +then checks that all four completed tests were accepted over OTLP. The killed +test cannot be emitted by an in-process collector because `SIGKILL` prevents +XCTest's completion callback; recovering that execution would require durable +start-state reconciliation outside the runner. + +This branch uses the Swift collector from the `swift-otel-executions` branch of +[`buildkite/bktest`](https://github.com/buildkite/bktest). Check it out before +running locally: + +```sh +git clone --branch swift-otel-executions --depth 1 \ + https://github.com/buildkite/bktest ../.bktest +``` Run it with: diff --git a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.pbxproj b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.pbxproj index 65a3cd5..8abff26 100644 --- a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.pbxproj +++ b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.pbxproj @@ -145,7 +145,7 @@ mainGroup = 15771E3CC944F418353D1657; minimizedProjectReferenceProxies = 0; packageReferences = ( - 0F260402842AAB649211B9B2 /* XCRemoteSwiftPackageReference "test-collector-swift" */, + 0F260402842AAB649211B9B2 /* XCLocalSwiftPackageReference "../.bktest/test-collector-swift" */, ); preferredProjectObjectVersion = 100; productRefGroup = 55AE842CDC491C8FB4A68112 /* Products */; @@ -415,21 +415,17 @@ }; /* End XCConfigurationList section */ -/* Begin XCRemoteSwiftPackageReference section */ - 0F260402842AAB649211B9B2 /* XCRemoteSwiftPackageReference "test-collector-swift" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/buildkite/test-collector-swift"; - requirement = { - kind = upToNextMajorVersion; - minimumVersion = 0.6.0; - }; +/* Begin XCLocalSwiftPackageReference section */ + 0F260402842AAB649211B9B2 /* XCLocalSwiftPackageReference "../.bktest/test-collector-swift" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = "../.bktest/test-collector-swift"; }; -/* End XCRemoteSwiftPackageReference section */ +/* End XCLocalSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ DB07F6513929E915435499BA /* BuildkiteTestCollector */ = { isa = XCSwiftPackageProductDependency; - package = 0F260402842AAB649211B9B2 /* XCRemoteSwiftPackageReference "test-collector-swift" */; + package = 0F260402842AAB649211B9B2 /* XCLocalSwiftPackageReference "../.bktest/test-collector-swift" */; productName = BuildkiteTestCollector; }; /* End XCSwiftPackageProductDependency section */ diff --git a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index cef9935..7cb84d3 100644 --- a/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/swift-xcuitest/SwiftXCUITest.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,13 +1,229 @@ { - "originHash" : "f28a54e288718485642adbe7b780a05abf05f95f0f435b3f87b6d3684b95d268", + "originHash" : "70638e11aba3a1472dfd0d75666b7caef58756d280301136e8d429ecf34ce1b3", "pins" : [ { - "identity" : "test-collector-swift", + "identity" : "grpc-swift", "kind" : "remoteSourceControl", - "location" : "https://github.com/buildkite/test-collector-swift", + "location" : "https://github.com/grpc/grpc-swift.git", "state" : { - "revision" : "dbb0f13987c3f9f54611aaf0ac0dc1085ebb5ea6", - "version" : "0.6.0" + "revision" : "a56a157218877ef3e9625f7e1f7b2cb7e46ead1b", + "version" : "1.26.1" + } + }, + { + "identity" : "opentelemetry-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/open-telemetry/opentelemetry-swift", + "state" : { + "revision" : "0b77b8e2b2121261562b1d304c638a004e1d2f19", + "version" : "2.2.0" + } + }, + { + "identity" : "opentelemetry-swift-core", + "kind" : "remoteSourceControl", + "location" : "https://github.com/open-telemetry/opentelemetry-swift-core", + "state" : { + "revision" : "fd787757decabfa93319cc3c04f03a49e0cf40b6", + "version" : "2.2.0" + } + }, + { + "identity" : "opentracing-objc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/undefinedlabs/opentracing-objc", + "state" : { + "revision" : "18c1a35ca966236cee0c5a714a51a73ff33384c1", + "version" : "0.5.2" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "9f542610331815e29cc3821d3b6f488db8715517", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", + "version" : "1.1.5" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-certificates", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-certificates.git", + "state" : { + "revision" : "24ccdeeeed4dfaae7955fcac9dbf5489ed4f1a25", + "version" : "1.18.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "bb4ba815dab96d4edc1e0b86d7b9acf9ff973a84", + "version" : "4.3.1" + } + }, + { + "identity" : "swift-http-structured-headers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-structured-headers.git", + "state" : { + "revision" : "76d7627bd88b47bf5a0f8497dd244885960dde0b", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types.git", + "state" : { + "revision" : "45eb0224913ea070ec4fba17291b9e7ecf4749ca", + "version" : "1.5.1" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "8c0f217f01000dd30f60d6e536569ad4e74291f9", + "version" : "1.11.0" + } + }, + { + "identity" : "swift-metrics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-metrics.git", + "state" : { + "revision" : "0743a9364382629da3bf5677b46a2c4b1ce5d2a6", + "version" : "2.7.1" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "558f24a4647193b5a0e2104031b71c55d31ff83a", + "version" : "2.97.1" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras.git", + "state" : { + "revision" : "abcf5312eb8ed2fb11916078aef7c46b06f20813", + "version" : "1.33.0" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "6d8d596f0a9bfebb925733003731fe2d749b7e02", + "version" : "1.42.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "df9c3406028e3297246e6e7081977a167318b692", + "version" : "2.36.1" + } + }, + { + "identity" : "swift-nio-transport-services", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-transport-services.git", + "state" : { + "revision" : "60c3e187154421171721c1a38e800b390680fb5d", + "version" : "1.26.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "81558271e243f8f47dfe8e9fdd55f3c2b5413f68", + "version" : "1.37.0" + } + }, + { + "identity" : "swift-service-lifecycle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-service-lifecycle.git", + "state" : { + "revision" : "9829955b385e5bb88128b73f1b8389e9b9c3191a", + "version" : "2.11.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "0b30161977799fd949f7f6848586b82ff6764f73", + "version" : "1.6.6" + } + }, + { + "identity" : "thrift-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/undefinedlabs/Thrift-Swift", + "state" : { + "revision" : "18ff09e6b30e589ed38f90a1af23e193b8ecef8e", + "version" : "1.1.2" } } ], diff --git a/swift-xcuitest/bin/test b/swift-xcuitest/bin/test index c33db23..fcb4bcb 100755 --- a/swift-xcuitest/bin/test +++ b/swift-xcuitest/bin/test @@ -5,9 +5,10 @@ set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" artifacts_dir="${XCUITEST_ARTIFACTS_DIR:-$root/artifacts}" result_bundle="$artifacts_dir/SwiftXCUITest.xcresult" +test_log="$artifacts_dir/xcodebuild.log" for name in \ - BUILDKITE_ANALYTICS_BASE_URL \ + BUILDKITE_ANALYTICS_KEY \ BUILDKITE_ANALYTICS_BRANCH \ BUILDKITE_ANALYTICS_DEBUG_ENABLED \ BUILDKITE_ANALYTICS_ENABLED \ @@ -17,6 +18,7 @@ for name in \ BUILDKITE_ANALYTICS_JOB_ID \ BUILDKITE_ANALYTICS_MESSAGE \ BUILDKITE_ANALYTICS_NUMBER \ + BUILDKITE_ANALYTICS_OTLP_ENDPOINT \ BUILDKITE_ANALYTICS_SHA \ BUILDKITE_ANALYTICS_TAGS \ BUILDKITE_ANALYTICS_TOKEN \ @@ -27,13 +29,27 @@ for name in \ BUILDKITE_BUILD_URL \ BUILDKITE_COMMIT \ BUILDKITE_JOB_ID \ - BUILDKITE_MESSAGE + BUILDKITE_MESSAGE \ + BUILDKITE_ORGANIZATION_SLUG \ + BUILDKITE_STEP_ID \ + BUILDKITE_TAG \ + BUILDKITE_TEST_ENGINE_SUITE_SLUG \ + BUILDKITE_TESTS_OTLP_TOKEN \ + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT \ + OTEL_EXPORTER_OTLP_TRACES_HEADERS \ + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL \ + OTEL_RESOURCE_ATTRIBUTES \ + TRACEPARENT \ + TRACESTATE do if [[ -n "${!name:-}" ]]; then export "TEST_RUNNER_${name}=${!name}" fi done +run_id="${XCUITEST_RUN_ID:-${BUILDKITE_BUILD_ID:-local}-${BUILDKITE_JOB_ID:-$(date +%s)}}" +export "TEST_RUNNER_XCUITEST_RUN_ID=$run_id" + destination="${XCUITEST_DESTINATION:-}" if [[ -z "$destination" ]]; then device_id="$({ @@ -57,6 +73,7 @@ xcodebuild \ -scheme SwiftXCUITest \ -resolvePackageDependencies +set +e xcodebuild \ -project "$root/SwiftXCUITest.xcodeproj" \ -scheme SwiftXCUITest \ @@ -64,4 +81,17 @@ xcodebuild \ -destination "$destination" \ -derivedDataPath "$artifacts_dir/DerivedData" \ -resultBundlePath "$result_bundle" \ - test + -retry-tests-on-failure \ + -test-iterations 2 \ + test 2>&1 | tee "$test_log" +xcodebuild_status="${PIPESTATUS[0]}" +set -e + +verification_status=0 +"$root/bin/verify-results" "$result_bundle" "$test_log" || verification_status="$?" + +if [[ "$xcodebuild_status" -ne 0 ]]; then + echo "xcodebuild exited $xcodebuild_status after the expected runner termination" +fi + +exit "$verification_status" diff --git a/swift-xcuitest/bin/verify-results b/swift-xcuitest/bin/verify-results new file mode 100755 index 0000000..c3f93cf --- /dev/null +++ b/swift-xcuitest/bin/verify-results @@ -0,0 +1,84 @@ +#!/bin/bash + +set -euo pipefail + +result_bundle="${1:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" +test_log="${2:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" +expected_result_count=5 +expected_completed_count=4 + +if summary="$(xcrun xcresulttool get test-results summary --path "$result_bundle" 2>/dev/null)"; then + result_count="$(plutil -extract totalTestCount raw -o - - <<< "$summary")" + passed_count="$(plutil -extract passedTests raw -o - - <<< "$summary")" + failed_count="$(plutil -extract failedTests raw -o - - <<< "$summary")" + failure_test="$(plutil -extract testFailures.0.testIdentifierString raw -o - - <<< "$summary")" + failure_text="$(plutil -extract testFailures.0.failureText raw -o - - <<< "$summary")" +else + if ! summary="$(xcrun xcresulttool get object --legacy --path "$result_bundle" --format json 2>/dev/null)"; then + summary="$(xcrun xcresulttool get --path "$result_bundle" --format json)" + fi + + action_result="actions._values.0.actionResult" + result_count="$(plutil -extract "$action_result.metrics.testsCount._value" raw -o - - <<< "$summary")" + failed_count="$(plutil -extract "$action_result.metrics.testsFailedCount._value" raw -o - - <<< "$summary")" + passed_count="$((result_count - failed_count))" + failure_test="$(plutil -extract "$action_result.issues.testFailureSummaries._values.0.testCaseName._value" raw -o - - <<< "$summary")" + failure_text="$(plutil -extract "$action_result.issues.testFailureSummaries._values.0.message._value" raw -o - - <<< "$summary")" +fi +exported_count="$(awk ' + { + line = $0 + while (match(line, /Exported OpenTelemetry test execution: test[^[:space:]]*/)) { + names[substr(line, RSTART, RLENGTH)] = 1 + line = substr(line, RSTART + RLENGTH) + } + } + END { + for (name in names) count++ + print count + 0 + } +' "$test_log")" + +echo "Expected .xcresult tests: $expected_result_count" +echo ".xcresult tests: $result_count ($passed_count passed, $failed_count runner crash)" +echo "Expected completed executions: $expected_completed_count" +echo "Collector unique executions accepted over OTLP: $exported_count" + +status=0 + +if [[ "$result_count" -ne "$expected_result_count" || "$passed_count" -ne 4 || "$failed_count" -ne 1 ]]; then + echo "Expected .xcresult to retain four passes and the deliberate runner crash" >&2 + status=1 +fi + +if [[ "$failure_test" != "ExampleAppUITests/test03RunnerCrashesOnce()" \ + && "$failure_test" != "ExampleAppUITests.test03RunnerCrashesOnce()" \ + || "$failure_text" != "Test crashed with signal kill." ]] +then + echo "Expected test03RunnerCrashesOnce to be the only failed test" >&2 + status=1 +fi + +if ! grep -Fq "Restarting after unexpected exit, crash, or test timeout" "$test_log"; then + echo "XCTest did not replace the terminated runner" >&2 + status=1 +fi + +for name in \ + test01AppLaunches \ + test02ExecutionTag \ + test04RunsInReplacementRunner \ + test05ExecutionTagAfterRestart +do + if ! grep -Fq "Exported OpenTelemetry test execution: $name" "$test_log"; then + echo "Collector did not export completed execution $name" >&2 + status=1 + fi +done + +if [[ "$exported_count" -ne "$expected_completed_count" ]]; then + echo "Collector did not hand every completed execution to the OTLP endpoint" >&2 + status=1 +fi + +exit "$status"