diff --git a/CustomAnalysisRules.Test.ruleset b/CustomAnalysisRules.Test.ruleset index 83956bf183..48edba4a52 100644 --- a/CustomAnalysisRules.Test.ruleset +++ b/CustomAnalysisRules.Test.ruleset @@ -129,4 +129,11 @@ + + + + diff --git a/Directory.Build.props b/Directory.Build.props index 81725068b9..e105dc9d77 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -29,14 +29,16 @@ - + xunit.runner.json PreserveNewest - + Never + + @@ -45,13 +47,15 @@ true false Exe + true + $(NoWarn);xUnit1051 - - + + diff --git a/Directory.Packages.props b/Directory.Packages.props index cb9e062442..f89ab52aa8 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -46,8 +46,10 @@ - + + + @@ -145,13 +147,13 @@ - - + + + + + + - - - - diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index 970f580b03..ba3e55cbb6 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -215,6 +215,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.R5.Fi EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Ignixa", "src\Microsoft.Health.Fhir.Ignixa\Microsoft.Health.Fhir.Ignixa.csproj", "{87B98ECB-D37E-4DD7-A51C-1AE7CB8765F7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Extensions.Xunit.UnitTests", "src\Microsoft.Health.Extensions.Xunit.UnitTests\Microsoft.Health.Extensions.Xunit.UnitTests.csproj", "{43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -517,6 +519,10 @@ Global {87B98ECB-D37E-4DD7-A51C-1AE7CB8765F7}.Debug|Any CPU.Build.0 = Debug|Any CPU {87B98ECB-D37E-4DD7-A51C-1AE7CB8765F7}.Release|Any CPU.ActiveCfg = Release|Any CPU {87B98ECB-D37E-4DD7-A51C-1AE7CB8765F7}.Release|Any CPU.Build.0 = Release|Any CPU + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}.Debug|Any CPU.Build.0 = Debug|Any CPU + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}.Release|Any CPU.ActiveCfg = Release|Any CPU + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -537,6 +543,7 @@ Global {C6759C03-6060-44D7-B44D-0BD89908B741} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} {87849B3F-5D12-41CA-A082-FAC065EF9FD8} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {4DDF9576-E22C-460A-937F-0EE0FEA6DB87} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} {E02E5224-32CD-490F-B1E5-8509AD669334} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {070759A9-51D9-4967-8651-39CCA8288C93} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} {D933AC8A-3CE5-4F03-AEE4-9AC0D0274D80} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} diff --git a/R4.slnf b/R4.slnf index 2786e2ab36..4aa8e6bf07 100644 --- a/R4.slnf +++ b/R4.slnf @@ -4,6 +4,7 @@ "projects": [ "samples\\apps\\SmartLauncher\\SmartLauncher.csproj", "src\\Microsoft.Health.Extensions.Xunit\\Microsoft.Health.Extensions.Xunit.csproj", + "src\\Microsoft.Health.Extensions.Xunit.UnitTests\\Microsoft.Health.Extensions.Xunit.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Api.UnitTests\\Microsoft.Health.Fhir.R4.Api.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Api\\Microsoft.Health.Fhir.Api.csproj", "src\\Microsoft.Health.Fhir.Api.OpenIddict\\Microsoft.Health.Fhir.Api.OpenIddict.csproj", diff --git a/R5.slnf b/R5.slnf index 092fe04bdc..e10c9ccb30 100644 --- a/R5.slnf +++ b/R5.slnf @@ -4,6 +4,7 @@ "projects": [ "samples\\apps\\SmartLauncher\\SmartLauncher.csproj", "src\\Microsoft.Health.Extensions.Xunit\\Microsoft.Health.Extensions.Xunit.csproj", + "src\\Microsoft.Health.Extensions.Xunit.UnitTests\\Microsoft.Health.Extensions.Xunit.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Api.UnitTests\\Microsoft.Health.Fhir.R5.Api.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Api\\Microsoft.Health.Fhir.Api.csproj", "src\\Microsoft.Health.Fhir.Api.OpenIddict\\Microsoft.Health.Fhir.Api.OpenIddict.csproj", diff --git a/THIRDPARTYNOTICES.md b/THIRDPARTYNOTICES.md index ac793ce6fa..a4afe13ffe 100644 --- a/THIRDPARTYNOTICES.md +++ b/THIRDPARTYNOTICES.md @@ -969,58 +969,3 @@ This file is based on or incorporates material from the projects listed below (T > WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > See the License for the specific language governing permissions and > limitations under the License. - -## YTest.MTP.XUnit2 1.0.3 -* Component Source: https://github.com/Youssef1313/YTest.MTP.XUnit2 -* Component Copyright and License: - > The MIT License (MIT) - > - > Copyright (c) Youssef Victor Gerges Fahmy - > - > All rights reserved. - > - > Permission is hereby granted, free of charge, to any person obtaining a copy - > of this software and associated documentation files (the "Software"), to deal - > in the Software without restriction, including without limitation the rights - > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - > copies of the Software, and to permit persons to whom the Software is - > furnished to do so, subject to the following conditions: - > - > The above copyright notice and this permission notice shall be included in all - > copies or substantial portions of the Software. - > - > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - > SOFTWARE. - -## Xunit.SkippableFact 1.4.13 -* Component Source: https://github.com/AArnott/Xunit.SkippableFact -* Component Copyright and License: - > This software is licensed under the Microsoft Public License - > http://opensource.org/licenses/ms-pl - > - > This license governs use of the accompanying software. If you use the software, you - > accept this license. If you do not accept the license, do not use the software. - > - > 1. Definitions - > The terms "reproduce," "reproduction," "derivative works," and "distribution" have the - > same meaning here as under U.S. copyright law. - > A "contribution" is the original software, or any additions or changes to the software. - > A "contributor" is any person that distributes its contribution under this license. - > "Licensed patents" are a contributor's patent claims that read directly on its contribution. - > - > 2. Grant of Rights - > (A) Copyright Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free copyright license to reproduce its contribution, prepare derivative works of its contribution, and distribute its contribution or any derivative works that you create. - > (B) Patent Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free license under its licensed patents to make, have made, use, sell, offer for sale, import, and/or otherwise dispose of its contribution in the software or derivative works of the contribution in the software. - > - > 3. Conditions and Limitations - > (A) No Trademark License- This license does not grant you rights to use any contributors' name, logo, or trademarks. - > (B) If you bring a patent claim against any contributor over patents that you claim are infringed by the software, your patent license from such contributor to the software ends automatically. - > (C) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software. - > (D) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license. - > (E) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement. - diff --git a/build/jobs/build.yml b/build/jobs/build.yml index 227ceeca10..72249ce33f 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -30,36 +30,96 @@ steps: workingDirectory: $(System.DefaultWorkingDirectory) - ${{ if eq(parameters.unitTest, 'true') }}: + - task: PowerShell@2 + displayName: 'Assert the unit test projects were found' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1' + arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -ManifestPath "$(Build.SourcesDirectory)/build/jobs/scripts/UnitTestProjects.txt"' - task: DotNetCoreCLI@2 displayName: 'dotnet test' inputs: command: test projects: '**/*UnitTests/*.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx' + arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --minimum-expected-tests 1' + # Disable the task's built-in result publishing so the PublishTestResults@2 task below, + # which controls the glob and how failures are treated, is the sole publisher. + publishTestResults: false + env: + platformOptions__resultDirectory: '$(Agent.TempDirectory)/unit-test-results' + - task: PowerShell@2 + displayName: 'Reconcile retried unit test failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/unit-test-results"' + condition: succeeded() + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '$(Agent.TempDirectory)/unit-test-results/**/*.trx' + mergeTestResults: true testRunTitle: 'Unit Tests' + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. + failTaskOnFailedTests: false + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and there is nothing to notice. + failTaskOnMissingResultsFile: true + condition: succeededOrFailed() - ${{ if eq(parameters.codeCoverage, 'true') }}: + - task: PowerShell@2 + displayName: 'Assert the unit test projects were found' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1' + arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -ManifestPath "$(Build.SourcesDirectory)/build/jobs/scripts/UnitTestProjects.txt"' - task: DotNetCoreCLI@2 displayName: 'dotnet test with coverage' inputs: command: test projects: '**/*UnitTests/*.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - testRunTitle: 'Unit Tests' - # Disable the task's built-in (non-retry-aware) result publishing so the - # explicit retry-aware PublishTestResults@2 task below is the sole publisher. + arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --minimum-expected-tests 1 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"' + # Disable the task's built-in result publishing so the PublishTestResults@2 task below, + # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false env: platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' + - task: PowerShell@2 + displayName: 'Reconcile retried unit test failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/coverage"' + condition: succeeded() - task: PublishTestResults@2 - displayName: 'Publish unit test results' - condition: succeededOrFailed() + displayName: 'Publish unit test with coverage results' inputs: testResultsFormat: 'VSTest' testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' mergeTestResults: true testRunTitle: 'Unit Tests' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. + failTaskOnFailedTests: false + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and there is nothing to notice. + failTaskOnMissingResultsFile: true + condition: succeededOrFailed() - task: reportgenerator@5 displayName: 'aggregate unit test coverage' condition: succeededOrFailed() diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index 0874229659..84293e54fd 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -11,6 +11,17 @@ parameters: - name: testRunTitleSuffix type: string default: '' +# The floor this leg's discovered test count must clear. A leg that selects positively on a +# DataStore trait reports success when its filter matches nothing, so the count is the only thing +# that distinguishes "everything passed" from "nothing ran". It is checked against a discovery +# pass rather than passed to the run as --minimum-expected-tests, because that option is applied +# again to the much smaller pass --retry-failed-tests starts: a floor above the size of a retry +# pass turns a flake that recovered into a failed leg (exit 9). Set well below the real counts so +# version differences and legitimately removed tests do not fail a leg that is working. There is +# deliberately no default: a new leg has to state its own floor, because the only value that would +# be safe to inherit is one too low to ever fire. +- name: minimumExpectedTests + type: string steps: - template: e2e-tests-extract.yml @@ -37,10 +48,43 @@ steps: throw "Could not find $testName.dll under $testRoot" } - $filter = "FullyQualifiedName~${{ parameters.appServiceType }}&${{ parameters.categoryFilter }}" - $args = @('--filter', $filter, '--retry-failed-tests', '3', '--report-trx') + # xUnit v3 MTP filter-query: match DataStore trait for this app service type and any extra trait filters + $categoryFilter = '${{ parameters.categoryFilter }}' + $parts = @($categoryFilter -split '&' | Where-Object { $_.Trim() -ne '' }) + $allParts = @("DataStore=${{ parameters.appServiceType }}") + $parts + $traitFilter = "[" + (($allParts | ForEach-Object { "($_)" }) -join '&') + "]" + $filterQuery = "/$traitFilter" + + # This filter is positive on DataStore, so a test case that reaches the runner without that + # trait is invisible to it rather than failing. Count what the filter selects before running + # anything, so a trait or filter regression that empties this leg fails it instead of + # passing it green and empty. Discovery needs no server, and unlike the runner's own + # --minimum-expected-tests this count is not re-applied to the smaller pass that + # --retry-failed-tests starts, where any real floor would fail a recovered flake. + $listOutput = & dotnet $dll.FullName --list-tests --filter-query $filterQuery + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for filter $filterQuery failed with exit code $LASTEXITCODE" + } - Write-Host "Running dotnet $($dll.FullName)" + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for filter $filterQuery from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for filter $filterQuery (floor ${{ parameters.minimumExpectedTests }})" + if ($discovered -lt ${{ parameters.minimumExpectedTests }}) { + throw "Filter $filterQuery selected $discovered test(s), below this leg's floor of ${{ parameters.minimumExpectedTests }}. The leg would have run a fraction of its tests and reported success." + } + + # Kept at 1 as a last-ditch check that the run itself published something. Anything higher + # is unsafe here for the retry reason above; the real floor is the discovery check. + # --parallel none is not optional and must not be dropped: these tests share one server, and + # the xunit.runner.json that would otherwise disable parallelism is deliberately not copied + # to a published layout, so the command line is the only place this is guaranteed. + $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') + + Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" & dotnet $dll.FullName @args if ($LASTEXITCODE -ne 0) { @@ -88,6 +132,14 @@ steps: 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults' + - task: PowerShell@2 + displayName: 'Reconcile retried E2E failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/testresults"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish test results' inputs: @@ -95,5 +147,16 @@ steps: testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx' mergeTestResults: true testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. + failTaskOnFailedTests: false + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and there is nothing to notice. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index 362bd96c4e..b4dfc71745 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -21,6 +21,13 @@ jobs: - ImageOverride -equals $(InternalLinuxImage) variables: AllowPtrToDetectTestRunRetryFiles: true + # The one place this leg's selection is written down. The discovery floor below is the only + # thing that notices a regression silently emptying this leg, and it can only say that about + # the selection it counts, so it has to count the same one the tests run under. Two copies of + # this text would let an edit to one of them leave the floor guarding a selection that is no + # longer used. The values carry no spaces, which is what lets the floor step split this back + # into arguments. + storeFilterArguments: '--filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer"' steps: - checkout: self fetchDepth: 1 @@ -71,15 +78,72 @@ jobs: projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' + - task: PowerShell@2 + displayName: 'Verify xUnit config and test counts' + inputs: + targetType: inline + pwsh: true + script: | + $ErrorActionPreference = 'Stop' + $projects = Get-ChildItem -Path "$(Pipeline.Workspace)/source/test" -Recurse -Filter "*${{ parameters.version }}.Tests.Integration.csproj" + if (-not $projects) { + throw "No integration test projects found under $(Pipeline.Workspace)/source/test." + } + + # Around 293 tests select here. This leg filters by excluding the other store, and an + # exclusion drops a case that names either value, so a class that starts carrying both - + # by inheriting traits from a shared collection definition, for instance - disappears from + # this leg and from the other one at the same time, leaving both green and much smaller. + # Counting discovery is the only thing that notices. The floor is checked here rather than + # passed as --minimum-expected-tests because that option is applied again to the smaller + # pass --retry-failed-tests starts, where any real floor would fail a recovered flake. + $minimumExpectedTests = 150 + + foreach ($project in $projects) { + $outputDir = [System.IO.Path]::Combine($project.Directory.FullName, "bin", "$(buildConfiguration)", "$(defaultBuildFramework)") + $configPath = Join-Path $outputDir "xunit.runner.json" + if (-not (Test-Path $configPath)) { + throw "xunit.runner.json not found for $($project.FullName) at $configPath" + } + + Write-Host "xUnit config for $($project.BaseName): $configPath" + Get-Content -Path $configPath | ForEach-Object { Write-Host $_ } + + $assemblyPath = Join-Path $outputDir "$($project.BaseName).dll" + if (-not (Test-Path $assemblyPath)) { + throw "Test assembly not found for $($project.FullName) at $assemblyPath" + } + + $storeFilter = @('$(storeFilterArguments)' -split '\s+' | Where-Object { $_ } | ForEach-Object { $_.Trim('"') }) + if ($storeFilter.Count -lt 2) { + throw "The store filter '$(storeFilterArguments)' did not split into arguments, so the floor below would count a selection this leg does not run." + } + + $listOutput = & dotnet $assemblyPath --list-tests @storeFilter + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for $($project.BaseName) failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for $($project.BaseName) from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for $($project.BaseName) (floor $minimumExpectedTests)" + if ($discovered -lt $minimumExpectedTests) { + throw "$($project.BaseName) selected $discovered test(s), below this leg's floor of $minimumExpectedTests. The leg would have run a fraction of its tests and reported success." + } + } + - task: DotNetCoreCLI@2 displayName: 'Run Cosmos Integration Tests with coverage' inputs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' - # Disable the task's built-in (non-retry-aware) result publishing so the - # explicit retry-aware PublishTestResults@2 task below is the sole publisher. + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- $(storeFilterArguments) --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx --minimum-expected-tests 1' + # Disable the task's built-in result publishing so the PublishTestResults@2 task below, + # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false env: 'CosmosDb__Host': $(CosmosDb--Host) @@ -91,6 +155,14 @@ jobs: 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)' 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) + - task: PowerShell@2 + displayName: 'Reconcile retried integration failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/coverage"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish integration test results' inputs: @@ -98,7 +170,18 @@ jobs: testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' mergeTestResults: true testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. + failTaskOnFailedTests: false + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and there is nothing to notice. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() - task: reportgenerator@5 @@ -133,6 +216,9 @@ jobs: containerAppName: '${{ parameters.containerAppName }}' appServiceType: 'CosmosDb' categoryFilter: '${{ parameters.mainCategoryFilter }}' + # Around 1350 cases across the FHIR versions. A floor of 700 leaves room for version + # differences and removed tests while failing a leg that has quietly lost its traits. + minimumExpectedTests: '700' - ${{ if eq(parameters.runReindexJob, true) }}: - job: 'cosmosE2eTests_Reindex' @@ -153,3 +239,5 @@ jobs: appServiceType: 'CosmosDb' categoryFilter: 'Category=IndexAndReindex' testRunTitleSuffix: ' Reindex' + # Around 52 cases. A floor of 25 fails a leg that has lost most of them. + minimumExpectedTests: '25' diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index 5397a4e637..708a824bbf 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -103,9 +103,33 @@ jobs: throw "Could not find $testName.dll under $testRoot" } - $args = @('--filter', 'FullyQualifiedName~CosmosDb&Category=ExportLongRunning', '--retry-failed-tests', '3', '--report-trx') + # Positive on DataStore, so a case missing that trait is invisible here rather than failing. + # Count what the filter selects before running, so a regression that empties this leg fails + # it. The floor cannot ride on --minimum-expected-tests: that is re-applied to the smaller + # pass --retry-failed-tests starts, so a floor above a retry pass fails a recovered flake. + # Around 3 cases run here, so 2 is below any legitimate count and above an emptied leg. + $filterQuery = '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]' + $minimumExpectedTests = 2 - Write-Host "Running dotnet $($dll.FullName)" + $listOutput = & dotnet $dll.FullName --list-tests --filter-query $filterQuery + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for filter $filterQuery failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for filter $filterQuery from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for filter $filterQuery (floor $minimumExpectedTests)" + if ($discovered -lt $minimumExpectedTests) { + throw "Filter $filterQuery selected $discovered test(s), below this leg's floor of $minimumExpectedTests. The leg would have run a fraction of its tests and reported success." + } + + $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') + + Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" & dotnet $dll.FullName @args if ($LASTEXITCODE -ne 0) { @@ -141,6 +165,14 @@ jobs: 'app_globalWriterUserApp_secret': $(app_globalWriterUserApp_secret) platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults' + - task: PowerShell@2 + displayName: 'Reconcile retried export failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/testresults"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish test results' inputs: @@ -148,7 +180,18 @@ jobs: testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx' mergeTestResults: true testRunTitle: 'Export ${{ parameters.version }} CosmosDB' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. + failTaskOnFailedTests: false + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and there is nothing to notice. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() - job: 'sqlE2eTests' @@ -245,9 +288,33 @@ jobs: throw "Could not find $testName.dll under $testRoot" } - $args = @('--filter', 'FullyQualifiedName~SqlServer&Category=ExportLongRunning', '--retry-failed-tests', '3', '--report-trx') + # Positive on DataStore, so a case missing that trait is invisible here rather than failing. + # Count what the filter selects before running, so a regression that empties this leg fails + # it. The floor cannot ride on --minimum-expected-tests: that is re-applied to the smaller + # pass --retry-failed-tests starts, so a floor above a retry pass fails a recovered flake. + # Around 5 cases run here, so 3 is below any legitimate count and above an emptied leg. + $filterQuery = '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]' + $minimumExpectedTests = 3 - Write-Host "Running dotnet $($dll.FullName)" + $listOutput = & dotnet $dll.FullName --list-tests --filter-query $filterQuery + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for filter $filterQuery failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for filter $filterQuery from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for filter $filterQuery (floor $minimumExpectedTests)" + if ($discovered -lt $minimumExpectedTests) { + throw "Filter $filterQuery selected $discovered test(s), below this leg's floor of $minimumExpectedTests. The leg would have run a fraction of its tests and reported success." + } + + $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') + + Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" & dotnet $dll.FullName @args if ($LASTEXITCODE -ne 0) { @@ -283,6 +350,14 @@ jobs: 'app_globalWriterUserApp_secret': $(app_globalWriterUserApp_secret) platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults' + - task: PowerShell@2 + displayName: 'Reconcile retried export failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/testresults"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish test results' inputs: @@ -290,6 +365,17 @@ jobs: testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx' mergeTestResults: true testRunTitle: 'Export ${{ parameters.version }} SQL' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. + failTaskOnFailedTests: false + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and there is nothing to notice. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 45d9a82d85..9be2f922fb 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -26,6 +26,13 @@ jobs: - ImageOverride -equals $(InternalLinuxImage) variables: AllowPtrToDetectTestRunRetryFiles: true + # The one place this leg's selection is written down. The discovery floor below is the only + # thing that notices a regression silently emptying this leg, and it can only say that about + # the selection it counts, so it has to count the same one the tests run under. Two copies of + # this text would let an edit to one of them leave the floor guarding a selection that is no + # longer used. The values carry no spaces, which is what lets the floor step split this back + # into arguments. + storeFilterArguments: '--filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb"' steps: - checkout: self fetchDepth: 1 @@ -63,15 +70,72 @@ jobs: projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' + - task: PowerShell@2 + displayName: 'Verify xUnit config and test counts' + inputs: + targetType: inline + pwsh: true + script: | + $ErrorActionPreference = 'Stop' + $projects = Get-ChildItem -Path "$(Pipeline.Workspace)/source/test" -Recurse -Filter "*${{ parameters.version }}.Tests.Integration.csproj" + if (-not $projects) { + throw "No integration test projects found under $(Pipeline.Workspace)/source/test." + } + + # Around 441 tests select here. This leg filters by excluding the other store, and an + # exclusion drops a case that names either value, so a class that starts carrying both - + # by inheriting traits from a shared collection definition, for instance - disappears from + # this leg and from the other one at the same time, leaving both green and much smaller. + # Counting discovery is the only thing that notices. The floor is checked here rather than + # passed as --minimum-expected-tests because that option is applied again to the smaller + # pass --retry-failed-tests starts, where any real floor would fail a recovered flake. + $minimumExpectedTests = 250 + + foreach ($project in $projects) { + $outputDir = [System.IO.Path]::Combine($project.Directory.FullName, "bin", "$(buildConfiguration)", "$(defaultBuildFramework)") + $configPath = Join-Path $outputDir "xunit.runner.json" + if (-not (Test-Path $configPath)) { + throw "xunit.runner.json not found for $($project.FullName) at $configPath" + } + + Write-Host "xUnit config for $($project.BaseName): $configPath" + Get-Content -Path $configPath | ForEach-Object { Write-Host $_ } + + $assemblyPath = Join-Path $outputDir "$($project.BaseName).dll" + if (-not (Test-Path $assemblyPath)) { + throw "Test assembly not found for $($project.FullName) at $assemblyPath" + } + + $storeFilter = @('$(storeFilterArguments)' -split '\s+' | Where-Object { $_ } | ForEach-Object { $_.Trim('"') }) + if ($storeFilter.Count -lt 2) { + throw "The store filter '$(storeFilterArguments)' did not split into arguments, so the floor below would count a selection this leg does not run." + } + + $listOutput = & dotnet $assemblyPath --list-tests @storeFilter + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for $($project.BaseName) failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for $($project.BaseName) from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for $($project.BaseName) (floor $minimumExpectedTests)" + if ($discovered -lt $minimumExpectedTests) { + throw "$($project.BaseName) selected $discovered test(s), below this leg's floor of $minimumExpectedTests. The leg would have run a fraction of its tests and reported success." + } + } + - task: DotNetCoreCLI@2 displayName: 'Run SQL Integration Tests with coverage' inputs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - testRunTitle: '${{ parameters.version }} SQL Integration Tests' - # Disable the task's built-in (non-retry-aware) result publishing so the - # explicit retry-aware PublishTestResults@2 task below is the sole publisher. + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- $(storeFilterArguments) --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx --minimum-expected-tests 1' + # Disable the task's built-in result publishing so the PublishTestResults@2 task below, + # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false env: 'SqlServer:ConnectionString': 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);' @@ -81,6 +145,14 @@ jobs: 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)' 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) + - task: PowerShell@2 + displayName: 'Reconcile retried integration failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/coverage"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish integration test results' inputs: @@ -88,7 +160,18 @@ jobs: testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' mergeTestResults: true testRunTitle: '${{ parameters.version }} SQL Integration Tests' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. + failTaskOnFailedTests: false + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and there is nothing to notice. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() - task: reportgenerator@5 @@ -123,6 +206,9 @@ jobs: containerAppName: '${{ parameters.containerAppName }}' appServiceType: 'SqlServer' categoryFilter: '${{ parameters.mainCategoryFilter }}' + # Around 2000 cases across the FHIR versions. A floor of 1000 leaves room for version + # differences and removed tests while failing a leg that has quietly lost its traits. + minimumExpectedTests: '1000' - ${{ if eq(parameters.runReindexJob, true) }}: - job: 'sqlE2eTests_Reindex' @@ -143,6 +229,8 @@ jobs: appServiceType: 'SqlServer' categoryFilter: 'Category=IndexAndReindex' testRunTitleSuffix: ' Reindex' + # Around 52 cases. A floor of 25 fails a leg that has lost most of them. + minimumExpectedTests: '25' - ${{ if eq(parameters.runBulkUpdateJob, true) }}: - job: 'sqlE2eTests_BulkUpdate' @@ -163,3 +251,5 @@ jobs: appServiceType: 'SqlServer' categoryFilter: 'Category=BulkUpdate' testRunTitleSuffix: ' BulkUpdate' + # Around 35 cases. A floor of 15 fails a leg that has lost most of them. + minimumExpectedTests: '15' diff --git a/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 b/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 new file mode 100644 index 0000000000..3e864e2c74 --- /dev/null +++ b/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 @@ -0,0 +1,155 @@ +<# +.SYNOPSIS + Fails a test leg where a failing test was retried and never actually passed. + +.DESCRIPTION + The test legs let the runner's exit code be the verdict, because the results of every retry + attempt are published and failing on those would cancel out every retry. That reasoning holds + only while "the runner exited zero" means "every failure was cleared by a later attempt", and it + does not: a test that fails and then skips is not counted as a failure by the retry extension, so + a run where a real failure turned into a skip exits zero and the leg goes green. + + This reads what the attempts actually reported. Every test that failed in any attempt has to be + recorded as passed in the final attempt. A test that failed and then skipped, or that failed and + then was not run at all, fails the leg here instead of disappearing. + + Results are keyed by the identifier the report gives each test, together with the assembly it + came from. Display names are not identities: a theory whose rows differ only in the case of an + argument produces two tests with names PowerShell's hashtables cannot tell apart, so keying by + name lets one test's pass clear a different test's failure. The assembly stays in the key + because this repository compiles the same shared test files into several assemblies, which can + give the same test the same identifier in each of them. + +.PARAMETER ResultsDirectory + The directory the runner was given for its results. Retry attempts are below it under Retries. +#> +#requires -Version 7.0 + +param( + [Parameter(Mandatory = $true)] + [string] $ResultsDirectory +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $ResultsDirectory)) { + throw "The results directory '$ResultsDirectory' does not exist, so no test results can be read from it." +} + +$allReports = @(Get-ChildItem -LiteralPath $ResultsDirectory -Recurse -Filter *.trx -File) +$attemptReports = @($allReports | Where-Object { $_.FullName -match '[\\/]Retries[\\/]' }) + +if ($attemptReports.Count -eq 0) { + Write-Host "No retry attempts were recorded under '$ResultsDirectory', so there is nothing to reconcile." + exit 0 +} + +$finalReports = @($allReports | Where-Object { $_.FullName -notmatch '[\\/]Retries[\\/]' }) + +if ($finalReports.Count -eq 0) { + throw "Retry attempts were recorded under '$ResultsDirectory' but no final report was found beside them, so what the retried tests ended up doing cannot be read." +} + +function Read-Outcomes { + param([System.IO.FileInfo] $Report) + + [xml] $document = Get-Content -LiteralPath $Report.FullName -Raw + + # An empty or is what a report that recorded nothing looks like, + # and the XML adapter surfaces an empty element as a string, so reaching through it for child + # elements fails. That would fail the leg with a PowerShell error in place of the verdict below, + # which is the one message that explains a report holding no results at all. Selecting from the + # document instead is unaffected by how any one element happens to be surfaced, and matching on + # local names keeps it working whether or not the TRX carries its namespace. + $definitions = @($document.SelectNodes('//*[local-name()="UnitTest"]')) + + $storageById = [System.Collections.Hashtable]::new([System.StringComparer]::Ordinal) + foreach ($definition in $definitions) { + if ($null -ne $definition) { + # Attributes are read with GetAttribute rather than as properties: strict mode turns a + # missing property into an error about PowerShell, which would replace the explanations + # below with noise for the one kind of report that most needs explaining. + $storageById[$definition.GetAttribute('id').ToLowerInvariant()] = $definition.GetAttribute('storage').ToLowerInvariant() + } + } + + $results = @($document.SelectNodes('//*[local-name()="UnitTestResult"]')) + + foreach ($result in $results) { + if ($null -eq $result) { + continue + } + + $testId = $result.GetAttribute('testId') + $testName = $result.GetAttribute('testName') + + # Without an identifier there is nothing to reconcile this result against, and guessing from + # the name is what this key exists to avoid. Refusing here fails the leg, which is the safe + # direction: the alternative is dropping a result that may be the failure being looked for. + if ([string]::IsNullOrWhiteSpace($testId)) { + throw "'$($Report.FullName)' records a result for '$testName' with no test id, so it cannot be matched to the same test in another attempt." + } + + $testId = $testId.ToLowerInvariant() + + $storage = '' + if ($storageById.ContainsKey($testId)) { + $storage = $storageById[$testId] + } + + [PSCustomObject]@{ + Key = "$storage|$testId" + Name = $testName + Outcome = $result.GetAttribute('outcome') + } + } +} + +$failedWhenRetried = [System.Collections.Hashtable]::new([System.StringComparer]::Ordinal) +foreach ($report in $attemptReports) { + foreach ($outcome in Read-Outcomes -Report $report) { + if ($outcome.Outcome -eq 'Failed') { + $failedWhenRetried[$outcome.Key] = $outcome.Name + } + } +} + +if ($failedWhenRetried.Count -eq 0) { + Write-Host "Retry attempts were recorded but none of them reported a failure, so there is nothing to reconcile." + exit 0 +} + +$finalOutcomes = [System.Collections.Hashtable]::new([System.StringComparer]::Ordinal) +foreach ($report in $finalReports) { + foreach ($outcome in Read-Outcomes -Report $report) { + # A test can be recorded more than once in a final report. Passed is what clears an earlier + # failure, so it wins over any other outcome recorded for the same test. + if ($outcome.Outcome -eq 'Passed' -or -not $finalOutcomes.ContainsKey($outcome.Key)) { + $finalOutcomes[$outcome.Key] = $outcome.Outcome + } + } +} + +$unresolved = @() +foreach ($key in $failedWhenRetried.Keys) { + $final = if ($finalOutcomes.ContainsKey($key)) { $finalOutcomes[$key] } else { 'not run at all' } + + if ($final -ne 'Passed') { + $unresolved += " $($failedWhenRetried[$key]) -> $final" + } +} + +if ($unresolved.Count -gt 0) { + $detail = $unresolved -join [Environment]::NewLine + throw @" +These tests failed and were retried, but the final attempt does not record them as passing: +$detail + +The runner treats anything other than a failure as retried successfully, so a test that fails and +then skips, or that fails and then does not run, exits the runner zero and would otherwise leave +this leg green with a real failure inside it. +"@ +} + +Write-Host "Every test that failed in a retry attempt passed in the final attempt ($($failedWhenRetried.Count) reconciled)." diff --git a/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 b/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 new file mode 100644 index 0000000000..12338d25fa --- /dev/null +++ b/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 @@ -0,0 +1,102 @@ +<# +.SYNOPSIS + Fails when the unit test leg's project glob matches fewer projects than expected. + +.DESCRIPTION + The unit test leg does not name its projects: it runs whatever '**/*UnitTests/*.csproj' + matches. Every other leg guards against running a fraction of its tests by counting the tests + a filter discovers, but that shape does not fit here, because this leg's failure is a whole + project leaving the glob - renamed, moved, or a directory suffix quietly changed - rather than + a filter selecting too little. Nothing notices: the remaining projects run, every one of them + passes, and the leg reports success with an entire assembly's tests missing. + + --minimum-expected-tests cannot cover this either. It is applied per assembly, so it says + nothing about an assembly that was never run, and it has to stay at 1 because the runner + re-applies it to the much smaller pass that --retry-failed-tests starts. + + Each expected project is checked by path rather than by counting them, because a count only + notices projects leaving while none arrive: renaming one project and adding another in the same + change leaves the count where it was, and the renamed project stops being tested silently. + + Adding projects is fine; only losing one that is listed fails. + +.PARAMETER SourcesDirectory + The repository root to search. + +.PARAMETER ManifestPath + The file listing the repository-relative path of every project this leg is expected to run. +#> +#requires -Version 7.0 + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $SourcesDirectory, + + [Parameter(Mandatory = $true)] + [string] $ManifestPath +) + +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $SourcesDirectory)) { + throw "Sources directory '$SourcesDirectory' does not exist." +} + +if (-not (Test-Path -LiteralPath $ManifestPath)) { + throw "The expected project list '$ManifestPath' does not exist, so there is nothing to check the leg's projects against." +} + +$expected = @( + Get-Content -LiteralPath $ManifestPath | + ForEach-Object { $_.Trim() } | + Where-Object { $_ -and -not $_.StartsWith('#') }) + +if ($expected.Count -eq 0) { + throw "The expected project list '$ManifestPath' names no projects, so it would accept a leg that ran nothing." +} + +$root = (Resolve-Path -LiteralPath $SourcesDirectory).Path.TrimEnd([System.IO.Path]::DirectorySeparatorChar) + +# Mirrors the 'projects' pattern of the leg's test task: any .csproj in a directory whose name +# ends in UnitTests. +$matched = @( + Get-ChildItem -LiteralPath $root -Recurse -Filter '*.csproj' -File | + Where-Object { $_.Directory.Name -like '*UnitTests' } | + ForEach-Object { $_.FullName.Substring($root.Length + 1).Replace([System.IO.Path]::DirectorySeparatorChar, '/') } | + Sort-Object) + +# Paths are compared without regard to case because the repository is developed on Windows and +# built on Linux, and a case-only difference is not a project leaving the leg. +$comparer = [System.StringComparer]::OrdinalIgnoreCase +$matchedSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$matched, $comparer) +$expectedSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$expected, $comparer) + +Write-Host "Matched $($matched.Count) unit test project(s) against $($expected.Count) expected:" +foreach ($project in $matched) { + Write-Host " $project" +} + +$missing = @($expected | Where-Object { -not $matchedSet.Contains($_) }) + +if ($missing.Count -gt 0) { + $detail = ($missing | ForEach-Object { " $_" }) -join [Environment]::NewLine + throw @" +These projects are expected to run in the unit test leg but no longer match its project glob: +$detail + +The leg runs whatever the glob matches, so it would have run without them and reported success. If +a project was renamed or moved on purpose, update '$ManifestPath' to say so. +"@ +} + +$unlisted = @($matched | Where-Object { -not $expectedSet.Contains($_) }) + +if ($unlisted.Count -gt 0) { + # An unlisted project still runs. Naming it here is what keeps the list from drifting so far + # behind that a later loss is hard to tell from a rename. + Write-Host "These projects run in this leg but are not listed in '$ManifestPath':" + foreach ($project in $unlisted) { + Write-Host " $project" + } +} diff --git a/build/jobs/scripts/UnitTestProjects.txt b/build/jobs/scripts/UnitTestProjects.txt new file mode 100644 index 0000000000..b98dbac05a --- /dev/null +++ b/build/jobs/scripts/UnitTestProjects.txt @@ -0,0 +1,30 @@ +# The unit test leg runs whatever '**/*UnitTests/*.csproj' matches, so a project that leaves the +# glob - renamed, moved, or its directory suffix quietly changed - stops being tested without +# anything failing. This is the list of projects that leg is expected to run, checked against what +# the glob actually matches before the tests start. +# +# Paths are relative to the repository root and use forward slashes. Adding a project here is not +# required for it to run, but a project listed here that no longer matches fails the leg. +# +# Microsoft.Health.Fhir.Api.UnitTests holds a project named for R4; the directory and the project +# names differ, which is why it does not look like the rest. +src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj +src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj +src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj +src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj +src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj +src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj +src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj +src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj +src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj +src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj +src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj +src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj +src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj +src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj +src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs new file mode 100644 index 0000000000..9abb3a9860 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle; +using Xunit; + +// Exercise the same custom framework the real test projects use, so these assets +// cover the discoverer and executor shims and not just the retry test case. +[assembly: TestFramework(typeof(CustomXunitTestFramework))] + +// Nine test assemblies declare an assembly fixture that no test class asks for and rely on its +// constructor alone to install the FHIR model info provider. Nothing they contain asserts that the +// fixture ran, so if the framework stopped creating unrequested fixtures those assemblies would fail +// in whatever way a missing provider happens to look like. Declaring one here the same way makes +// that assumption something a test can see. +[assembly: AssemblyFixture(typeof(RecordingAssemblyFixture))] diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Directory.Build.props b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Directory.Build.props new file mode 100644 index 0000000000..6f4b5a0b30 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Directory.Build.props @@ -0,0 +1,15 @@ + + + + + false + + + + + diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj new file mode 100644 index 0000000000..c93863a50d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj @@ -0,0 +1,39 @@ + + + + + Exe + true + false + $(NoWarn);xUnit1051 + + + + + + + + + + + + + + + diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs new file mode 100644 index 0000000000..7339ee2e5d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs @@ -0,0 +1,99 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ArgumentSetOverride +{ + /// + /// A class whose methods narrow one dimension of the class-level fixture argument set while + /// inheriting the other. A method-level declaration replaces the class-level values dimension by + /// dimension rather than wholesale: a dimension naming at least one flag overrides the class's, + /// and a dimension naming none falls back to it. Both halves of that merge decide which variants + /// of a test exist at all, so a mistake in either silently drops tests from a run that still + /// reports success. + /// + [TwoDimensionArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos, AssetOtherDimension.Some)] + public class ArgumentSetOverrideTests : IClassFixture + { + private readonly OverrideFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public ArgumentSetOverrideTests(OverrideFixture fixture) + { + _fixture = fixture; + } + + /// + /// Declaring nothing of its own leaves this method with the class's values in both + /// dimensions. + /// + [Fact] + public void InheritsBothDimensions() + { + AssertFixtureMatchesDisplayName(); + } + + /// + /// The first dimension names a flag, so it replaces the class's two values with one; the + /// second names none, so it keeps the class's. Reading the method's declaration as a whole + /// rather than dimension by dimension would leave this method with no second dimension and + /// so no variants at all. + /// + [Fact] + [TwoDimensionArgumentSets(AssetDataStore.Cosmos, AssetOtherDimension.None)] + public void OverridesTheFirstDimensionOnly() + { + AssertFixtureMatchesDisplayName(); + } + + /// + /// The mirror image: the first dimension names no flag and falls back to the class's two + /// values, while the second names one of its own. The fallback has to work at any position, + /// not only the last. + /// + [Fact] + [TwoDimensionArgumentSets((AssetDataStore)0, AssetOtherDimension.Some)] + public void OverridesTheSecondDimensionOnly() + { + AssertFixtureMatchesDisplayName(); + } + + /// + /// Declares fewer dimensions than the class, by carrying an attribute type that takes fewer + /// of them. The dimensions the method says nothing about are the class's, exactly as if it + /// had named them and left them empty - dropping them instead would leave these tests + /// running without the values, and so without the traits, that a CI leg selects them by. + /// + [Fact] + [AssetArgumentSets(AssetDataStore.Cosmos)] + public void DeclaresFewerDimensionsThanTheClass() + { + AssertFixtureMatchesDisplayName(); + } + + /// + /// Ties the values the fixture was built from to the name the variant reports under. + /// Asserting only that the fixture holds some valid value would hold for every variant + /// alike, so a merge that produced the right number of variants from the wrong values - + /// or built every fixture from the same one - would pass unnoticed. + /// + private void AssertFixtureMatchesDisplayName() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.EndsWith( + $" ({_fixture.DataStore}, {_fixture.OtherDimension})", + displayName, + StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/OverrideFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/OverrideFixture.cs new file mode 100644 index 0000000000..3bca4fff1f --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/OverrideFixture.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ArgumentSetOverride +{ + /// + /// A class fixture built from both fixture argument set dimensions, so a test can tell which + /// value of each dimension it actually received rather than only which name it ran under. + /// + public class OverrideFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The value of the first dimension this variant was constructed for. + /// The value of the second dimension this variant was constructed for. + public OverrideFixture(AssetDataStore dataStore, AssetOtherDimension otherDimension) + { + DataStore = dataStore; + OtherDimension = otherDimension; + } + + /// + /// Gets the value of the first dimension this variant was constructed for. + /// + public AssetDataStore DataStore { get; } + + /// + /// Gets the value of the second dimension this variant was constructed for. + /// + public AssetOtherDimension OtherDimension { get; } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureLifecycleTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureLifecycleTests.cs new file mode 100644 index 0000000000..a13e9f991d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureLifecycleTests.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle +{ + /// + /// Observes whether an assembly fixture that nothing asks for was still constructed. + /// + public class AssemblyFixtureLifecycleTests + { + /// + /// The fixture is declared on the assembly and requested by no constructor anywhere, which is + /// exactly how the real test assemblies use one. + /// + [Fact] + public void GivenAnAssemblyFixtureNothingAsksFor_WhenATestRuns_ThenTheFixtureWasAlreadyConstructed() + { + Assert.True( + AssemblyFixtureProbe.Constructed, + "The assembly fixture had not been constructed by the time a test ran, so an assembly fixture used only for what its constructor does would have had no effect."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureProbe.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureProbe.cs new file mode 100644 index 0000000000..8c13cb896a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureProbe.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle +{ + /// + /// Records that the assembly fixture was constructed, so a test can tell whether that happened + /// before it ran. + /// + public static class AssemblyFixtureProbe + { + private static volatile bool _constructed; + + /// + /// Gets or sets a value indicating whether the assembly fixture has been constructed. + /// + public static bool Constructed + { + get => _constructed; + set => _constructed = value; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/RecordingAssemblyFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/RecordingAssemblyFixture.cs new file mode 100644 index 0000000000..b33f302630 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/RecordingAssemblyFixture.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle +{ + /// + /// An assembly fixture whose only effect is what its constructor does, which is how the real test + /// assemblies use one: no test class asks for it, so a framework that created assembly fixtures + /// only on demand would never construct it and the effect would simply not happen. + /// + public sealed class RecordingAssemblyFixture + { + /// + /// Initializes a new instance of the class. + /// + public RecordingAssemblyFixture() + { + AssemblyFixtureProbe.Constructed = true; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismFourTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismFourTests.cs new file mode 100644 index 0000000000..cea28659d9 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismFourTests.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The fourth of four classes, and so the fourth of four collections, in this scenario. + /// + public class BoundedParallelismFourTests : BoundedParallelismTestBase + { + /// + /// Runs the shared body that observes how many tests are running at once. + /// + /// A task that completes when the test has finished. + [Fact] + public Task Runs() => RunBody(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismOneTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismOneTests.cs new file mode 100644 index 0000000000..c53b69afc5 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismOneTests.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The first of four classes, and so the first of four collections, in this scenario. + /// + public class BoundedParallelismOneTests : BoundedParallelismTestBase + { + /// + /// Runs the shared body that observes how many tests are running at once. + /// + /// A task that completes when the test has finished. + [Fact] + public Task Runs() => RunBody(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTestBase.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTestBase.cs new file mode 100644 index 0000000000..223f70b137 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTestBase.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The body every class in this scenario runs, so that each class contributes one test that + /// overlaps with the others for long enough to be observed. + /// + /// + /// Each class is its own collection, and collections are the unit of parallelization, so running + /// this scenario with a thread limit of one must serialize them. Asserting on the count taken as + /// a test enters, rather than on a maximum read at the end, keeps the check free of races: if the + /// limit is not applied the classes start together and all but one see a count above the limit. + /// + public abstract class BoundedParallelismTestBase + { + /// + /// Occupies the thread limit for long enough that any other class allowed to start would be + /// seen doing so. + /// + /// A task that completes when the body has finished. + protected static async Task RunBody() + { + int concurrent = ConcurrencyTracker.Enter(); + + try + { + Assert.True( + concurrent <= 1, + $"{concurrent} tests ran at once under a thread limit of one, so collection parallelism is not being bounded."); + + await Task.Delay(TimeSpan.FromMilliseconds(300)); + } + finally + { + ConcurrencyTracker.Exit(); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismThreeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismThreeTests.cs new file mode 100644 index 0000000000..e0b9083bc4 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismThreeTests.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The third of four classes, and so the third of four collections, in this scenario. + /// + public class BoundedParallelismThreeTests : BoundedParallelismTestBase + { + /// + /// Runs the shared body that observes how many tests are running at once. + /// + /// A task that completes when the test has finished. + [Fact] + public Task Runs() => RunBody(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTwoTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTwoTests.cs new file mode 100644 index 0000000000..1b3e210422 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTwoTests.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The second of four classes, and so the second of four collections, in this scenario. + /// + public class BoundedParallelismTwoTests : BoundedParallelismTestBase + { + /// + /// Runs the shared body that observes how many tests are running at once. + /// + /// A task that completes when the test has finished. + [Fact] + public Task Runs() => RunBody(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/ConcurrencyTracker.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/ConcurrencyTracker.cs new file mode 100644 index 0000000000..4692e7d7a3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/ConcurrencyTracker.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// Counts how many tests of this scenario are inside their body at the same time. + /// + public static class ConcurrencyTracker + { + private static int _current; + + /// + /// Records entry into a test body. + /// + /// The number of tests inside a body once this one has entered. + public static int Enter() => Interlocked.Increment(ref _current); + + /// + /// Records that a test body has been left. + /// + public static void Exit() => Interlocked.Decrement(ref _current); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeArgumentSetsAttribute.cs new file mode 100644 index 0000000000..8fe3869992 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeArgumentSetsAttribute.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancellationLookalikeFault +{ + /// + /// An argument set attribute whose constructor throws an exception shaped like cancellation. + /// + /// + /// is what anything that awaits with a timeout throws, so an + /// attribute that reaches out to something while being constructed can raise it without the run + /// having been cancelled at all. Discovery is not handed a cancellation token, so the exception's + /// type is the only thing that distinguishes this from a real Ctrl+C - which is why the type + /// alone must not be allowed to decide. Taken for cancellation, this is rethrown, the class is + /// dropped, and the run ends green with its tests missing. + /// + public sealed class CancellationLookalikeArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the + /// class, which never completes. + /// + /// The data stores the class would have been expanded over. + public CancellationLookalikeArgumentSetsAttribute(AssetDataStore dataStore) + : base(dataStore) + { + throw new TaskCanceledException("This argument set attribute gave up waiting."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeFaultTests.cs new file mode 100644 index 0000000000..951ee7fbb6 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeFaultTests.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancellationLookalikeFault +{ + /// + /// A class whose argument set attribute fails with an exception shaped like cancellation. + /// + /// + /// Nothing cancelled this run, so the failure belongs to the class and has to be reported as one. + /// Reading it as a cancelled run instead would rethrow it and drop the class, and because a + /// cancelled run is not a fault, the run would end green with these tests silently absent. + /// + [CancellationLookalikeArgumentSets(AssetDataStore.Sql)] + public class CancellationLookalikeFaultTests + { + /// + /// Never runs: the class carries an argument set attribute that cannot be constructed. + /// + [Fact] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs new file mode 100644 index 0000000000..3001899c56 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt +{ + /// + /// Fails partway through the retrying test's second attempt, tripping --stop-on-fail so the + /// run is cancelled while is still executing. + /// + [Collection("CancellationTrigger")] + public class CancellationTriggerTests + { + /// + /// Deliberately fails to trigger cancellation of the whole run. + /// + [Fact] + public async Task FailsToTriggerCancellation() + { + // Waiting for the sibling collection to announce its second attempt is what makes this + // failure land while that attempt is still running. A sleep here would only be a guess + // at how long the sibling takes to get there. The wait is asynchronous so that it does + // not hold a worker thread the sibling collection may need in order to get there at all. + await Task.WhenAny( + PassingAttemptHandshake.SecondAttemptRunning, + Task.Delay(PassingAttemptHandshake.Budget)); + + Assert.Fail("ASSET: deliberate failure that cancels the run"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs new file mode 100644 index 0000000000..7116c4a038 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt +{ + /// + /// An attempt that runs to completion while cancellation is already requested has reported + /// its own result, so an earlier attempt's deferred failure must not also be reported. + /// + /// + /// Run this namespace with --stop-on-fail on. When cancellation alone was treated as "the + /// attempt reported nothing", the earlier failure was replayed on top of the pass, + /// producing a second orphaned result with no display name and inflating the run total. + /// + [Collection("PassesWhileCancelling")] + public class PassesWhileCancellingTests + { + private static int _attempts; + + /// + /// Fails on the first attempt, then stays alive long enough on the second attempt for + /// the sibling collection to cancel the run, and passes anyway. + /// + [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = 50, RetryOnAssertionFailure = true)] + public async Task PassingAttemptIsNotOverriddenByEarlierFailure() + { + if (Interlocked.Increment(ref _attempts) == 1) + { + Assert.Fail("ASSET: first attempt fails so a failure is deferred"); + } + + // Tell the sibling collection it may now cancel the run. Until this point cancelling + // would land before the attempt that has to survive it had started. + PassingAttemptHandshake.AnnounceSecondAttempt(); + + // Stay in the second attempt until the sibling collection actually cancels the run, then + // prove that is really what happened. Without this the attempt simply passes on its own + // and the scenario would report the expected result while exercising nothing. Waiting for + // the cancellation rather than sleeping for a fixed period keeps that true on a machine + // slow enough that a fixed window would close first. + bool cancelled = await PassingAttemptHandshake.WaitForCancellationAsync(TestContext.Current.CancellationToken); + + Assert.True( + cancelled, + "ASSET: the run was not cancelled while this attempt was running, so this scenario did not exercise a pass during cancellation."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassingAttemptHandshake.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassingAttemptHandshake.cs new file mode 100644 index 0000000000..a88187d6d9 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassingAttemptHandshake.cs @@ -0,0 +1,72 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt +{ + /// + /// Orders the two collections in this scenario against each other. + /// + /// + /// + /// The scenario needs one collection to cancel the run while the other is part way through a + /// retry attempt. Arranging that with sleeps makes the outcome depend on how fast the machine + /// is: on a loaded agent the cancelling collection's failure can land after the other + /// collection's window has closed, and the scenario then fails with a message about itself + /// rather than reporting anything about the behaviour under test. Waiting for an announcement + /// orders the two by construction instead. + /// + /// + /// Both waits here are asynchronous, which is not a matter of taste. The runner executes + /// collections on a bounded set of worker threads, so a collection that blocks a thread while + /// waiting for its sibling can leave the sibling with no thread to run on. That was measured: + /// with a blocking wait the collection being waited for did not start until after the waiter + /// had already given up, which is the same failure this handshake exists to remove. + /// + /// + internal static class PassingAttemptHandshake + { + /// + /// Bounds every wait so that a scenario which has stopped interleaving fails with its own + /// explanation, rather than hanging until the runner's timeout reports nothing useful. + /// + internal static readonly TimeSpan Budget = TimeSpan.FromSeconds(30); + + private static readonly TaskCompletionSource SecondAttemptRunningSource = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Gets a task that completes once the retrying test's second attempt is running, which is + /// the point from which cancelling the run exercises what this scenario exists to check. + /// + internal static Task SecondAttemptRunning => SecondAttemptRunningSource.Task; + + /// + /// Announces that the retrying test's second attempt is running. + /// + internal static void AnnounceSecondAttempt() => SecondAttemptRunningSource.TrySetResult(); + + /// + /// Waits for the run to be cancelled, giving up after . + /// + /// The run's cancellation token. + /// true if the run was cancelled; false if the budget expired first. + internal static async Task WaitForCancellationAsync(CancellationToken cancellationToken) + { + try + { + await Task.Delay(Budget, cancellationToken); + return false; + } + catch (OperationCanceledException) + { + return true; + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs new file mode 100644 index 0000000000..30864941b3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs @@ -0,0 +1,37 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay +{ + /// + /// Fails shortly after the run starts so that --stop-on-fail cancels the run while + /// is still waiting to retry. + /// + [Collection("CancellationTrigger")] + public class CancellationTriggerTests + { + /// + /// Deliberately fails to trigger cancellation of the whole run. + /// + [Fact] + public async Task FailsToTriggerCancellation() + { + // Wait to be told the sibling has failed an attempt, so this failure cannot land before + // there is a retry delay to cancel during. The wait is asynchronous so it does not hold + // a worker thread the sibling collection may need. + await Task.WhenAny(RetryDelayHandshake.AttemptFailing, Task.Delay(RetryDelayHandshake.Budget)); + + // A short settle on top of that ordering, so the sibling is inside its 30s delay rather + // than on the boundary of entering it. This is a margin, not a race: the ordering above + // already happened. + await Task.Delay(700); + + Assert.Fail("ASSET: deliberate failure that cancels the run"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs new file mode 100644 index 0000000000..2fe41a867b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay +{ + /// + /// A retrying test whose failure is deferred must still be reported when the run is + /// cancelled before a later attempt can supersede it. + /// + /// + /// Run this namespace with --stop-on-fail on, so the sibling collection cancels the run + /// while this test is waiting to retry. Before the deferred failure was carried across + /// attempts, this test vanished from the results entirely -- neither passed, failed nor + /// skipped -- and the run reported success. + /// + [Collection("DeferredFailure")] + public class DeferredFailureTests + { + /// + /// Fails on every attempt. The long delay guarantees the run is cancelled while this + /// test is waiting to retry, so the deferred first-attempt failure is the only record + /// that it ever ran. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 30000, RetryOnAssertionFailure = true)] + public void FailureSurvivesCancellationDuringRetryDelay() + { + // Tell the sibling collection it may now cancel the run: the retry delay this scenario + // needs it cancelled during begins the moment this attempt fails. + RetryDelayHandshake.AnnounceAttemptFailing(); + + Assert.Fail("ASSET: failure that must survive cancellation"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/RetryDelayHandshake.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/RetryDelayHandshake.cs new file mode 100644 index 0000000000..d13dd7a074 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/RetryDelayHandshake.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay +{ + /// + /// Orders the two collections in this scenario against each other. + /// + /// + /// + /// The scenario needs one collection to cancel the run while the other is waiting to retry. + /// Deciding when to cancel by sleeping makes that depend on how fast the machine is, so the + /// cancelling collection waits to be told the retrying test has failed an attempt instead. + /// + /// + /// The wait is asynchronous so that it does not hold one of the runner's bounded worker + /// threads. A blocking wait here can leave the collection being waited for with no thread to + /// run on, which turns the handshake into the stall it was meant to prevent. + /// + /// + internal static class RetryDelayHandshake + { + /// + /// Bounds the wait so that a scenario which has stopped interleaving fails with its own + /// explanation, rather than hanging until the runner's timeout reports nothing useful. + /// + internal static readonly TimeSpan Budget = TimeSpan.FromSeconds(30); + + private static readonly TaskCompletionSource AttemptFailingSource = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Gets a task that completes as the retrying test fails an attempt, immediately before the + /// retry delay this scenario needs the run to be cancelled during. + /// + internal static Task AttemptFailing => AttemptFailingSource.Task; + + /// + /// Announces that the retrying test is failing an attempt. + /// + internal static void AnnounceAttemptFailing() => AttemptFailingSource.TrySetResult(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs new file mode 100644 index 0000000000..cc75bd49d5 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs @@ -0,0 +1,50 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ClassAttributeFault +{ + /// + /// A class whose own fixture argument set declaration cannot be read, so the fault belongs to the + /// class rather than to any one of its methods. + /// + /// + /// The class carries two different fixture argument set attributes, so asking it for the single + /// one it declares throws before the walk over its methods begins. That is what makes this a + /// class-level fault: every method is lost at once, so each one gets its own failure standing in + /// for it. The methods declare different traits so that failures carrying only their own + /// declarations can be told apart from failures carrying every method's pooled together. + /// + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public class ClassAttributeFaultTests + { + /// + /// Never runs. Being declared first, it is the method whose ordinary traits a pooled failure + /// would most plausibly have borrowed, which is what the assertions look for. + /// + [Fact] + [Trait("Category", "DeclaredFirst")] + public void FirstNeverRuns() + { + Assert.True(true); + } + + /// + /// Never runs, and declares a trait no earlier method does, together with an argument set of + /// its own, so a leg selecting on both at once only sees the failure if it carries both. + /// + [Fact] + [Trait("Category", "DeclaredLast")] + [AssetArgumentSets(AssetDataStore.Cosmos)] + public void SecondNeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackCollection.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackCollection.cs new file mode 100644 index 0000000000..a8c099c8cf --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackCollection.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraitFallback +{ + /// + /// A collection definition that declares a trait its members inherit, and declares it nowhere else. + /// + /// + /// Xunit v3 gives every member of a collection the traits its definition carries, which is how a + /// class can end up filtered on a trait that appears nowhere in its own source. That propagation + /// is the whole reason this scenario exists: it makes the collection the only place the trait can + /// be read from. + /// + [CollectionDefinition("CollectionTraitFallbackProbe")] + [Trait("Category", "CollectionTraitFallbackProbe")] + public class CollectionTraitFallbackCollection + { + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackTests.cs new file mode 100644 index 0000000000..0f72dbe41f --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackTests.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraitFallback +{ + /// + /// A class whose discovery faults, whose traits cannot be read in one go, and whose only trait + /// comes from the collection it joined. Expected: 1 result, failed, carrying that trait. + /// + /// + /// The throwing trait attribute forces the failure's traits to be gathered one attribute at a + /// time. Gathering them from the class and the method alone would silently drop the collection's + /// trait, because the class does not declare it - and the export and E2E legs select positively, + /// so a leg naming that trait would match nothing and report success with the method missing. + /// That is the same silence reporting discovery faults exists to break, so the fallback has to be + /// no quieter than the ordinary read it stands in for. + /// + [Collection("CollectionTraitFallbackProbe")] + public class CollectionTraitFallbackTests + { + /// + /// Never runs: it declares two fixture argument set attributes, so which one applies cannot be + /// determined. It also carries a trait attribute that throws. + /// + [Fact] + [ThrowingTrait] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/JoinsTheCollectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/JoinsTheCollectionTests.cs new file mode 100644 index 0000000000..00036f2b21 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/JoinsTheCollectionTests.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits +{ + /// + /// Joins a trait-carrying collection without declaring that trait itself. + /// + [Collection("TraitCarryingCollection")] + public class JoinsTheCollectionTests + { + /// + /// Present or absent under a filter depending on whether the collection's trait reached it. + /// + [Fact] + public void InheritsWhateverTheCollectionCarries() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/StaysOutOfTheCollectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/StaysOutOfTheCollectionTests.cs new file mode 100644 index 0000000000..a9600736c6 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/StaysOutOfTheCollectionTests.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits +{ + /// + /// Stays out of the collection, so a filter that removes the collection's members can be told + /// apart from one that selected nothing at all. + /// + public class StaysOutOfTheCollectionTests + { + /// + /// Selected by any filter that does not name a trait this test carries. + /// + [Fact] + public void CarriesNoCollectionTrait() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/TraitCarryingCollectionDefinition.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/TraitCarryingCollectionDefinition.cs new file mode 100644 index 0000000000..055f4b6364 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/TraitCarryingCollectionDefinition.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits +{ + /// + /// Defines a collection that carries a trait of its own. + /// + /// + /// Whether a collection's traits land on the tests of every class in it decides what several CI + /// legs run. Those legs exclude a category, and a class sharing a collection with an excluded + /// one would then be excluded too - silently, because an exclusion filter reports success for + /// the tests it did not select. This scenario is what says which of the two xunit does. + /// + [CollectionDefinition("TraitCarryingCollection")] + [Trait("Category", "CollectionOwned")] + public class TraitCarryingCollectionDefinition + { + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/VariantsInTheCollectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/VariantsInTheCollectionTests.cs new file mode 100644 index 0000000000..56ed088b13 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/VariantsInTheCollectionTests.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits +{ + /// + /// Joins the trait-carrying collection and is also expanded into one variant per data store, the + /// shape the shipping suite uses wherever a class both selects a data store and joins a + /// collection to be serialised. + /// + /// + /// Each variant has to end up carrying both the trait the collection owns and the data store + /// trait the expansion injects. Losing either one hides the variant from a leg naming it, and + /// the legs that name both are the ones that select positively, so nothing would report the loss. + /// + [Collection("TraitCarryingCollection")] + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class VariantsInTheCollectionTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public VariantsInTheCollectionTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Selected only by a leg naming both the collection's trait and this variant's data store. + /// + [Fact] + public void CarriesBothTheCollectionTraitAndItsDataStore() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/AssetOtherDimension.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/AssetOtherDimension.cs new file mode 100644 index 0000000000..9a81f23b70 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/AssetOtherDimension.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault +{ + /// + /// A second fixture argument set dimension. Only the zero value is used, because a dimension + /// that names no flag is what makes the discoverer fall back to the class-level dimension of + /// the same position -- a position the class does not have. + /// + [Flags] + public enum AssetOtherDimension + { + /// + /// No flags, so this dimension contributes no values of its own. + /// + None = 0, + + /// + /// An arbitrary flag, present so the type is a usable flags enum. + /// + Some = 1, + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs new file mode 100644 index 0000000000..81bd51274a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault +{ + /// + /// A class whose fixture argument sets cannot be expanded, so discovering it throws. + /// + /// + /// The class declares one dimension and the method below declares two, the second of which + /// names no flag. A dimension that names no flag means "use the class-level dimension in this + /// position", and the class has no second position, so the expansion has nothing to inherit and + /// refuses. The misconfiguration itself is not the point: it is simply the cheapest way to make + /// discovery of a real class throw, which is what this scenario is for. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class DiscoveryFaultTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public DiscoveryFaultTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Never runs: discovery of this class throws before any test case is produced. + /// + [Fact] + [TwoDimensionArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos, AssetOtherDimension.None)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/TwoDimensionArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/TwoDimensionArgumentSetsAttribute.cs new file mode 100644 index 0000000000..82e8eec26a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/TwoDimensionArgumentSetsAttribute.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault +{ + /// + /// Declares two fixture argument set dimensions, so a method carrying it declares more + /// dimensions than a class that declares only one. + /// + public sealed class TwoDimensionArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The first dimension. + /// The second dimension. + public TwoDimensionArgumentSetsAttribute(AssetDataStore first, AssetOtherDimension second) + : base(first, second) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionArgumentSetsAttribute.cs new file mode 100644 index 0000000000..5509e6285e --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionArgumentSetsAttribute.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DuplicateDimension +{ + /// + /// Declares two fixture argument set dimensions of the same enum type. + /// + /// + /// This reads as though it asks for every pairing of two data stores - a primary and a + /// secondary, say - which is exactly the kind of thing a test author would reach for. Fixture + /// arguments are matched to a fixture's constructor by their enum type, so the two cannot be + /// told apart, and what would actually run is the first value twice. + /// + public sealed class DuplicateDimensionArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The first dimension. + /// The second dimension, of the same type as the first. + public DuplicateDimensionArgumentSetsAttribute(AssetDataStore first, AssetDataStore second) + : base(first, second) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionTests.cs new file mode 100644 index 0000000000..14a157b4d0 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionTests.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DuplicateDimension +{ + /// + /// A class one of whose methods declares two fixture argument set dimensions of the same enum + /// type. + /// + /// + /// Dimensions are declared by position but bound to the fixture's constructor by type, so two of + /// the same type cannot be told apart: the second is dropped and the first value is used for + /// both. The combinations that were meant to differ would all be the same run, each reported + /// under a name claiming otherwise - a suite reporting coverage it does not have. The discoverer + /// refuses the declaration instead, so the mistake is a red test rather than a false green. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class DuplicateDimensionTests : IClassFixture + { + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public DuplicateDimensionTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// Declares in both of its dimensions, so discovery of this + /// method fails and a failing test case stands in for it. It never runs, which is why it can + /// assert nothing. + /// + [Fact] + [DuplicateDimensionArgumentSets(AssetDataStore.Sql, AssetDataStore.Cosmos)] + public void NeverRuns() + { + throw new InvalidOperationException("This method must never be executed: its discovery is expected to fail."); + } + + /// + /// Declares nothing of its own, so it expands normally. A fault on one method must not cost + /// its siblings their variants. + /// + [Fact] + public void SiblingStillRuns() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.Contains($"({_fixture.DataStore})", displayName, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/EmptyArgumentSet/EmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/EmptyArgumentSet/EmptyArgumentSetTests.cs new file mode 100644 index 0000000000..837aa21069 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/EmptyArgumentSet/EmptyArgumentSetTests.cs @@ -0,0 +1,32 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.EmptyArgumentSet +{ + /// + /// A class declaring a fixture argument set that names no value, so it expands to no variants. + /// + /// + /// The method asks to be expanded over an argument set of zero, which names no flag, so the + /// product of the declared dimensions is empty and the method produces no test cases at all. + /// The class deliberately takes no fixture, so nothing else can fail ahead of the reported case + /// and the message it carries is the one the discoverer wrote. + /// + public class EmptyArgumentSetTests + { + /// + /// Never runs: the argument set it asks for expands to nothing. + /// + [Fact] + [AssetArgumentSets((AssetDataStore)0)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalFlowTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalFlowTests.cs new file mode 100644 index 0000000000..f32b285026 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalFlowTests.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContextFlow +{ + /// + /// Pins what happens to a value the class fixture constructor wrote into the execution context. + /// + /// + /// Under xunit.v2 the custom executor ran test methods in the execution context captured just + /// after each class fixture was built, so this value was readable here. xunit.v3 builds fixtures + /// inside an async method, and an async method's state machine restores the caller's execution + /// context as it returns, so the write is discarded before any runner code can capture it. This + /// test records that loss deliberately: it fails if the flow ever comes back, which would mean + /// the note in FixtureArgumentSetClassRunner and the fixtures written against it are out + /// of date. + /// + public class AsyncLocalFlowTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The fixture whose constructor writes the probe value. + public AsyncLocalFlowTests(AsyncLocalWritingFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the fixture this class was constructed with. + /// + protected AsyncLocalWritingFixture Fixture { get; } + + /// + /// Fails if a value written to the execution context by a class fixture constructor becomes + /// readable from a test method again. + /// + [Fact] + public void DoesNotSeeTheValueWrittenByTheFixtureConstructor() + { + Assert.Null(AsyncLocalProbe.Value.Value); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalProbe.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalProbe.cs new file mode 100644 index 0000000000..71a24a56e0 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalProbe.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContextFlow +{ + /// + /// Carries a value from a class fixture constructor to a test method through the execution + /// context, which is the only channel that can show whether the context the fixture was built + /// in is still the one the tests run in. + /// + public static class AsyncLocalProbe + { + /// + /// The value the fixture constructor writes. + /// + public const string ExpectedValue = "set-in-fixture-constructor"; + + /// + /// The value written by the fixture constructor. An lives in the + /// execution context rather than in a field, so a test reads back what the fixture wrote + /// only if the runner deliberately restores that context. + /// + public static readonly AsyncLocal Value = new AsyncLocal(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs new file mode 100644 index 0000000000..7318d18874 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContextFlow +{ + /// + /// A class fixture that writes to an from its + /// constructor, the way FhirRequestContextAccessor holds its context, so that a fixture + /// built before the test runs decides what the test can read back. + /// + public sealed class AsyncLocalWritingFixture + { + /// + /// Initializes a new instance of the class, writing + /// the probe value into the execution context that is current while the fixture is built. + /// + public AsyncLocalWritingFixture() + { + AsyncLocalProbe.Value.Value = AsyncLocalProbe.ExpectedValue; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FailThenSkipOnRetry/FailThenSkipOnRetryTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FailThenSkipOnRetry/FailThenSkipOnRetryTests.cs new file mode 100644 index 0000000000..8898cecb27 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FailThenSkipOnRetry/FailThenSkipOnRetryTests.cs @@ -0,0 +1,73 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.IO; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FailThenSkipOnRetry +{ + /// + /// Fails on its first attempt and skips on the next one, which is what a test guarded by a + /// run-time condition does when that condition changes between attempts. + /// + /// + /// The platform re-runs failed tests in a new process, so the attempt is counted through a file + /// named by the harness rather than in memory. + /// + public class FailThenSkipOnRetryTests + { + /// The environment variable naming the file that counts attempts. + public const string AttemptFileVariable = "XUNIT_EXT_ASSET_ATTEMPT_FILE"; + + /// + /// Fails once, then skips. + /// + [Fact] + public void FailsOnTheFirstAttemptAndSkipsOnTheNext() + { + string attemptFile = Environment.GetEnvironmentVariable(AttemptFileVariable); + + if (string.IsNullOrEmpty(attemptFile)) + { + throw new InvalidOperationException( + $"ASSET: {AttemptFileVariable} was not set, so this scenario cannot tell which attempt it is on."); + } + + if (File.Exists(attemptFile)) + { + Assert.Skip("ASSET: skipping on a later attempt, the way a run-time condition would."); + } + + File.WriteAllText(attemptFile, "attempted"); + Assert.Fail("ASSET: failing on the first attempt."); + } + + /// + /// Fails once, then passes, so that the retry attempt has a test that actually runs. + /// + [Fact] + public void FailsOnTheFirstAttemptAndPassesOnTheNext() + { + string attemptFile = Environment.GetEnvironmentVariable(AttemptFileVariable); + + if (string.IsNullOrEmpty(attemptFile)) + { + throw new InvalidOperationException( + $"ASSET: {AttemptFileVariable} was not set, so this scenario cannot tell which attempt it is on."); + } + + string companion = attemptFile + ".passing"; + + if (File.Exists(companion)) + { + return; + } + + File.WriteAllText(companion, "attempted"); + Assert.Fail("ASSET: failing on the first attempt."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetArgumentSetsAttribute.cs new file mode 100644 index 0000000000..0c1deb12dc --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetArgumentSetsAttribute.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants +{ + /// + /// Declares the fixture argument sets a test class is expanded over. + /// + public sealed class AssetArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The data stores to expand the test class over. + public AssetArgumentSetsAttribute(AssetDataStore dataStore) + : base(dataStore) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetDataStore.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetDataStore.cs new file mode 100644 index 0000000000..af4cd3842c --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetDataStore.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants +{ + /// + /// The data stores a fixture variant can be built for. Fixture argument sets are required + /// to be flags enums. + /// + [Flags] + public enum AssetDataStore + { + /// + /// SQL Server. + /// + Sql = 1, + + /// + /// Cosmos DB. + /// + Cosmos = 2, + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetFixture.cs new file mode 100644 index 0000000000..a4c9603fb2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetFixture.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants +{ + /// + /// A class fixture whose constructor argument is supplied by the fixture argument set, + /// which is what the custom executor injects. + /// + public class AssetFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The data store this variant was constructed for. + public AssetFixture(AssetDataStore dataStore) + { + DataStore = dataStore; + } + + /// + /// Gets the data store this variant was constructed for. + /// + public AssetDataStore DataStore { get; } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs new file mode 100644 index 0000000000..4aee5eb2dd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs @@ -0,0 +1,46 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants +{ + /// + /// Expanded into one variant per data store. Each variant must report under a distinct + /// display name, otherwise a failure cannot be attributed to a fixture argument set. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class FixtureVariantTests : IClassFixture + { + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public FixtureVariantTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// Passes for every variant, and ties the reported name to the fixture it actually got. + /// Asserting only that the fixture holds some valid store would hold equally for both + /// variants, so a transposed mapping - or the same value injected into both - would go + /// unnoticed while the display names still looked right. + /// + [Fact] + public void EachVariantIsReportedUnderItsOwnName() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.Contains( + $"({_fixture.DataStore})", + displayName, + StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs new file mode 100644 index 0000000000..91ac7c67f7 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MethodAttributeFault +{ + /// + /// A class where reading one method's fixture argument set attribute is itself what fails, and the + /// methods around it are declared normally. + /// + /// + /// The failing method carries two different fixture argument set attributes, so asking it for the + /// single one it declares throws before any expansion is attempted. This is a different failure + /// from a method whose argument set expands to nothing: it happens while reading the attribute + /// rather than while using it, which is the step that used to run for every method of the class at + /// once and so cost the whole class its tests. The class takes no fixture, so nothing can fail + /// ahead of the reported case and the message it carries is the discoverer's own. One sibling + /// declares an argument set of its own, and a different one, so a fault case that borrowed a + /// sibling's values instead of using the failing method's can be told apart from one that did not. + /// + public class MethodAttributeFaultTests + { + /// + /// Runs normally, and is declared before the failing method. + /// + [Fact] + public void RunsBeforeTheFault() + { + Assert.True(true); + } + + /// + /// Never runs: it declares two fixture argument set attributes, so which one applies cannot be + /// determined. + /// + [Fact] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + + /// + /// Runs normally, and is declared after the failing method. It declares an argument set of its + /// own so that the class has argument set values that can still be read once the failing + /// method's cannot. + /// + [Fact] + [AssetArgumentSets(AssetDataStore.Cosmos)] + public void RunsAfterTheFault() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/MismatchedDimensionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/MismatchedDimensionTests.cs new file mode 100644 index 0000000000..4d198048fa --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/MismatchedDimensionTests.cs @@ -0,0 +1,62 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MismatchedDimension +{ + /// + /// A class one of whose methods declares a different enum than the class does in the same + /// fixture argument set dimension. + /// + /// + /// The merge pairs dimensions by position while the executor binds fixture arguments by type, and + /// only convention keeps the two agreeing. Where they disagree the method's variants carry no + /// value for the dimension the class declared, and so none of the traits a CI leg selects by: the + /// leg would run none of this method's tests and still report success. The discoverer refuses the + /// declaration instead, so the mistake is a red test rather than a silent absence. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class MismatchedDimensionTests : IClassFixture + { + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public MismatchedDimensionTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// Declares where the class declares + /// , so discovery of this method fails and a failing test case + /// stands in for it. It never runs, which is why it can assert nothing. + /// + [Fact] + [OtherDimensionFirstArgumentSets(AssetOtherDimension.Some)] + public void NeverRuns() + { + throw new InvalidOperationException("This method must never be executed: its discovery is expected to fail."); + } + + /// + /// Declares nothing of its own, so it expands normally. A fault on one method must not cost + /// its siblings their variants. + /// + [Fact] + public void SiblingStillRuns() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.Contains($"({_fixture.DataStore})", displayName, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/OtherDimensionFirstArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/OtherDimensionFirstArgumentSetsAttribute.cs new file mode 100644 index 0000000000..b85e623027 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/OtherDimensionFirstArgumentSetsAttribute.cs @@ -0,0 +1,30 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MismatchedDimension +{ + /// + /// Declares one fixture argument set dimension, taking a different enum than + /// takes in the same position. + /// + /// + /// A method carrying this under a class carrying that one is the declaration the discoverer has + /// to refuse: legal C#, reading as though it narrows the class's dimension, while in fact + /// replacing it with an unrelated one. + /// + public sealed class OtherDimensionFirstArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The first, and only, dimension. + public OtherDimensionFirstArgumentSetsAttribute(AssetOtherDimension first) + : base(first) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MixedFixtureAttributes/MixedAttributeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MixedFixtureAttributes/MixedAttributeTests.cs new file mode 100644 index 0000000000..70a73db739 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MixedFixtureAttributes/MixedAttributeTests.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MixedFixtureAttributes +{ + /// + /// A class carrying no class-level argument sets, where only some methods declare their own. + /// A class shaped like this bypasses the discoverer's fast path yet has no class-level sets to + /// expand, so its undecorated methods take a passthrough path that hands them to xUnit directly. + /// That path has to preserve ordinary discovery, including expanding a theory's data rows: + /// dropping them would silently shrink the suite rather than fail it. + /// + public class MixedAttributeTests + { + /// + /// Declares its own argument sets, so this method is expanded into one variant per store. + /// Its presence is what keeps the class off the wholly-undecorated fast path. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + [Fact] + public void ExpandedMethod() + { + Assert.True(true); + } + + /// + /// Takes the passthrough path. A plain fact must still be discovered exactly once. + /// + [Fact] + public void PassthroughFact() + { + Assert.True(true); + } + + /// + /// Takes the passthrough path while carrying data rows, which must all be discovered. + /// + /// The row value supplied by the data attribute. + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public void PassthroughTheory(int value) + { + Assert.True(value >= 1, "ASSET: unexpected theory row value."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/OverloadedFault/OverloadedFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/OverloadedFault/OverloadedFaultTests.cs new file mode 100644 index 0000000000..3226110ed6 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/OverloadedFault/OverloadedFaultTests.cs @@ -0,0 +1,60 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.OverloadedFault +{ + /// + /// A class where two overloads of the same test method both fail discovery, so two failures have + /// to be reported against methods that share a name. + /// + /// + /// Each overload carries two different fixture argument set attributes, so asking either for the + /// single one it declares throws and each is reported on its own. Overloads are what make this + /// scenario worth having: anything identifying the reported failure by method name alone gives + /// both failures the same identity, and one of them is then dropped - the method it stood for + /// silently absent from a run that still reports the other. + /// + /// xunit's own analyzer rejects overloaded test method names, which is why no such class exists + /// in this repository and why this is defence rather than a live bug. The analyzer is suppressed + /// here rather than the scenario dropped, because the rule is a warning a consuming project can + /// turn off, and a fault case that loses a method is not something the reader of a green run can + /// see. + /// + /// +#pragma warning disable xUnit1024 + public class OverloadedFaultTests + { + /// + /// Never runs: it declares two fixture argument set attributes. + /// + /// Unused; present only to give this overload a distinct signature. + [Theory] + [InlineData(1)] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns(int value) + { + Assert.True(value > 0); + } + + /// + /// Never runs either, and shares its name with the overload above. + /// + /// Unused; present only to give this overload a distinct signature. + [Theory] + [InlineData("x")] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns(string value) + { + Assert.NotNull(value); + } + } +#pragma warning restore xUnit1024 +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartialDiscoveryFault/PartialDiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartialDiscoveryFault/PartialDiscoveryFaultTests.cs new file mode 100644 index 0000000000..7cf9ec25ca --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartialDiscoveryFault/PartialDiscoveryFaultTests.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartialDiscoveryFault +{ + /// + /// A class where one method's fixture argument set cannot be expanded and the methods around it + /// are declared normally. + /// + /// + /// The failing method is declared between the two healthy ones so that the scenario covers both + /// sides of it: a method discovered before the failure and a method discovered after it. The + /// class takes no fixture, so nothing can fail ahead of the reported case and the message it + /// carries is the discoverer's own. + /// + public class PartialDiscoveryFaultTests + { + /// + /// Runs normally, and is declared before the failing method. + /// + [Fact] + public void RunsBeforeTheFault() + { + Assert.True(true); + } + + /// + /// Never runs: the argument set it asks for names no value, so it expands to nothing. It + /// carries an ordinary trait so that a scenario can check the reported failure keeps it. + /// + [Fact] + [Trait("AssetCategory", "PartialFault")] + [AssetArgumentSets((AssetDataStore)0)] + public void NeverRuns() + { + Assert.True(true); + } + + /// + /// Runs normally, and is declared after the failing method. + /// + [Fact] + public void RunsAfterTheFault() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartiallyEmptyArgumentSet/PartiallyEmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartiallyEmptyArgumentSet/PartiallyEmptyArgumentSetTests.cs new file mode 100644 index 0000000000..e0d47ea5cd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartiallyEmptyArgumentSet/PartiallyEmptyArgumentSetTests.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartiallyEmptyArgumentSet +{ + /// + /// A class declaring a fixture argument set that names a value in one dimension and none in the + /// other, so it expands to no variants. + /// + /// + /// The product of the declared dimensions is empty, exactly as it is when no dimension names + /// anything, so the method produces no test cases and the failure standing in for it is all a + /// leg can see. What differs is that one dimension does name a value, which is enough to keep + /// the other dimension from being widened - so the stand-in carries the dimension that was + /// declared and says nothing about the one that was not. A leg selecting positively on the + /// undeclared dimension, which is how the E2E and export legs select a data store, matches + /// nothing and reports success with the method's tests absent. + /// + /// The class deliberately declares nothing itself. A class-level declaration would contribute + /// its own combinations to the stand-in and hide the hole. + /// + /// + public class PartiallyEmptyArgumentSetTests + { + /// + /// Never runs: one of the two argument sets it asks for expands to nothing, so their product + /// does too. + /// + [Fact] + [TwoDimensionArgumentSets((AssetDataStore)0, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs new file mode 100644 index 0000000000..4355cc84cc --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs @@ -0,0 +1,111 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryOutcomes +{ + /// + /// The full outcome matrix, run without cancellation. + /// Expected: 8 results, 3 failed, 4 passed and 1 skipped, with exactly one result per test. + /// + public class RetryOutcomeTests + { + private static int _flakyAttempts; + private static int _clampedAttempts; + private static int _clampedDelayAttempts; + + /// + /// An assertion failure with retries disabled must be reported once and not retried. + /// This is the case that originally disappeared from the results entirely, because the + /// failure was deferred for a retry that then never happened. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = false)] + public void NonRetriableAssertionFailure_IsReportedOnce() + { + Assert.Fail("ASSET: non-retriable assertion failure"); + } + + /// + /// A test that fails every attempt must be reported as failed once, after the retries + /// are exhausted, rather than losing the final attempt's failure. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + public void ExhaustedRetries_IsReportedFailedOnce() + { + Assert.Fail("ASSET: fails on every attempt"); + } + + /// + /// A test that fails once then succeeds must report a single pass, with the superseded + /// failure discarded rather than also reported. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + public void FlakyThenPasses_IsReportedPassedOnce() + { + if (Interlocked.Increment(ref _flakyAttempts) == 1) + { + Assert.Fail("ASSET: first attempt fails, second succeeds"); + } + } + + /// + /// Non-assertion exceptions are retried regardless of RetryOnAssertionFailure, and must + /// still be reported once the attempts are exhausted. + /// + [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = 10)] + public void NonAssertionException_IsReportedFailedOnce() + { + throw new InvalidOperationException("ASSET: transient-style exception"); + } + + /// + /// A plain passing test must be reported as passed. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void AlwaysPasses_IsReportedPassed() + { + Assert.True(true); + } + + /// + /// A test that skips itself must be reported as skipped, and must not be retried on the way + /// there: a skip is not a failure, so the attempt loop has to stop at it rather than run the + /// test again and report the last attempt as a pass. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void SkippedAtRunTime_IsReportedSkipped() + { + Assert.Skip("ASSET: skipped at run time"); + } + + /// + /// A MaxRetries below one would skip the attempt loop and report nothing, so it is clamped + /// up to one and the test runs exactly once. This test cannot say anything about the delay: + /// clamping MaxRetries to one means no retry, and the delay is only ever reached before one. + /// + [RetryFact(MaxRetries = 0, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + public void ClampedRetryConfiguration_RunsExactlyOnce() + { + Assert.Equal(1, Interlocked.Increment(ref _clampedAttempts)); + } + + /// + /// A negative delay would make Task.Delay throw, which would lose the retry this test needs + /// to pass. Reaching the delay at all takes a failed first attempt, so this fails once and + /// then succeeds: a passing result is the statement that the clamp held. + /// + [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = -500, RetryOnAssertionFailure = true)] + public void ClampedNegativeDelay_StillRetries() + { + if (Interlocked.Increment(ref _clampedDelayAttempts) == 1) + { + Assert.Fail("ASSET: failing so the retry delay is reached"); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs new file mode 100644 index 0000000000..957be64f64 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs @@ -0,0 +1,73 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using Xunit; +using Xunit.Sdk; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryPolicy +{ + /// + /// Tests that tell the two RetryOnAssertionFailure policies apart. + /// Expected: 3 results, 2 failed and 1 passed. + /// + /// + /// A test that fails on every attempt is reported failed whether or not it was retried, and one + /// that fails on none is reported passed either way, so neither pins the policy that decides + /// between them. Both tests here fail once and would pass on a second attempt, so the reported + /// outcome states plainly whether the second attempt was spent. + /// + public class RetryPolicyTests + { + private static int _assertionAttempts; + private static int _exceptionAttempts; + private static int _wrappedAssertionAttempts; + + /// + /// An assertion failure is taken to be deterministic, so with the default policy the + /// remaining attempts are not spent and the recovery never happens: this must be reported + /// failed. Were it retried it would pass, so the outcome distinguishes the two. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = false)] + public void AssertionFailureUnderTheDefaultPolicy_IsNotRetried() + { + if (Interlocked.Increment(ref _assertionAttempts) == 1) + { + Assert.Fail("ASSET: first attempt fails an assertion, a second attempt would pass"); + } + } + + /// + /// A non-assertion exception is the transient kind these attributes exist for, so it is + /// retried even under the default policy: this must be reported passed. Were the policy + /// applied to every failure alike it would be reported failed. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = false)] + public void NonAssertionExceptionUnderTheDefaultPolicy_IsRetried() + { + if (Interlocked.Increment(ref _exceptionAttempts) == 1) + { + throw new InvalidOperationException("ASSET: first attempt throws, a second attempt would pass"); + } + } + + /// + /// An assertion failure that reaches the policy wrapped in an + /// is the same deterministic failure and must be treated as one: reported failed, not retried. + /// Were the wrapper taken at face value it would look like an ordinary exception, be retried, + /// and pass - so the outcome states whether the wrapper was seen through. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = false)] + public void WrappedAssertionFailureUnderTheDefaultPolicy_IsNotRetried() + { + if (Interlocked.Increment(ref _wrappedAssertionAttempts) == 1) + { + throw new AggregateException( + new XunitException("ASSET: first attempt fails a wrapped assertion, a second attempt would pass")); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTheoryOutcomes/RetryTheoryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTheoryOutcomes/RetryTheoryOutcomeTests.cs new file mode 100644 index 0000000000..8e2507ae11 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTheoryOutcomes/RetryTheoryOutcomeTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Concurrent; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryTheoryOutcomes +{ + /// + /// The outcome matrix. Expected: 3 results, one per data row, + /// with the rows that recover reported as passed and the row that never recovers reported as + /// failed. + /// + /// + /// Each row keeps its own attempt count, which only works if every row is discovered as its own + /// test case and wrapped individually. A discoverer that collapsed the rows would either share + /// the counter or lose rows outright, and both show up in the published results. + /// + public class RetryTheoryOutcomeTests + { + private static readonly ConcurrentDictionary Attempts = new ConcurrentDictionary(); + + /// + /// Rows that recover on a later attempt must be reported as passed exactly once, and the row + /// that fails every attempt must be reported as failed exactly once. + /// + /// Identifies the row so each keeps a separate attempt count. + /// Whether this row should stop failing after its first attempt. + [RetryTheory(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + [InlineData(1, true)] + [InlineData(2, false)] + [InlineData(3, true)] + public void FlakyRow(int row, bool recovers) + { + int attempt = Attempts.AddOrUpdate(row, 1, (_, current) => current + 1); + + if (!recovers || attempt == 1) + { + Assert.Fail($"ASSET: row {row} failing on attempt {attempt}"); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTimeouts/RetryTimeoutTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTimeouts/RetryTimeoutTests.cs new file mode 100644 index 0000000000..ef057976c2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTimeouts/RetryTimeoutTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryTimeouts +{ + /// + /// A timeout is the canonical flaky failure, so a retry harness has to treat it as retriable. + /// Expected: 2 results, 1 passed and 1 failed. + /// + public class RetryTimeoutTests + { + private static int _flakyTimeoutAttempts; + + /// + /// A test that exceeds its timeout on the first attempt and completes on the second must be + /// reported as passed. xUnit reports a timeout as Xunit.Sdk.TestTimeoutException, + /// whose name contains "Xunit", so a retry decision made by matching that substring would + /// take it for a deterministic assertion failure and spend none of the remaining attempts. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, Timeout = 500)] + public async Task FlakyTimeout_IsRetriedAndPasses() + { + if (Interlocked.Increment(ref _flakyTimeoutAttempts) == 1) + { + await Task.Delay(5000); + } + } + + /// + /// A test that exceeds its timeout on every attempt must still be reported as failed once, + /// rather than losing the final attempt's failure the way a deferred failure would. + /// + [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = 10, Timeout = 500)] + public async Task AlwaysTimesOut_IsReportedFailedOnce() + { + await Task.Delay(5000); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryVariants/RetryVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryVariants/RetryVariantTests.cs new file mode 100644 index 0000000000..99c421c7b3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryVariants/RetryVariantTests.cs @@ -0,0 +1,117 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Concurrent; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryVariants +{ + /// + /// A class that is both expanded per data store and uses the retrying test attributes, which is + /// the shape most of this repository's integration tests take. + /// + /// + /// Variant expansion and retrying are separate mechanisms that meet here: expansion rewrites the + /// test method and its traits, while the retry discoverers build their own test case type and + /// copy the traits across. Either one can drop what the other added, and the result is a variant + /// that no leg selecting positively on a data store can see - so it never runs and the leg still + /// reports success. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + [Trait("Category", "RetryVariant")] + public class RetryVariantTests : IClassFixture + { + private static readonly ConcurrentDictionary Attempts = new ConcurrentDictionary(); + + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public RetryVariantTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// A retrying fact that passes, so that its variants and traits can be asserted on. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void PassingRetryFact() + { + Assert.NotEqual(default, _fixture.DataStore); + } + + /// + /// A retrying fact that fails once and then passes, proving retries still happen inside an + /// expanded variant and that each variant retries on its own count. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + public void FlakyRetryFact() + { + int attempt = Attempts.AddOrUpdate(_fixture.DataStore.ToString(), 1, (_, current) => current + 1); + + if (attempt == 1) + { + Assert.Fail($"ASSET: {_fixture.DataStore} failing on attempt {attempt}"); + } + } + + /// + /// A retrying theory, so that a row of a theory is expanded per data store as well. + /// + /// Identifies the row. + [RetryTheory(MaxRetries = 2, DelayBetweenRetriesMs = 10)] + [InlineData(1)] + [InlineData(2)] + public void RetryTheoryRow(int value) + { + Assert.InRange(value, 1, 2); + } + +#pragma warning disable xUnit1003 // Theory methods must have test data +#pragma warning disable xUnit1026 // Theory methods should use all of their parameters + + /// + /// A retrying theory declaring no data, which xunit reports as an error case rather than a + /// test. It has to reach a leg selecting positively on a data store, carrying the reason it + /// could not run. + /// + /// Never supplied: the theory declares no data. + [RetryTheory] + public void MalformedRetryTheory(int value) + { + Assert.True(true); + } + +#pragma warning restore xUnit1026 +#pragma warning restore xUnit1003 + + /// + /// Supplies no rows at all, so that the theory below reaches xunit's error case through the + /// data attribute rather than through having no data attribute. + /// + /// An empty sequence. + public static TheoryData NoRows() + { + return new TheoryData(); + } + + /// + /// A retrying theory whose data attribute yields no rows, which is the other way xunit ends + /// up reporting a theory it cannot run. It has to reach a leg selecting positively on a data + /// store, carrying the reason it could not run. + /// + /// Never supplied: the data attribute yields no rows. + [RetryTheory] + [MemberData(nameof(NoRows))] + public void EmptyDataRetryTheory(int value) + { + Assert.InRange(value, 1, 2); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/ConcurrencyProbe.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/ConcurrencyProbe.cs new file mode 100644 index 0000000000..797b6dce90 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/ConcurrencyProbe.cs @@ -0,0 +1,50 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants +{ + /// + /// Records whether any two tests that used it were ever inside their bodies at the same time. + /// + internal static class ConcurrencyProbe + { + private static int _active; + private static int _observedOverlap; + + /// + /// Gets a value indicating whether two tests were ever active at once. + /// + internal static bool ObservedOverlap => Volatile.Read(ref _observedOverlap) != 0; + + /// + /// Marks the caller as active for a fixed window, so that a concurrent caller overlaps it. + /// + /// + /// The window has to be long enough that a genuinely parallel run overlaps rather than + /// merely interleaving between the increment and the decrement. Tests in one xUnit + /// collection run one after another, so the window costs a serialized run only its own + /// duration and never makes the assertion flaky in the direction that matters: a serial + /// run cannot observe an overlap however long the window is. + /// + internal static void Occupy() + { + if (Interlocked.Increment(ref _active) > 1) + { + Interlocked.Exchange(ref _observedOverlap, 1); + } + + try + { + Thread.Sleep(250); + } + finally + { + Interlocked.Decrement(ref _active); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs new file mode 100644 index 0000000000..c16581807a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants +{ + /// + /// One of two classes placed in the same collection, each expanded into one variant per data + /// store. The collection is declared by name only, without a [CollectionDefinition] + /// class behind it, which xUnit accepts and treats like any other collection. + /// + /// + /// Classes are put in a collection to stop them running at the same time, usually because they + /// share something external. Expanding a class into variants must not quietly undo that: if the + /// variants land in collections of their own, xUnit is free to run them concurrently and the + /// grouping the author asked for is lost. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + [Collection("SerializedVariants")] + public class FirstSerializedVariantTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public FirstSerializedVariantTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Fails if any other test in this collection was running at the same time. + /// + [Fact] + public void RunsWithoutOverlappingTheOtherVariants() + { + ConcurrencyProbe.Occupy(); + + Assert.False( + ConcurrencyProbe.ObservedOverlap, + "Two tests from the same [Collection] ran concurrently, so expanding a class into fixture argument set variants broke the collection's serialization."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/SecondSerializedVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/SecondSerializedVariantTests.cs new file mode 100644 index 0000000000..f629b0fbb3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/SecondSerializedVariantTests.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants +{ + /// + /// The second class in the same name-only collection as , + /// so that the scenario covers variants of different classes and not only variants of one class. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + [Collection("SerializedVariants")] + public class SecondSerializedVariantTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public SecondSerializedVariantTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Fails if any other test in this collection was running at the same time. + /// + [Fact] + public void RunsWithoutOverlappingTheOtherVariants() + { + ConcurrencyProbe.Occupy(); + + Assert.False( + ConcurrencyProbe.ObservedOverlap, + "Two tests from the same [Collection] ran concurrently, so expanding a class into fixture argument set variants broke the collection's serialization."); + } + + /// + /// Ties the variant's reported name to the fixture it was actually given, so that sharing a + /// collection between variants cannot quietly hand them each other's fixture. + /// + [Fact] + public void ReceivesTheFixtureItsNameClaims() + { + Assert.Contains( + $"({Fixture.DataStore})", + TestContext.Current.Test.TestDisplayName, + System.StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs new file mode 100644 index 0000000000..77cd3362e2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipAfterFailure +{ + /// + /// A retrying test that fails on its first attempt and skips itself on the second. + /// Expected: 1 result, failed. + /// + /// + /// A pass supersedes an earlier failure because it is a claim that the code works - that is what + /// retrying is for. A skip claims nothing: it says the attempt should not have run. Letting it + /// supersede would erase a failure the test really did show, so the skip is discarded instead and + /// the failure stands. + /// + public class SkipAfterFailureTests + { + private static int _attempts; + + /// + /// Throws on the first attempt - a non-assertion exception, so it is retried under the + /// default policy - and skips itself on the second. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void FailsThenSkips() + { + if (Interlocked.Increment(ref _attempts) == 1) + { + throw new InvalidOperationException("ASSET: first attempt fails, the second skips"); + } + + Assert.Skip("ASSET: skipped on the second attempt"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipShims/SkipShimTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipShims/SkipShimTests.cs new file mode 100644 index 0000000000..42f2f5275c --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipShims/SkipShimTests.cs @@ -0,0 +1,87 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipShims +{ + /// + /// Exercises the source-compatibility shims that keep the repository's legacy + /// SkippableFact and Skip.If call sites working on xunit.v3. + /// + /// + /// The shims stand in for a package that no longer exists on v3, so nothing but these scenarios + /// says whether they still do what the several hundred call sites using them assume. The + /// distinction that matters is skipped against failed: a conditional skip reported as a failure + /// would break every leg that skips tests it cannot run, and a skip that silently passed would + /// let a test that never ran count as one that did. + /// + public class SkipShimTests + { + /// + /// A true condition skips, and is reported as skipped rather than as a failure. + /// + [SkippableFact] + public void SkipIfTrue_IsSkipped() + { + Skip.If(true, "the condition held"); + Assert.Fail("Skip.If(true) should have stopped the test before this point."); + } + + /// + /// A false condition does not skip, so the test runs to completion and passes. + /// + [SkippableFact] + public void SkipIfFalse_Runs() + { + Skip.If(false, "the condition did not hold"); + Assert.True(true); + } + + /// + /// is the inverse, so a false condition skips. + /// + [SkippableFact] + public void SkipIfNotFalse_IsSkipped() + { + Skip.IfNot(false, "the condition did not hold"); + Assert.Fail("Skip.IfNot(false) should have stopped the test before this point."); + } + + /// + /// with a true condition does not skip. + /// + [SkippableFact] + public void SkipIfNotTrue_Runs() + { + Skip.IfNot(true, "the condition held"); + Assert.True(true); + } + + /// + /// The reason given at the call site has to reach the report, otherwise a skipped leg gives + /// no indication of why its tests did not run. + /// + [SkippableFact] + public void SkipWithReason_IsSkipped() + { + Skip.If(true, "a distinctive skip reason"); + Assert.Fail("Skip.If(true) should have stopped the test before this point."); + } + + /// + /// The theory shim skips per data row, so the same method both skips and passes. + /// + /// Whether this row should skip. + [SkippableTheory] + [InlineData(true)] + [InlineData(false)] + public void SkippableTheory_SkipsPerRow(bool skip) + { + Skip.If(skip, "the row asked to be skipped"); + Assert.False(skip); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipThenTeardownThrows/SkipThenTeardownThrowsTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipThenTeardownThrows/SkipThenTeardownThrowsTests.cs new file mode 100644 index 0000000000..d4c9f738dd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipThenTeardownThrows/SkipThenTeardownThrowsTests.cs @@ -0,0 +1,56 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipThenTeardownThrows +{ + /// + /// A retrying test that fails, then on its second attempt skips itself and throws from teardown. + /// Expected: 1 result, failed. + /// + /// + /// This is the narrowest corner of the retry state machine. Holding a skip so it cannot erase an + /// earlier failure also holds every message that follows it, teardown's failure among them, which + /// raised the question of whether one attempt could report both. + /// + /// It does not: xUnit collects the skip and the teardown exception into one AggregateException and + /// reports the attempt failed, so no skip is ever published and the abstention-holding path is not + /// entered. The test is therefore retried as the genuine failure it is, and reported failed once. + /// + /// + public class SkipThenTeardownThrowsTests : IDisposable + { + private static int _attempts; + + /// + /// Throws from teardown on every attempt after the first, which is the attempt that skips. + /// + public void Dispose() + { + if (Volatile.Read(ref _attempts) > 1) + { + throw new InvalidOperationException("ASSET: teardown throws after the skip"); + } + } + + /// + /// Throws on the first attempt - a non-assertion exception, so it is retried under the + /// default policy - and skips itself on the second, whose teardown then throws. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void FailsThenSkipsWithThrowingTeardown() + { + if (Interlocked.Increment(ref _attempts) == 1) + { + throw new InvalidOperationException("ASSET: first attempt fails"); + } + + Assert.Skip("ASSET: skipped on the second attempt"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsAttribute.cs new file mode 100644 index 0000000000..274ddc5a06 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsAttribute.cs @@ -0,0 +1,33 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingArgumentSets +{ + /// + /// An argument set attribute whose constructor throws. + /// + /// + /// Reading an attribute runs its constructor, so a class carrying this one cannot be expanded and + /// cannot have its declared values read back the ordinary way either. The values are still in the + /// assembly's metadata, and the failure standing in for the lost tests has to be given them, or a + /// leg selecting positively on a data store would match nothing and report success. + /// + public sealed class ThrowingArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class, which + /// never completes. + /// + /// The data stores the class would have been expanded over. + public ThrowingArgumentSetsAttribute(AssetDataStore dataStore) + : base(dataStore) + { + throw new InvalidOperationException("This argument set attribute cannot be constructed."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsTests.cs new file mode 100644 index 0000000000..898e088d80 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingArgumentSets +{ + /// + /// A class whose argument set attribute cannot be constructed, so none of its tests can be built. + /// + /// + /// The failure standing in for the lost tests still has to carry the data store trait the tests + /// would have carried, which can only come from metadata here. A leg selecting positively on a + /// data store, as the export and E2E legs do, would otherwise see nothing at all and pass. + /// + [ThrowingArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class ThrowingArgumentSetsTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public ThrowingArgumentSetsTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Never runs: the class carries an argument set attribute that cannot be constructed. + /// + [Fact] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingAttributeConstructorTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingAttributeConstructorTests.cs new file mode 100644 index 0000000000..0ac8099886 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingAttributeConstructorTests.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingAttributeConstructor +{ + /// + /// A class whose discovery faults, and which carries an attribute whose constructor throws + /// alongside the trait a CI leg would select on. + /// Expected: 1 result, failed, still carrying the class trait. + /// + /// + /// Two things have to go wrong at once to reach the read this covers. The method carries a trait + /// attribute that throws, which is what makes xUnit's own trait read fail and sends the failure + /// down the per-attribute fallback. The class then carries an attribute whose constructor + /// throws, and because a declaration's attributes are constructed together, that read fails for + /// the class as a whole. If the fallback gives up on the declaration at that point, the class + /// trait goes with it, and a leg selecting positively on Category cannot match the + /// failure standing in for these tests - reporting success with the class missing. + /// + /// The attribute that throws is not a trait attribute and knows nothing about traits. It just + /// happens to sit on the same class, which is what makes this easy to arrive at without meaning + /// to. + /// + /// + [Trait("Category", "ThrowingConstructorProbe")] + [ThrowingConstructor] + public class ThrowingAttributeConstructorTests + { + /// + /// Never runs: it declares two fixture argument set attributes, so which one applies cannot + /// be determined. Its throwing trait attribute is what forces the fallback trait read. + /// + [Fact] + [ThrowingTrait] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingConstructorAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingConstructorAttribute.cs new file mode 100644 index 0000000000..056d30f265 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingConstructorAttribute.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingAttributeConstructor +{ + /// + /// An ordinary attribute - not a trait attribute - whose constructor throws. + /// + /// + /// Reading a declaration's attributes constructs all of them, so this one is enough to make that + /// read fail for the declaration it sits on, taking the sound trait attributes beside it with it. + /// It deliberately is not an ITraitAttribute: the traits are lost because of where the + /// attribute sits, not because of what it is. + /// + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] + public sealed class ThrowingConstructorAttribute : Attribute + { + public ThrowingConstructorAttribute() + { + throw new InvalidOperationException("This attribute's constructor always throws."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitAttribute.cs new file mode 100644 index 0000000000..cf304bd112 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitAttribute.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait +{ + /// + /// A trait attribute that throws when asked for its traits, standing in for one that computes a + /// trait from configuration that is not present. + /// + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = true)] + public sealed class ThrowingTraitAttribute : Attribute, ITraitAttribute + { + /// + /// Throws, as a trait attribute computing its value at discovery time may. + /// + /// Never returns. + public IReadOnlyCollection> GetTraits() + { + throw new InvalidOperationException("ASSET: this trait attribute cannot produce its traits"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitTests.cs new file mode 100644 index 0000000000..b1fa8fcfbc --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitTests.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait +{ + /// + /// A class whose discovery faults on a method that also carries a trait attribute that throws. + /// Expected: 1 result, failed, still carrying the ordinary traits its declaration named. + /// + /// + /// The failure standing in for the lost method has to carry the traits a CI leg selects on, and + /// those are read from the method as a whole. One trait attribute refusing to produce its value + /// must therefore not cost the method the traits its other attributes declared: a leg selecting + /// positively on Category would not match the failure, and would report success with the + /// method missing. + /// + [Trait("Category", "ThrowingTraitProbe")] + public class ThrowingTraitTests + { + /// + /// Never runs: it declares two fixture argument set attributes, so which one applies cannot + /// be determined. It also carries a trait attribute that throws, alongside ordinary ones. + /// + [Fact] + [Trait("Owner", "ThrowingTraitOwner")] + [ThrowingTrait] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs new file mode 100644 index 0000000000..6906a295de --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs @@ -0,0 +1,84 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.VariantTraits +{ + /// + /// A class whose variants must carry the argument set's trait onto every test case discovered + /// from them, including the one xunit builds for a test it cannot run. + /// + /// + /// Every CI leg in this repository selects by trait, so a test case reaching the runner without + /// the data store trait is one no leg selecting a data store can see. That is harmless for a + /// passing test and silent for a broken one: the leg runs neither, and reports success either + /// way. Both a malformed test and a healthy one carrying a trait of its own are declared here, + /// because the two reach the runner by different routes. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class VariantTraitsTests : IClassFixture + { + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public VariantTraitsTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// Declaring a parameter on a fact is a test xunit cannot run, so it reports the method as an + /// error test case rather than discovering it normally. That case is built without traits, + /// which is what makes it worth declaring here: the analyser rule against writing it is + /// suppressed because the malformed declaration is the point. + /// + /// Never supplied: the declaration is deliberately malformed. + [Trait("Category", "ExportLongRunning")] +#pragma warning disable xUnit1001 // Fact methods cannot have parameters + [Fact] + public void MalformedFactIsStillReportedToAFilteringLeg(int value) +#pragma warning restore xUnit1001 + { + throw new InvalidOperationException("This method can never run: xunit reports it as an error instead."); + } + + /// + /// A theory with no data rows is the other way to reach xunit's error case, and it reaches it + /// from a different discoverer than the malformed fact above. Both are declared because a fix + /// that covered only one would leave the other invisible to a filtering leg. + /// + /// Never supplied: no data row is declared. + [Trait("Category", "ExportLongRunning")] +#pragma warning disable xUnit1003 // Theory methods must have test data +#pragma warning disable xUnit1026 // Theory method does not use parameter + [Theory] + public void MalformedTheoryIsStillReportedToAFilteringLeg(int value) +#pragma warning restore xUnit1026 +#pragma warning restore xUnit1003 + { + throw new InvalidOperationException("This method can never run: xunit reports it as an error instead."); + } + + /// + /// Carries a trait of its own alongside the injected one, in the shape the export and E2E legs + /// select by - a data store and a category together. A variant that kept only the injected + /// trait would drop this one and vanish from those legs. + /// + [Trait("Category", "ExportLongRunning")] + [Fact] + public void HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.Contains($"({_fixture.DataStore})", displayName, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs new file mode 100644 index 0000000000..bdd826aae3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs @@ -0,0 +1,98 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers the merge of a method-level fixture argument set declaration with its class's. The + /// merge runs dimension by dimension - a dimension naming at least one flag overrides the + /// class's, one naming none inherits it - and it decides which variants of a test exist at all. + /// Getting it wrong drops tests from a run that still reports success, which is the one failure + /// mode this whole mechanism exists to prevent. + /// + public class ArgumentSetOverrideTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ArgumentSetOverride.ArgumentSetOverrideTests"; + + /// + /// The whole merge in one run: a method declaring nothing keeps both of the class's + /// dimensions, and each method narrowing one dimension keeps the class's values in the other. + /// Asserting the exact set is what makes a variant that quietly stopped being discovered + /// visible - a count alone would still pass if one variant were replaced by another. + /// + [Fact] + public void GivenMethodsNarrowingOneDimension_WhenTheyAreDiscovered_ThenEachInheritsTheClassValuesForTheOther() + { + TestAssetRun run = TestAssetRunner.Run("ArgumentSetOverride"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".InheritsBothDimensions (Sql, Some)"] = "Passed", + [ScenarioClass + ".InheritsBothDimensions (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheFirstDimensionOnly (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Sql, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".DeclaresFewerDimensionsThanTheClass (Cosmos, Some)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + + /// + /// Each variant asserts that the fixture it was handed matches the name it runs under, so a + /// passing run is also evidence the merged values reached the fixture rather than only the + /// display name. Selecting by one dimension's value shows the pairing from the other side: + /// only the variants built for that value may answer, including the ones that inherited it + /// rather than declaring it. + /// + [Fact] + public void GivenMergedArgumentSets_WhenTestsAreSelectedByOneDimension_ThenOnlyTheVariantsBuiltForItRun() + { + TestAssetRun run = TestAssetRunner.Run("ArgumentSetOverride", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".InheritsBothDimensions (Sql, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Sql, Some)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + + /// + /// A method whose attribute takes fewer dimensions than its class's says nothing about the + /// rest, and so inherits them. Selecting by an inherited dimension's trait is the only thing + /// that proves it: a dimension that was dropped instead of inherited takes its trait with + /// it, so the test would still run and pass under an unfiltered run while being invisible to + /// the CI leg that selects on that trait - a green leg with the test silently absent. + /// + [Fact] + public void GivenAMethodDeclaringFewerDimensions_WhenTestsAreSelectedByAnInheritedDimension_ThenItStillRuns() + { + TestAssetRun run = TestAssetRunner.Run("ArgumentSetOverride", filterTrait: "AssetOtherDimension=Some"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".InheritsBothDimensions (Sql, Some)"] = "Passed", + [ScenarioClass + ".InheritsBothDimensions (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheFirstDimensionOnly (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Sql, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".DeclaresFewerDimensionsThanTheClass (Cosmos, Some)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssemblyFixtureLifecycleTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssemblyFixtureLifecycleTests.cs new file mode 100644 index 0000000000..625b46eb6b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssemblyFixtureLifecycleTests.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that the custom framework still creates an assembly fixture that no test asks for. + /// + /// + /// This repository has nine test assemblies whose only use of an assembly fixture is the side + /// effect of its constructor - installing the FHIR model info provider - and none of them names + /// the fixture as a constructor argument anywhere. If unrequested fixtures were created lazily, + /// or not at all, the provider would never be installed and the failure would land somewhere far + /// from the cause. Nothing else in the repository states that expectation, so it is stated here. + /// + public class AssemblyFixtureLifecycleTests + { + /// + /// The fixture is declared on the assets assembly and requested by nothing, so a test seeing + /// it already constructed is the whole of what the nine assemblies depend on. + /// + [Fact] + public void GivenAnAssemblyFixtureNoTestAsksFor_WhenTheRunReachesATest_ThenTheFixtureHasBeenConstructed() + { + TestAssetRun run = TestAssetRunner.Run("AssemblyFixtureLifecycle"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + ["Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle.AssemblyFixtureLifecycleTests.GivenAnAssemblyFixtureNothingAsksFor_WhenATestRuns_ThenTheFixtureWasAlreadyConstructed"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs new file mode 100644 index 0000000000..e789adc0dc --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs @@ -0,0 +1,132 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Xunit; +using Xunit.Sdk; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins the environment variable names the custom executor copies xunit's assertion formatting + /// options into. + /// + /// + /// The executor substitutes its own assembly runner, so it does not go through the base + /// implementation that normally makes this copy and has to make it itself. xunit keeps the names + /// in an internal class that cannot be referenced, so the executor repeats them as literals. These + /// tests read that class back by reflection and compare, so a rename in a future xunit version + /// fails here rather than leaving options such as --print-max-object-depth accepted on the + /// command line and then quietly doing nothing. + /// + public class AssertionFormattingEnvironmentVariableTests + { + /// + /// Checks one option's environment variable name against the one xunit reads it from. + /// + /// The name of the field holding the variable name in xunit. + /// The literal the executor writes the option to. + [Theory] + [InlineData("AssertEquivalentMaxDepth", CustomXunitTestFrameworkExecutor.AssertEquivalentMaxDepthVariable)] + [InlineData("PrintMaxEnumerableLength", CustomXunitTestFrameworkExecutor.PrintMaxEnumerableLengthVariable)] + [InlineData("PrintMaxObjectDepth", CustomXunitTestFrameworkExecutor.PrintMaxObjectDepthVariable)] + [InlineData("PrintMaxObjectMemberCount", CustomXunitTestFrameworkExecutor.PrintMaxObjectMemberCountVariable)] + [InlineData("PrintMaxStringLength", CustomXunitTestFrameworkExecutor.PrintMaxStringLengthVariable)] + public void GivenAnAssertionFormattingOption_WhenItsNameIsComparedToXunits_ThenTheyMatch(string fieldName, string expectedVariableName) + { + Type environmentVariables = typeof(ITestFrameworkExecutionOptions).Assembly + .GetTypes() + .SingleOrDefault(type => type.FullName == "Xunit.Internal.EnvironmentVariables"); + + Assert.NotNull(environmentVariables); + + FieldInfo field = environmentVariables.GetField(fieldName, BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + + Assert.NotNull(field); + Assert.Equal(expectedVariableName, field.GetRawConstantValue()); + } + + /// + /// Pins which option goes into which variable, not just that the names are right. + /// + /// + /// Every one of these options is an , so two of them swapped compiles, sets + /// every variable, and leaves all five names correct - the only symptom is assertion failure + /// messages truncating at the wrong place, which the name comparison above cannot see. Giving + /// each option a distinct value is what makes a swap fail here. + /// + [Fact] + public void GivenEveryAssertionFormattingOption_WhenTheyAreCopiedToTheEnvironment_ThenEachReachesItsOwnVariable() + { + var options = new RecordingExecutionOptions(); + options.SetAssertEquivalentMaxDepth(11); + options.SetPrintMaxEnumerableLength(12); + options.SetPrintMaxObjectDepth(13); + options.SetPrintMaxObjectMemberCount(14); + options.SetPrintMaxStringLength(15); + + Dictionary environment = CustomXunitTestFrameworkExecutor + .BuildAssertionFormattingEnvironment(options) + .ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + + Assert.Equal( + new Dictionary(StringComparer.Ordinal) + { + [CustomXunitTestFrameworkExecutor.AssertEquivalentMaxDepthVariable] = 11, + [CustomXunitTestFrameworkExecutor.PrintMaxEnumerableLengthVariable] = 12, + [CustomXunitTestFrameworkExecutor.PrintMaxObjectDepthVariable] = 13, + [CustomXunitTestFrameworkExecutor.PrintMaxObjectMemberCountVariable] = 14, + [CustomXunitTestFrameworkExecutor.PrintMaxStringLengthVariable] = 15, + }, + environment); + } + + /// + /// An option nobody passed has to stay unset rather than being written as some default. The + /// variables are process-wide and outlive the run that set them, so writing a value xunit + /// would otherwise have chosen for itself would silently override whatever the environment + /// already said. + /// + [Fact] + public void GivenNoAssertionFormattingOptions_WhenTheyAreCopiedToTheEnvironment_ThenNoVariableIsGivenAValue() + { + IReadOnlyList> environment = + CustomXunitTestFrameworkExecutor.BuildAssertionFormattingEnvironment(new RecordingExecutionOptions()); + + // Asserting only that every value is null would hold just as well for a list that named no + // variables at all, which is the one result that would mean the options are never read. + Assert.NotEmpty(environment); + Assert.All(environment, pair => Assert.Null(pair.Value)); + } + + /// + /// The smallest thing that can stand in for the runner's options: xunit's own extension + /// methods supply the option names, so the test states only the pairing and never repeats a + /// name that could drift from the one the runner really uses. + /// + private sealed class RecordingExecutionOptions : ITestFrameworkExecutionOptions + { + private readonly Dictionary _values = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public TValue GetValue(string name) + { + return _values.TryGetValue(name, out object value) ? (TValue)value : default; + } + + public void SetValue(string name, TValue value) + { + _values[name] = value; + } + + public string ToJson() + { + return "{}"; + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/BoundedParallelismTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/BoundedParallelismTests.cs new file mode 100644 index 0000000000..fbff189028 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/BoundedParallelismTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that substituting the collection runner does not remove the limit on how many + /// collections run at once. + /// + public class BoundedParallelismTests + { + private const string ScenarioNamespace = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism."; + + /// + /// xunit bounds collection parallelism inside its own assembly runner context, in the very + /// dispatch this framework replaces in order to substitute its collection runner. Replacing + /// it without carrying the bound over leaves the thread limit unenforced, so every collection + /// in an assembly starts at once however the run was configured. That does not fail anything + /// by itself, which is what makes it worth pinning: it shows up only as a machine under far + /// more load than asked for, and as the flaky timing-sensitive tests that come with it. + /// + [Fact] + public void GivenAThreadLimitOfOne_WhenCollectionsRun_ThenTheyDoNotOverlap() + { + TestAssetRun run = TestAssetRunner.Run("BoundedParallelism", maxThreads: "1"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioNamespace + "BoundedParallelismOneTests.Runs"] = "Passed", + [ScenarioNamespace + "BoundedParallelismTwoTests.Runs"] = "Passed", + [ScenarioNamespace + "BoundedParallelismThreeTests.Runs"] = "Passed", + [ScenarioNamespace + "BoundedParallelismFourTests.Runs"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/CancellationLookalikeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CancellationLookalikeFaultTests.cs new file mode 100644 index 0000000000..df1af07eea --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CancellationLookalikeFaultTests.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a discovery failure that is shaped like cancellation without the run having been + /// cancelled. + /// + /// + /// A cancelled run is rightly not reported as a fault, because a class that happened to be mid + /// expansion when Ctrl+C arrived has nothing wrong with it. That exclusion is only safe while it + /// recognises cancellation by the run actually having been cancelled. Recognising it by the + /// exception's type instead hands every class a way out of being reported: expansion runs the + /// attributes declared on the class, and an attribute that awaits anything with a timeout throws + /// of its own accord. Treated as + /// cancellation, that drops the class and leaves the run green with its tests missing, which is + /// the one outcome reporting discovery faults exists to prevent. + /// + public class CancellationLookalikeFaultTests + { + private const string ErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancellationLookalikeFault.CancellationLookalikeFaultTests.NeverRuns (fixture argument set discovery: Sql)"; + + /// + /// The failure is reported as a test case and the run fails, rather than the class being + /// dropped in the belief that the run was cancelled. + /// + [Fact] + public void GivenAnAttributeThatFailsLikeACancellation_WhenNothingWasCancelled_ThenTheFaultIsReported() + { + TestAssetRun run = TestAssetRunner.Run("CancellationLookalikeFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The cause has to survive into the output, or whoever reads the results is told a test + /// failed without being told the class never expanded. + /// + [Fact] + public void GivenAnAttributeThatFailsLikeACancellation_WhenItIsDiscovered_ThenTheCauseIsWrittenToTheOutput() + { + TestAssetRun run = TestAssetRunner.Run("CancellationLookalikeFault"); + + Assert.Contains("This argument set attribute gave up waiting.", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs new file mode 100644 index 0000000000..54dfd10ee7 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs @@ -0,0 +1,244 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a fault belonging to a whole class is reported so that every leg which would + /// have run any of its tests still sees a failure, and no leg sees one it should not. + /// + /// + /// A class-level fault loses every method at once, which is where a merged failure does the most + /// damage. One failure carrying every method's traits is dropped by a leg excluding any one of + /// them; one carrying only the first method's is invisible to a leg selecting by another's. + /// Either way a leg passes green with tests missing, and nothing in its output says so. The + /// failures are therefore reported one per lost method, per combination that method would have + /// run under. + /// + public class ClassAttributeFaultTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ClassAttributeFault.ClassAttributeFaultTests"; + private const string FirstSqlCase = ScenarioClass + ".FirstNeverRuns (fixture argument set discovery: Sql)"; + private const string FirstSqlSomeCase = ScenarioClass + ".FirstNeverRuns (fixture argument set discovery: Sql, Some)"; + private const string SecondSqlCase = ScenarioClass + ".SecondNeverRuns (fixture argument set discovery: Sql)"; + private const string SecondSqlSomeCase = ScenarioClass + ".SecondNeverRuns (fixture argument set discovery: Sql, Some)"; + private const string SecondCosmosCase = ScenarioClass + ".SecondNeverRuns (fixture argument set discovery: Cosmos)"; + + /// + /// A fault in the class's own declaration loses every method, so every method gets a failure + /// standing in for it, under each combination it would have run. + /// + /// + /// Reporting one failure for the class was the obvious thing and is wrong: a class of many + /// methods that disagree about their traits cannot be represented by one case, because a + /// filter selects or drops that case whole. + /// + [Fact] + public void GivenAClassWhoseOwnAttributeCannotBeRead_WhenItIsDiscovered_ThenEveryLostMethodGetsAFailure() + { + TestAssetRun run = TestAssetRunner.Run("ClassAttributeFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstSqlCase] = "Failed", + [FirstSqlSomeCase] = "Failed", + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + [SecondCosmosCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The class's declaration cannot be read as the single attribute the expansion would have + /// used, but each attribute on its own can be. Answering the failure with no values at all + /// would leave every failure carrying no argument set trait, and a leg selecting by one would + /// pass with the whole class missing. + /// + [Fact] + public void GivenAClassDeclaringTwoArgumentSetAttributes_WhenItIsDiscovered_ThenTheFailuresCarryBothDeclarations() + { + TestAssetRun run = TestAssetRunner.Run("ClassAttributeFault", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstSqlCase] = "Failed", + [FirstSqlSomeCase] = "Failed", + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// This is the shape the repository's own E2E and export legs use: a query selecting on an + /// argument set value and an ordinary trait at once. A failure missing either half matches + /// nothing, so this is the only filter that pins both halves reaching the same case. + /// + [Fact] + public void GivenAClassLevelFault_WhenTestsAreSelectedByACompoundQuery_ThenTheMatchingFailureIsReported() + { + TestAssetRun run = TestAssetRunner.Run( + "ClassAttributeFault", + filterQueryTraits: "(AssetDataStore=Cosmos)&(Category=DeclaredLast)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SecondCosmosCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The other half of the compound query: only the second method declares + /// AssetDataStore.Cosmos, and only the first declares Category=DeclaredFirst, so + /// no test would ever have run under both. A failure matching this query would mean the + /// failures are carrying traits pooled across methods - which is what makes a leg excluding + /// one method's trait drop the failures standing in for all the others. + /// + /// + /// A query selecting nothing would prove nothing if the runner had not understood the query + /// at all. What rules that out is + /// , + /// which selects a failure with this same query shape. Anchoring it again here would mean a + /// second whole-assembly run - query filtering is not scoped to one scenario's namespace - + /// and that added load makes the sibling query runs flaky. + /// + [Fact] + public void GivenAClassLevelFault_WhenAQuerySelectsACombinationNoMethodDeclared_ThenNothingMatches() + { + TestAssetRun run = TestAssetRunner.Run( + "ClassAttributeFault", + filterQueryTraits: "(AssetDataStore=Cosmos)&(Category=DeclaredFirst)"); + + Assert.Empty(run.Results); + } + + /// + /// The form the repository's SQL and Cosmos integration legs use: everything except one data + /// store's tests. Excluding a value one method declares must not drop the failures standing in + /// for the methods that never declared it. + /// + /// + /// This is the exact hole a single per-class failure left. That failure would have carried + /// every method's values, Cosmos among them, so this filter would have dropped it and + /// the leg would have passed green with the entire class missing - the second method's Sql + /// variants included, which that leg was the only one to run. + /// + [Fact] + public void GivenAClassLevelFault_WhenALegExcludesOneDataStore_ThenTheOtherMethodsFailuresSurvive() + { + TestAssetRun run = TestAssetRunner.Run("ClassAttributeFault", filterNotTrait: "AssetDataStore=Cosmos"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstSqlCase] = "Failed", + [FirstSqlSomeCase] = "Failed", + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The same exclusion written as a query rather than as --filter-not-trait. Both + /// spellings are used by this repository's CI - the integration legs exclude a data store with + /// the flag, and the E2E legs pass a query whose Category exclusions take this form. A + /// case carrying no value for the trait at all satisfies != as readily as one carrying a + /// different value, so this also pins that the failures carry their values rather than + /// omitting them. + /// + [Fact] + public void GivenAClassLevelFault_WhenAQueryExcludesOneDataStore_ThenOnlyTheOtherFailuresAreReported() + { + TestAssetRun run = TestAssetRunner.Run( + "ClassAttributeFault", + filterQueryTraits: "(AssetDataStore!=Cosmos)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstSqlCase] = "Failed", + [FirstSqlSomeCase] = "Failed", + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The shape the repository's E2E main leg uses: one argument set value selected positively + /// and a category excluded at the same time. It is the combination that matters - a failure + /// carrying the right data store but the excluded category is dropped, and one carrying + /// neither never matches - so this pins that each half is decided against the same case. + /// + [Fact] + public void GivenAClassLevelFault_WhenALegSelectsOneDataStoreAndExcludesACategory_ThenOnlyTheOtherMethodsFailuresAreReported() + { + TestAssetRun run = TestAssetRunner.Run( + "ClassAttributeFault", + filterQueryTraits: "(AssetDataStore=Sql)&(Category!=DeclaredFirst)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// Two overloads that both fail discovery are two lost methods, so both have to be reported. + /// Identifying a failure by its method name alone gave them one identity between them, and + /// xunit kept a single case - leaving one method's loss invisible. + /// + /// + /// The failures share a display name, because that is built from the method name too, so this + /// counts results rather than naming them. Each overload is reported under both combinations + /// its two declarations expand to, so four failures stand for the two lost methods. + /// + [Fact] + public void GivenTwoOverloadsThatBothFailDiscovery_WhenTheyAreDiscovered_ThenBothAreReported() + { + TestAssetRun run = TestAssetRunner.Run("OverloadedFault"); + + Assert.Equal(0, run.ErrorCount); + Assert.Equal( + 4, + run.Results.Count(result => + string.Equals(result.Outcome, "Failed", StringComparison.Ordinal) && + result.Name != null && + result.Name.Contains("OverloadedFaultTests.NeverRuns", StringComparison.Ordinal))); + + Assert.NotEqual(0, run.ExitCode); + + // The class holds nothing but the two lost overloads, so any passing result would mean a + // fault was reported as a test that ran - which counting only failures cannot see. + Assert.DoesNotContain(run.Results, result => string.Equals(result.Outcome, "Passed", StringComparison.Ordinal)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitFallbackTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitFallbackTests.cs new file mode 100644 index 0000000000..0fb3aaaef9 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitFallbackTests.cs @@ -0,0 +1,68 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a discovery failure whose traits have to be gathered one attribute at a time, and whose + /// only trait comes from the collection its class joined. + /// + /// + /// Xunit v3 gives every member of a collection the traits its definition declares, so a class can + /// be selected on a trait that appears nowhere in its own source - the propagation this PR exists + /// because of. When one trait attribute throws, the failure standing in for the lost method has + /// its traits gathered attribute by attribute instead, and that gathering has to reach as far as + /// the ordinary read it replaces. Reading only the class and the method drops the collection's + /// trait, and a leg selecting positively on it would then match nothing and report success with + /// the method missing. + /// + public class CollectionTraitFallbackTests + { + private const string OneDimensionErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraitFallback.CollectionTraitFallbackTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string TwoDimensionErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraitFallback.CollectionTraitFallbackTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + + /// + /// Unfiltered, the failure is reported and the run fails. + /// + [Fact] + public void GivenAFaultWhoseTraitsMustBeGatheredSeparately_WhenItIsDiscovered_ThenTheFaultIsReported() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraitFallback"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [OneDimensionErrorCaseName] = "Failed", + [TwoDimensionErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The shape that matters. A leg naming the collection's trait still has to see the failure, + /// which it can only do if the fallback read the collection definition as well as the class. + /// + [Fact] + public void GivenAFaultWhoseTraitsMustBeGatheredSeparately_WhenALegSelectsTheCollectionTrait_ThenThatLegStillSeesTheFailure() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraitFallback", filterTrait: "Category=CollectionTraitFallbackProbe"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [OneDimensionErrorCaseName] = "Failed", + [TwoDimensionErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs new file mode 100644 index 0000000000..f972f43e8d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs @@ -0,0 +1,87 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers whether a collection's traits reach the tests of the classes in it. + /// + /// + /// Several CI legs in this repository select by excluding a category, so a trait arriving on a + /// test from somewhere its own class never declared decides whether those legs run it. A class + /// that joins a collection to be serialised against another one would, if collection traits + /// propagate, also inherit that collection's categories and disappear from every leg excluding + /// them - and an exclusion filter reports success for what it did not select, so nothing would + /// say so. This pins the behaviour the test projects are written against. + /// + public class CollectionTraitsTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits.JoinsTheCollectionTests"; + private const string OutsiderClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits.StaysOutOfTheCollectionTests"; + private const string VariantClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits.VariantsInTheCollectionTests"; + + /// + /// Unfiltered, the test in the collection runs. + /// + [Fact] + public void GivenAClassInATraitCarryingCollection_WhenItIsRunUnfiltered_ThenItIsReported() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraits"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".InheritsWhateverTheCollectionCarries"] = "Passed", + [OutsiderClass + ".CarriesNoCollectionTrait"] = "Passed", + [VariantClass + ".CarriesBothTheCollectionTraitAndItsDataStore (Sql)"] = "Passed", + [VariantClass + ".CarriesBothTheCollectionTraitAndItsDataStore (Cosmos)"] = "Passed", + }); + } + + /// + /// The shape that matters: a leg excluding the collection's category. xunit v3 puts a + /// collection's traits on the tests of every class in it, so the member disappears even + /// though its own class never declared that category, while the class outside the collection + /// stays. Nothing in the leg's output would say the member was dropped, which is why any + /// class joining a collection has to be read as carrying that collection's categories too. + /// + [Fact] + public void GivenAClassInATraitCarryingCollection_WhenALegExcludesThatCategory_ThenOnlyTheCollectionsMemberIsDropped() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraits", filterNotTrait: "Category=CollectionOwned"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [OutsiderClass + ".CarriesNoCollectionTrait"] = "Passed", + }); + } + + /// + /// The shape the shipping suite actually uses: a class that both joins a trait-carrying + /// collection and is expanded into one variant per data store. A leg selecting positively on + /// the collection's category and one data store must still find that variant, which it can + /// only do if the injected data store trait is added to the collection's rather than either + /// one replacing the other. + /// + [Fact] + public void GivenVariantsInATraitCarryingCollection_WhenALegSelectsOnBothTraits_ThenTheMatchingVariantIsFound() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraits", filterQueryTraits: "(AssetDataStore=Sql)&(Category=CollectionOwned)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [VariantClass + ".CarriesBothTheCollectionTraitAndItsDataStore (Sql)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs new file mode 100644 index 0000000000..0a166d5c04 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs @@ -0,0 +1,122 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers which exceptions out of fixture argument set expansion are reported as failing test + /// cases and which go back to xunit unchanged. + /// + /// + /// Reporting a discovery fault as a test is what stops a broken expansion from leaving a run + /// green with its tests silently absent, and the scenario assets cover that. The exclusions + /// cannot be reached the same way: cancellation would have to arrive at the moment a class is + /// being expanded, which no scenario can arrange reliably. The decision is therefore made by one + /// function so that every case can be pinned directly, because getting an exclusion wrong is not + /// visible in a passing run - it invents a red test naming a class with nothing wrong with it. + /// + public class DiscoveryFaultDispositionTests + { + /// + /// A cancelled run is not a discovery fault. Without this, pressing Ctrl+C or hitting a + /// runner-imposed timeout would produce a failing test case for whichever class happened to + /// be mid-expansion, and a cancelled run would be indistinguishable from a broken one. + /// + [Fact] + public void GivenACancellation_WhenExpansionEnds_ThenItIsRethrownRatherThanReportedAsATest() + { + Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new OperationCanceledException(), + isCallbackFailure: false, + isCancellationRequested: true)); + } + + /// + /// derives from + /// and is what an awaited task throws when the run is cancelled, so it has to be treated the + /// same way. Matching on the exact type instead would let the common case through. + /// + [Fact] + public void GivenATaskCancellation_WhenExpansionEnds_ThenItIsRethrownRatherThanReportedAsATest() + { + Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new TaskCanceledException(), + isCallbackFailure: false, + isCancellationRequested: true)); + } + + /// + /// A failure raised by xunit's own callback is not a fault in expanding argument sets, and + /// the only way to report it would be to hand that same callback another test case. + /// + [Fact] + public void GivenAFailureFromTheCallback_WhenExpansionEnds_ThenItIsRethrownRatherThanReportedAsATest() + { + Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new InvalidOperationException("the sink threw"), + isCallbackFailure: true, + isCancellationRequested: false)); + } + + /// + /// Anything else belongs to the expansion, and is the case the whole mechanism exists for: + /// reported as a failing test case so it reaches the results and the exit code instead of + /// leaving the run green with the class's tests missing. + /// + [Fact] + public void GivenAnExpansionFailure_WhenExpansionEnds_ThenItIsReportedAsATest() + { + Assert.False(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new InvalidOperationException("the argument set was misdeclared"), + isCallbackFailure: false, + isCancellationRequested: false)); + } + + /// + /// An out of a run nobody cancelled did not come from + /// the runner: expansion reads the attributes declared on the class, and one of those is free + /// to throw it for reasons of its own. Rethrowing it would drop the class and leave the run + /// green with its tests missing, which is what reporting discovery faults exists to prevent. + /// + [Fact] + public void GivenACancellationExceptionButNoCancellation_WhenExpansionEnds_ThenItIsReportedAsATest() + { + Assert.False(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new OperationCanceledException("an attribute gave up waiting"), + isCallbackFailure: false, + isCancellationRequested: false)); + } + + /// + /// The derived type gets the same treatment, because it is the one an attribute that awaits + /// anything with a timeout will actually throw. + /// + [Fact] + public void GivenATaskCancellationButNoCancellation_WhenExpansionEnds_ThenItIsReportedAsATest() + { + Assert.False(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new TaskCanceledException("an attribute gave up waiting"), + isCallbackFailure: false, + isCancellationRequested: false)); + } + + /// + /// Cancellation wins even when it also reached the callback, because there is nothing to + /// report a cancelled run against. + /// + [Fact] + public void GivenACancellationFromTheCallback_WhenExpansionEnds_ThenItIsRethrownRatherThanReportedAsATest() + { + Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new OperationCanceledException(), + isCallbackFailure: true, + isCancellationRequested: true)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs new file mode 100644 index 0000000000..b74d7af99b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs @@ -0,0 +1,173 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a class whose fixture argument sets cannot be expanded is reported as a failure + /// rather than dropped from the run. + /// + public class DiscoveryFaultTests + { + private const string SqlErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string CosmosErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Cosmos)"; + + // The class declares one dimension and the method two, the second of which names nothing. + // The two declarations are closed over separately, so the class's contributes the cases + // above and the method's contributes these - its silent dimension widened to the only value + // its type declares, so the failure can be selected on that dimension as well as the other. + private const string SqlWidenedErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + private const string CosmosWidenedErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Cosmos, Some)"; + + /// + /// xUnit reports an exception thrown out of discovery only as a diagnostic message, which is + /// suppressed unless the run was started with --xunit-diagnostics, and carries on + /// without the class. A run containing any other healthy class therefore still reports + /// success, so a broken expansion is indistinguishable from a class that has no tests. The + /// discoverer turns the fault into a failing test case instead, which puts it in the results + /// and in the exit code. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheFaultIsReportedAsAFailedTest() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SqlErrorCaseName] = "Failed", + [CosmosErrorCaseName] = "Failed", + [SqlWidenedErrorCaseName] = "Failed", + [CosmosWidenedErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The console line the discoverer writes is where the original exception survives in full, + /// including the stack that points at the line which threw. Without it a fault would be + /// reported with no way to find out what it was. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheCauseIsWrittenToTheOutput() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault"); + + Assert.Contains("[FixtureArgumentSets] ERROR", run.Output, StringComparison.Ordinal); + Assert.Contains("fixture argument set dimensions where its class declares", run.Output, StringComparison.Ordinal); + } + + /// + /// The failure the runner reports has to carry the cause itself, not just the console. A + /// fault case stands in for the variants of a class that never got built, and those variants + /// are how the executor knows which argument to hand the class fixture; a case that does not + /// say which one it stands for leaves the executor unable to build the fixture, and the + /// report then shows that fixture failure instead of the discovery error behind it. Anyone + /// reading the results would be sent to look at a fixture that is not the problem. This pins + /// the reported message to the real exception so that cannot come back. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheReportedFailureCarriesTheOriginalException() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault"); + + foreach (string caseName in new[] { SqlErrorCaseName, CosmosErrorCaseName }) + { + string reported = ExtractReportedFailure(run.Output, caseName); + + Assert.Contains("fixture argument set dimensions where its class declares", reported, StringComparison.Ordinal); + Assert.DoesNotContain("had multiple values", reported, StringComparison.Ordinal); + } + } + + /// + /// Reporting the fault as a test is only worth anything if the legs that would have run the + /// class still select it. Some CI legs pick their tests with a positive trait filter, and the + /// variants this class never produced are what would have carried those traits, so a fault + /// case with no traits of its own is filtered out and the leg passes with the class silently + /// missing - which is the failure this whole mechanism exists to prevent, only harder to see. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenTestsAreSelectedByArgumentSetTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SqlErrorCaseName] = "Failed", + [SqlWidenedErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The repository's integration legs select their tests by excluding the other data + /// store, and a trait exclusion drops a case when any of its values under that trait matches. + /// A single fault case declaring every value the class asked for would therefore be dropped by + /// the SQL leg for holding the Cosmos value and by the Cosmos leg for holding the SQL one, so + /// the fault would reach neither and both legs would stay green with the class missing. One + /// case per combination is what stops that: excluding one value leaves the others reported. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenTestsAreSelectedByExcludingAnArgumentSetTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault", filterNotTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [CosmosErrorCaseName] = "Failed", + [CosmosWidenedErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// Reads back what the runner reported for one failed case, which is the block it prints + /// after the case's own failed line and before whatever it prints next. + /// + /// + /// The message has to be taken from the console rather than the TRX report, because the TRX + /// writer records no error information for these cases. Searching the whole output instead + /// would prove nothing: the discoverer's own diagnostic line names the same exception, so an + /// assertion over all of it would pass even with the reported failure saying something else + /// entirely. + /// + /// The full console output of the run. + /// The display name of the case whose failure is wanted. + /// The text the runner printed for that failure. + private static string ExtractReportedFailure(string output, string caseName) + { + const string failedMarker = "failed "; + + int start = output.IndexOf(failedMarker + caseName, StringComparison.Ordinal); + Assert.True(start >= 0, $"The run reported no failure for '{caseName}'.{Environment.NewLine}{output}"); + + start += failedMarker.Length + caseName.Length; + + int next = output.IndexOf(Environment.NewLine + failedMarker, start, StringComparison.Ordinal); + int summary = output.IndexOf("Test run summary", start, StringComparison.Ordinal); + + int end = next >= 0 ? next : output.Length; + if (summary >= 0 && summary < end) + { + end = summary; + } + + return output.Substring(start, end - start); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DuplicateDimensionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DuplicateDimensionTests.cs new file mode 100644 index 0000000000..378d9358dd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DuplicateDimensionTests.cs @@ -0,0 +1,86 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers an argument set attribute declaring two dimensions of the same enum type. + /// + /// + /// Dimensions are declared by position but bound to the fixture's constructor by type, so two of + /// the same type cannot be told apart: the second is dropped and every variant is built with the + /// first value, while still being named - and traited - for the combination it was supposed to + /// be. Every combination would report as having run, none of them would have, and the run would + /// be green. On origin/main the merge used Dictionary.Add and this threw; the + /// rewrite made it a silent overwrite, so the refusal below is what keeps it loud. + /// + public class DuplicateDimensionTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DuplicateDimension.DuplicateDimensionTests"; + + /// + /// The refusal is reported as a failure standing in for the offending method, under each of + /// the combinations it would have claimed, while the method that declared nothing unusual + /// runs normally. + /// + [Fact] + public void GivenAMethodDeclaringTwoDimensionsOfOneType_WhenItIsDiscovered_ThenItIsReportedAndItsSiblingStillRuns() + { + TestAssetRun run = TestAssetRunner.Run("DuplicateDimension"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Cosmos)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql, Cosmos)"] = "Failed", + [ScenarioClass + ".SiblingStillRuns (Sql)"] = "Passed", + [ScenarioClass + ".SiblingStillRuns (Cosmos)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failure names the attribute and the type it repeated, because the declaration that + /// caused it is legal C# and reads as though it asks for every pairing of the two. + /// + [Fact] + public void GivenAMethodDeclaringTwoDimensionsOfOneType_WhenItIsDiscovered_ThenTheFailureNamesTheRepeatedType() + { + TestAssetRun run = TestAssetRunner.Run("DuplicateDimension"); + + Assert.Contains("DuplicateDimensionArgumentSetsAttribute", run.Output, StringComparison.Ordinal); + Assert.Contains("AssetDataStore", run.Output, StringComparison.Ordinal); + } + + /// + /// The leg's own filter shape. A failure the leg cannot see is no better than the silent + /// expansion the refusal replaced, so a leg excluding the other data store has to be left + /// with a failure of its own. The stand-in for the combination naming both stores carries + /// both traits and is excluded with them, which is why the per-value stand-ins matter. + /// + [Fact] + public void GivenAMethodDeclaringTwoDimensionsOfOneType_WhenALegExcludesTheOtherDataStore_ThenTheFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("DuplicateDimension", filterNotTrait: "AssetDataStore=Cosmos"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"] = "Failed", + [ScenarioClass + ".SiblingStillRuns (Sql)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs new file mode 100644 index 0000000000..6c7886305a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs @@ -0,0 +1,80 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a fixture argument set which names no value fails the run rather than quietly + /// removing the tests that asked for it. + /// + public class EmptyArgumentSetTests + { + private const string ErrorCaseSql = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.EmptyArgumentSet.EmptyArgumentSetTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string ErrorCaseCosmos = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.EmptyArgumentSet.EmptyArgumentSetTests.NeverRuns (fixture argument set discovery: Cosmos)"; + + private static Dictionary BothErrorCases() => + new Dictionary + { + [ErrorCaseSql] = "Failed", + [ErrorCaseCosmos] = "Failed", + }; + + /// + /// An argument set naming no flag collapses the product of the declared dimensions to nothing, + /// so the method produces no test cases. Left alone that is indistinguishable from a class with + /// no tests: the run reports success with the tests simply absent, which is the worst way for a + /// test to stop running because nothing in the report says it ever existed. A declared argument + /// set that expands to nothing is always a misconfiguration, so it is reported as a failure. + /// + [Fact] + public void GivenAnArgumentSetThatNamesNoValue_WhenTheClassIsDiscovered_ThenTheRunFails() + { + TestAssetRun run = TestAssetRunner.Run("EmptyArgumentSet"); + + TestAssetRunAssertions.PublishedExactly(run, BothErrorCases()); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The declaration names no value, so there is none to report the failure under - but it still + /// names a type, and the failure is reported once per value that type declares. Without this a + /// leg selecting positively on the argument set would see nothing and report success with the + /// method's tests absent, which is the outcome reporting these failures exists to prevent. + /// + [Fact] + public void GivenAnArgumentSetThatNamesNoValue_WhenALegSelectsOnTheArgumentSet_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("EmptyArgumentSet", filterQueryTraits: "(AssetDataStore=Sql)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ErrorCaseSql] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The class takes no fixture, so nothing is aggregated ahead of the reported case and the + /// message it carries is the discoverer's own. That is what makes this failure actionable: + /// it names the method and says what is wrong with the argument set it declared. + /// + [Fact] + public void GivenAnArgumentSetThatNamesNoValue_WhenTheClassIsDiscovered_ThenTheFailureNamesTheMethod() + { + TestAssetRun run = TestAssetRunner.Run("EmptyArgumentSet"); + + Assert.Contains("EmptyArgumentSetTests.NeverRuns", run.Output, StringComparison.Ordinal); + Assert.Contains("expanded to no fixture argument sets", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ExecutionContextFlowTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ExecutionContextFlowTests.cs new file mode 100644 index 0000000000..74c072929a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ExecutionContextFlowTests.cs @@ -0,0 +1,39 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Records what the custom executor does with the execution context a class fixture is built in. + /// + public class ExecutionContextFlowTests + { + private const string Prefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContextFlow.AsyncLocalFlowTests."; + + /// + /// A value written to an by a class fixture + /// constructor is not readable from the test methods of that class. xunit.v2 preserved that + /// flow and xunit.v3 cannot, because it builds fixtures inside an async method whose state + /// machine restores the caller's execution context on return. The scenario asserts the value + /// is absent, so this run turning red means the flow came back and the notes describing its + /// absence need revisiting. + /// + [Fact] + public void GivenAFixtureThatWritesToTheExecutionContext_WhenItsTestsRun_ThenTheWriteIsNotVisibleToThem() + { + TestAssetRun run = TestAssetRunner.Run("ExecutionContextFlow"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "DoesNotSeeTheValueWrittenByTheFixtureConstructor"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultArgumentSetCombinationTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultArgumentSetCombinationTests.cs new file mode 100644 index 0000000000..6ecd484be3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultArgumentSetCombinationTests.cs @@ -0,0 +1,138 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers which failures a class whose expansion faulted is reported as, given the argument set + /// attributes it declared. + /// + /// + /// The scenario assets exercise this end to end, but only over the one shape this repository + /// happens to use: a single enum dimension whose values every method agrees on. The shapes that + /// are not covered there are the ones where getting this wrong is invisible - a combination that + /// should exist but does not is a CI leg passing green with the class's tests missing, which is + /// precisely what reporting the fault as a test exists to prevent. They are pinned directly here + /// rather than by adding assets no product code resembles. + /// + public class FaultArgumentSetCombinationTests + { + [Flags] + private enum Store + { + Sql = 1, + Cosmos = 2, + } + + [Flags] + private enum Format + { + Json = 1, + Ndjson = 2, + } + + /// + /// A class declaring nothing still has to be reported, or a fault in a class that takes no + /// argument sets would produce no failing test at all. + /// + [Fact] + public void GivenNoArgumentSets_WhenCombinationsAreBuilt_ThenASingleEmptyCombinationIsProduced() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations(Array.Empty()); + + Assert.Empty(Assert.Single(combinations)); + } + + /// + /// Every variant an attribute would have expanded to needs its own failure, so that a leg + /// selecting one variant by trait still sees a failure standing in for it. + /// + [Fact] + public void GivenTwoDimensionsOnOneAttribute_WhenCombinationsAreBuilt_ThenTheirProductIsProduced() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations( + new[] { new[] { Flags(Store.Sql, Store.Cosmos), Flags(Format.Json, Format.Ndjson) } }); + + Assert.Equal( + new[] { "Sql+Json", "Sql+Ndjson", "Cosmos+Json", "Cosmos+Ndjson" }.OrderBy(x => x, StringComparer.Ordinal), + combinations.Select(Describe).OrderBy(x => x, StringComparer.Ordinal)); + } + + /// + /// This is the case that makes each attribute's values its own product rather than one pooled + /// product. Were the two pooled, the only combinations produced would carry a format, and a + /// leg running --filter-not-trait Format=Json alongside one running + /// --filter-not-trait Format=Ndjson would between them exclude every failure the class + /// produced - both legs green, the whole class silently absent. + /// + [Fact] + public void GivenTwoAttributesUsingDifferentDimensions_WhenCombinationsAreBuilt_ThenNeitherBorrowsTheOthersValues() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations( + new[] + { + new[] { Flags(Store.Sql, Store.Cosmos) }, + new[] { Flags(Format.Json) }, + }); + + Assert.Equal( + new[] { "Cosmos", "Json", "Sql" }, + combinations.Select(Describe).OrderBy(x => x, StringComparer.Ordinal)); + } + + /// + /// Methods usually declare the same argument sets as each other, and two failures standing for + /// the same combination would collide on unique ID as well as reporting the fault twice. + /// + [Fact] + public void GivenTwoAttributesDeclaringTheSameValues_WhenCombinationsAreBuilt_ThenTheCombinationIsProducedOnce() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations( + new[] + { + new[] { Flags(Store.Sql, Store.Cosmos) }, + new[] { Flags(Store.Cosmos, Store.Sql) }, + }); + + Assert.Equal( + new[] { "Cosmos", "Sql" }, + combinations.Select(Describe).OrderBy(x => x, StringComparer.Ordinal)); + } + + /// + /// An attribute whose values could not be expanded arrives here as no dimensions at all. It + /// must not take the readable attributes' combinations down with it. + /// + [Fact] + public void GivenAnAttributeWithNoUsableDimensions_WhenCombinationsAreBuilt_ThenTheOtherAttributesAreStillClosed() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations( + new[] + { + new[] { Array.Empty() }, + new[] { Flags(Store.Sql, Store.Cosmos) }, + }); + + Assert.Equal( + new[] { string.Empty, "Cosmos", "Sql" }, + combinations.Select(Describe).OrderBy(x => x, StringComparer.Ordinal)); + } + + private static SingleFlag[] Flags(params Enum[] values) + { + return values.Select(value => new SingleFlag(value)).ToArray(); + } + + private static string Describe(IReadOnlyList combination) + { + return string.Join("+", combination.Select(flag => flag.EnumValue.ToString())); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs new file mode 100644 index 0000000000..31d7ee1641 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs @@ -0,0 +1,211 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Reflection; +using Xunit; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers how a class-level discovery failure chooses the methods it is reported against, and how + /// it tells two of them apart. + /// + /// + /// Both decisions have the same failure mode. A test method that is not recognised as one gets no + /// failure standing in for it, and two methods given the same identity get one failure between + /// them, xunit keeping whichever arrived first. Either way the run reports success with tests + /// missing and nothing in its output saying so, which is what this whole mechanism exists to + /// prevent. Both are asserted directly rather than through a scenario because arranging them in a + /// real test assembly needs an attribute xunit has no discoverer for. + /// + public class FaultMethodSelectionTests + { + /// + /// xunit discovers tests by the interface, not by the + /// class, so an attribute implementing the interface without + /// deriving from the class still marks a test. Recognising only the class would leave such a + /// method out of the methods a class-level failure is reported against. + /// + [Fact] + public void GivenAMethodMarkedByAnAttributeImplementingTheFactInterface_WhenItIsInspected_ThenItCountsAsATest() + { + MethodInfo method = typeof(Subject).GetMethod(nameof(Subject.MarkedByInterfaceOnly)); + + Assert.False(typeof(FactAttribute).IsAssignableFrom(typeof(InterfaceOnlyFactAttribute))); + Assert.True(CustomXunitTestFrameworkDiscoverer.IsTestMethod(method)); + } + + /// + /// A method carrying no fact attribute at all is not a test, so it is not one of the methods a + /// failure is reported against - a failure named after a helper carries none of the traits a + /// CI leg selects the class's tests by. + /// + [Fact] + public void GivenAMethodCarryingNoFactAttribute_WhenItIsInspected_ThenItDoesNotCountAsATest() + { + MethodInfo method = typeof(Subject).GetMethod(nameof(Subject.NotATest)); + + Assert.False(CustomXunitTestFrameworkDiscoverer.IsTestMethod(method)); + } + + /// + /// Generic overloads differing only in arity share a name and both take no parameters, so a key + /// built from the name and parameter types alone is the same for both. The two failures would + /// then share a unique ID and xunit would keep one, losing the other method silently. + /// + [Fact] + public void GivenGenericOverloadsDifferingOnlyInArity_WhenTheirFaultKeysAreBuilt_ThenTheKeysDiffer() + { + MethodInfo one = typeof(Subject).GetMethod(nameof(Subject.GenericOverload), 1, Type.EmptyTypes); + MethodInfo two = typeof(Subject).GetMethod(nameof(Subject.GenericOverload), 2, Type.EmptyTypes); + + Assert.NotEqual( + CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(one), + CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(two)); + } + + /// + /// Ordinary overloads differing only in parameter types must stay distinguishable too, so that + /// widening the key for generic arity did not narrow it anywhere else. + /// + [Fact] + public void GivenOverloadsDifferingOnlyInParameterTypes_WhenTheirFaultKeysAreBuilt_ThenTheKeysDiffer() + { + MethodInfo one = typeof(Subject).GetMethod(nameof(Subject.Overload), new[] { typeof(int) }); + MethodInfo two = typeof(Subject).GetMethod(nameof(Subject.Overload), new[] { typeof(string) }); + + Assert.NotEqual( + CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(one), + CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(two)); + } + + /// + /// A method whose attributes throw while being read is not known to be a test, and must not + /// be reported as known not to be one: the two answers lead to opposite decisions below. + /// + [Fact] + public void GivenAMethodWhoseAttributesCannotBeRead_WhenItIsInspected_ThenItsTestNessIsUnknown() + { + MethodInfo method = typeof(Subject).GetMethod(nameof(Subject.AttributesCannotBeRead)); + + Assert.Null(CustomXunitTestFrameworkDiscoverer.IsTestMethod(method)); + } + + /// + /// The attributes that cannot be read are very often the ones that caused the failure being + /// reported, so a method whose test-ness is unknown is kept. Dropping it would leave a real + /// test with no failure standing in for it, and a leg selecting by a trait only that method + /// carried would pass green with it absent - the presence of a readable sibling is exactly + /// what would hide it, because the fallback never fires. + /// + [Fact] + public void GivenAMethodWhoseAttributesCannotBeReadAlongsideAReadableTest_WhenTheMethodsAreSelected_ThenItIsKept() + { + MethodInfo unreadable = typeof(Subject).GetMethod(nameof(Subject.AttributesCannotBeRead)); + MethodInfo readable = typeof(Subject).GetMethod(nameof(Subject.MarkedByInterfaceOnly)); + + MethodInfo[] selected = CustomXunitTestFrameworkDiscoverer.SelectMethodsToReportAgainst(new[] { readable, unreadable }); + + Assert.Equal(new[] { readable, unreadable }, selected); + } + + /// + /// With nothing that is, or may be, a test there is still a failure to report, so the first + /// method the class declares stands in for it. Reporting nowhere would put the class back to + /// vanishing from a green run. + /// + [Fact] + public void GivenNoMethodIsATest_WhenTheMethodsAreSelected_ThenTheFirstMethodIsUsed() + { + MethodInfo first = typeof(Subject).GetMethod(nameof(Subject.NotATest)); + MethodInfo second = typeof(Subject).GetMethod(nameof(Subject.Overload), new[] { typeof(int) }); + + MethodInfo[] selected = CustomXunitTestFrameworkDiscoverer.SelectMethodsToReportAgainst(new[] { first, second }); + + Assert.Equal(new[] { first }, selected); + } + + /// + /// An attribute that marks a test the way xunit recognises one - by implementing the interface - + /// without deriving from . + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + private sealed class InterfaceOnlyFactAttribute : Attribute, IFactAttribute + { + public string DisplayName => null; + + public bool Explicit => false; + + public string Skip => null; + + public Type[] SkipExceptions => null; + + public Type SkipType => null; + + public string SkipUnless => null; + + public string SkipWhen => null; + + public string SourceFilePath => null; + + public int? SourceLineNumber => null; + + public int Timeout => 0; + } + + /// + /// An attribute whose construction throws, standing in for one whose type cannot be loaded - + /// reading the attributes of a method carrying it throws rather than answering. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + private sealed class UnreadableAttribute : Attribute + { + public UnreadableAttribute() + { + throw new InvalidOperationException("This attribute cannot be constructed."); + } + } + + /// + /// The methods the assertions above reflect over. None of them is discovered as a test: they + /// are inspected directly. + /// + private sealed class Subject + { + [InterfaceOnlyFact] + public void MarkedByInterfaceOnly() + { + } + + public void NotATest() + { + } + + [InterfaceOnlyFact] + [Unreadable] + public void AttributesCannotBeRead() + { + } + + public void GenericOverload() + { + } + + public void GenericOverload() + { + } + + public void Overload(int value) + { + } + + public void Overload(string value) + { + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/FixtureVariantExpansionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FixtureVariantExpansionTests.cs new file mode 100644 index 0000000000..d809641bad --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FixtureVariantExpansionTests.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a test class carrying fixture argument sets is expanded into one variant per + /// argument set, and that each variant reports under its own display name. + /// + public class FixtureVariantExpansionTests + { + private const string TestName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants.FixtureVariantTests.EachVariantIsReportedUnderItsOwnName"; + + /// + /// Without the argument set suffix every variant reports under the same name, so the + /// variants collide in the results and a failure cannot be attributed to the data store + /// that produced it. + /// + [Fact] + public void GivenAClassWithFixtureArgumentSets_WhenTheRunCompletes_ThenEachVariantIsNamedAfterItsArgumentSet() + { + TestAssetRun run = TestAssetRunner.Run("FixtureVariants"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [TestName + " (Sql)"] = "Passed", + [TestName + " (Cosmos)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs new file mode 100644 index 0000000000..086a9c9d3a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs @@ -0,0 +1,106 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a method whose fixture argument set attribute cannot even be read costs only + /// that method, rather than every method of its class. + /// + public class MethodAttributeFaultTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MethodAttributeFault.MethodAttributeFaultTests"; + private const string SqlErrorCaseName = ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"; + private const string SqlSomeErrorCaseName = ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql, Some)"; + + /// + /// Reading the attribute is a separate step from expanding it, and it can fail on its own: a + /// method carrying two different fixture argument set attributes has no single one to expand. + /// Reading every method's attribute up front, before the walk that isolates each method, put + /// that failure outside the isolation and cost the whole class its tests - reported as one + /// failure claiming the class never ran, with the healthy methods gone and no sign of it. + /// Reading each method's attribute inside the walk keeps the loss to the method at fault. + /// + [Fact] + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTheClassIsDiscovered_ThenTheOtherMethodsStillRun() + { + TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".RunsBeforeTheFault"] = "Passed", + [ScenarioClass + ".RunsAfterTheFault (Cosmos)"] = "Passed", + [SqlErrorCaseName] = "Failed", + [SqlSomeErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failure has to carry the values the failing method itself declared, so that the leg + /// which would have run it is the leg that sees it. Asking the method for the single attribute + /// it declares is what threw, and answering that with nothing left the failure carrying only + /// values borrowed from a sibling - reported to the Cosmos leg, which would never have run + /// this method, and hidden from the SQL leg, which would. Reading each of the method's + /// attributes on its own keeps the failure where its tests were. + /// + [Fact] + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTestsAreSelectedByItsOwnArgumentSetTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SqlErrorCaseName] = "Failed", + [SqlSomeErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The other side of the same guarantee: the sibling's value must not leak onto the failure. + /// A failure tagged with a value its method never declared is a red test on a leg that would + /// never have run it, and - worse - is dropped by that leg's exclusion filter in the runs + /// where the fault should have been reported elsewhere. + /// + [Fact] + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTestsAreSelectedByASiblingsArgumentSetTrait_ThenOnlyTheSiblingRuns() + { + TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault", filterTrait: "AssetDataStore=Cosmos"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".RunsAfterTheFault (Cosmos)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + + /// + /// The failure has to name the method whose attribute could not be read, otherwise a class of + /// many methods reports a fault with no indication of which declaration to go and fix. + /// + [Fact] + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTheClassIsDiscovered_ThenTheFailureNamesOnlyThatMethod() + { + TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault"); + + Assert.Contains("MethodAttributeFaultTests.NeverRuns", run.Output, StringComparison.Ordinal); + Assert.Contains("Other methods of the class were discovered normally", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj b/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj new file mode 100644 index 0000000000..282a88096c --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj @@ -0,0 +1,64 @@ + + + + + $(MSBuildThisFileDirectory)..\Microsoft.Health.Extensions.Xunit.TestAssets\bin\$(Configuration)\$(TargetFramework)\Microsoft.Health.Extensions.Xunit.TestAssets.dll + + $(MSBuildThisFileDirectory)..\..\build\jobs\scripts\Assert-RetriedFailuresPassed.ps1 + + $(MSBuildThisFileDirectory)..\..\build\jobs\scripts\Assert-UnitTestProjectsDiscovered.ps1 + $(MSBuildThisFileDirectory)..\..\build\jobs\scripts\UnitTestProjects.txt + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + <_Parameter1>TestAssetsAssembly + <_Parameter2>$(TestAssetsAssembly) + + + <_Parameter1>RetryReconciliationScript + <_Parameter2>$(RetryReconciliationScript) + + + <_Parameter1>UnitTestProjectsScript + <_Parameter2>$(UnitTestProjectsScript) + + + <_Parameter1>UnitTestProjectManifest + <_Parameter2>$(UnitTestProjectManifest) + + + + diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/MismatchedDimensionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MismatchedDimensionTests.cs new file mode 100644 index 0000000000..0ab947d300 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MismatchedDimensionTests.cs @@ -0,0 +1,87 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a method whose argument set attribute names a different enum than its class's does in + /// the same dimension. + /// + /// + /// The merge pairs dimensions by position while the executor binds fixture arguments by enum type, + /// and nothing but convention keeps the two agreeing. Where they disagree the method's variants + /// carry no value for the dimension the class declared, and so none of the traits the SQL and + /// Cosmos legs select by: both legs would run none of this method's tests and both would still + /// report success. The discoverer therefore refuses the declaration and reports a failure, which + /// is loud, in place of an expansion that would have been silent. + /// + public class MismatchedDimensionTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MismatchedDimension.MismatchedDimensionTests"; + + /// + /// The refusal is reported as a failure standing in for the offending method, under each of + /// the class's combinations, while the method that declared nothing unusual runs normally. + /// + [Fact] + public void GivenAMethodDeclaringADifferentDimensionThanItsClass_WhenItIsDiscovered_ThenItIsReportedAndItsSiblingStillRuns() + { + TestAssetRun run = TestAssetRunner.Run("MismatchedDimension"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Cosmos)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Some)"] = "Failed", + [ScenarioClass + ".SiblingStillRuns (Sql)"] = "Passed", + [ScenarioClass + ".SiblingStillRuns (Cosmos)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failure names both enums and the position they disagree in, because the declaration + /// that caused it is legal C# and reads as though it should work. + /// + [Fact] + public void GivenAMethodDeclaringADifferentDimensionThanItsClass_WhenItIsDiscovered_ThenTheFailureNamesBothDimensions() + { + TestAssetRun run = TestAssetRunner.Run("MismatchedDimension"); + + Assert.Contains("AssetOtherDimension", run.Output, StringComparison.Ordinal); + Assert.Contains("AssetDataStore", run.Output, StringComparison.Ordinal); + Assert.Contains("argument set dimension 0", run.Output, StringComparison.Ordinal); + } + + /// + /// The SQL leg's own filter shape. The refused method must still be reported to the leg that + /// would have run it - a failure the leg cannot see is no better than the silent expansion the + /// refusal replaced. + /// + [Fact] + public void GivenAMethodDeclaringADifferentDimensionThanItsClass_WhenALegExcludesTheOtherDataStore_ThenTheFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("MismatchedDimension", filterNotTrait: "AssetDataStore=Cosmos"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Some)"] = "Failed", + [ScenarioClass + ".SiblingStillRuns (Sql)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs new file mode 100644 index 0000000000..1f6e381dcd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs @@ -0,0 +1,83 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that one method whose fixture argument sets cannot be expanded costs only that + /// method, and that the failure standing in for it is selected by the same filters it was. + /// + public class PartialDiscoveryFaultTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartialDiscoveryFault.PartialDiscoveryFaultTests"; + private const string ErrorCaseSql = ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"; + private const string ErrorCaseCosmos = ScenarioClass + ".NeverRuns (fixture argument set discovery: Cosmos)"; + + /// + /// Test cases are published to xUnit as the discoverer walks the class's methods, so a + /// failure let out of that walk leaves the methods already published running while the + /// methods after it are silently dropped - and reports that as a single failure claiming + /// the whole class never ran, which is wrong in both directions. Expanding each method + /// separately keeps the blast radius to the method that is actually misdeclared. + /// + [Fact] + public void GivenOneMethodThatCannotBeExpanded_WhenTheClassIsDiscovered_ThenTheOtherMethodsStillRun() + { + TestAssetRun run = TestAssetRunner.Run("PartialDiscoveryFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".RunsBeforeTheFault"] = "Passed", + [ScenarioClass + ".RunsAfterTheFault"] = "Passed", + [ErrorCaseSql] = "Failed", + [ErrorCaseCosmos] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failure has to say which method was lost, otherwise a class of many methods reports + /// a fault with no indication of where to look for it. + /// + [Fact] + public void GivenOneMethodThatCannotBeExpanded_WhenTheClassIsDiscovered_ThenTheFailureNamesOnlyThatMethod() + { + TestAssetRun run = TestAssetRunner.Run("PartialDiscoveryFault"); + + Assert.Contains("PartialDiscoveryFaultTests.NeverRuns", run.Output, StringComparison.Ordinal); + Assert.Contains("Other methods of the class were discovered normally", run.Output, StringComparison.Ordinal); + } + + /// + /// CI legs combine ordinary traits with argument set values in filters such as + /// (DataStore=CosmosDb)&(Category=ExportLongRunning). The argument set half of + /// such a filter is covered elsewhere; this covers the ordinary half, which the failing + /// method declares itself. Without it the leg that would have run the method passes with + /// the method silently missing, which is the failure this mechanism exists to prevent. + /// + [Fact] + public void GivenOneMethodThatCannotBeExpanded_WhenTestsAreSelectedByItsOrdinaryTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("PartialDiscoveryFault", filterTrait: "AssetCategory=PartialFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ErrorCaseSql] = "Failed", + [ErrorCaseCosmos] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartiallyEmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartiallyEmptyArgumentSetTests.cs new file mode 100644 index 0000000000..96385f15ec --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartiallyEmptyArgumentSetTests.cs @@ -0,0 +1,74 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a declaration that names a value in one argument set dimension and none in the other. + /// + /// + /// Naming nothing in every dimension is already rescued: the failure standing in for the method + /// is reported once per value each dimension's type declares, so a leg selecting positively on + /// one of them still sees it. Naming nothing in only some of them is the same hole with a + /// narrower mouth - the product is still empty, the method still produces no tests, but the + /// stand-in says nothing about the dimension that named nothing and a leg selecting on that + /// dimension still sees an empty, green run. + /// + public class PartiallyEmptyArgumentSetTests + { + private const string SqlCase = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartiallyEmptyArgumentSet.PartiallyEmptyArgumentSetTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + private const string CosmosCase = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartiallyEmptyArgumentSet.PartiallyEmptyArgumentSetTests.NeverRuns (fixture argument set discovery: Cosmos, Some)"; + + private static Dictionary EveryDataStore() => + new Dictionary + { + [SqlCase] = "Failed", + [CosmosCase] = "Failed", + }; + + /// + /// The dimension that named nothing is widened to every value its type declares, so the + /// failure is reported once per data store rather than once with no data store at all. + /// + [Fact] + public void GivenADeclarationNamingNothingInOneDimension_WhenItExpandsToNoVariants_ThenTheFailureIsReportedForEveryValueOfThatDimension() + { + TestAssetRun run = TestAssetRunner.Run("PartiallyEmptyArgumentSet"); + + TestAssetRunAssertions.PublishedExactly(run, EveryDataStore()); + } + + /// + /// This is the shape of the E2E and export legs' filter. Without the widening the stand-in + /// carries no data store trait, this selects nothing, and the leg passes with the method's + /// tests missing. + /// + [Fact] + public void GivenADeclarationNamingNothingInOneDimension_WhenALegSelectsPositivelyOnThatDimension_ThenTheFailureIsSelected() + { + TestAssetRun run = TestAssetRunner.Run("PartiallyEmptyArgumentSet", filterQueryTraits: "(AssetDataStore=Sql)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary { [SqlCase] = "Failed" }); + } + + /// + /// The dimension that did name a value keeps it. Widening the one that named nothing must + /// not spill into the one that did, or the failure would be reported under values the + /// declaration explicitly did not ask for. + /// + [Fact] + public void GivenADeclarationNamingNothingInOneDimension_WhenALegSelectsOnTheDeclaredDimension_ThenTheFailureIsSelected() + { + TestAssetRun run = TestAssetRunner.Run("PartiallyEmptyArgumentSet", filterQueryTraits: "(AssetOtherDimension=Some)"); + + TestAssetRunAssertions.PublishedExactly(run, EveryDataStore()); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs new file mode 100644 index 0000000000..eb1ea37c05 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs @@ -0,0 +1,69 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies discovery for a class that declares fixture argument sets on some methods but not on + /// the class itself. Such a class cannot take the fast path that hands wholly undecorated classes + /// straight to xUnit, yet it has no class-level sets to expand either, so its undecorated methods + /// take a passthrough path of their own. + /// + public class PassthroughDiscoveryTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MixedFixtureAttributes.MixedAttributeTests."; + + /// + /// A method on the passthrough path is handed to xUnit with its arguments left unresolved, so + /// that data attributes are still expanded downstream. Resolving them here instead would cost + /// a theory every one of its rows, and losing rows shrinks the suite silently: the run stays + /// green while the cases that would have failed simply never execute. + /// + [Fact] + public void GivenAClassWithArgumentSetsOnOnlySomeMethods_WhenTheRunCompletes_ThenUndecoratedMethodsAreDiscoveredInFull() + { + TestAssetRun run = TestAssetRunner.Run("MixedFixtureAttributes"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "ExpandedMethod (Sql)"] = "Passed", + [ClassName + "ExpandedMethod (Cosmos)"] = "Passed", + [ClassName + "PassthroughFact"] = "Passed", + [ClassName + "PassthroughTheory(value: 1)"] = "Passed", + [ClassName + "PassthroughTheory(value: 2)"] = "Passed", + [ClassName + "PassthroughTheory(value: 3)"] = "Passed", + }); + } + + /// + /// The expanded method's variants have to carry the argument set trait even though the class + /// declares no argument sets of its own, because the export and E2E legs select positively on + /// that trait. A variant without it is not merely mislabelled: it is unreachable to those legs, + /// which then run nothing of the sort and report green. + /// + /// + /// The methods on the passthrough path carry no argument set trait, which is correct - they + /// were never expanded and there is no value to name - so this also pins that a leg naming one + /// data store gets the one variant and nothing else. + /// + [Fact] + public void GivenAClassWithArgumentSetsOnOnlySomeMethods_WhenALegSelectsOneDataStore_ThenOnlyThatVariantIsSelected() + { + TestAssetRun run = TestAssetRunner.Run("MixedFixtureAttributes", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "ExpandedMethod (Sql)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetriedFailureOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetriedFailureOutcomeTests.cs new file mode 100644 index 0000000000..182823b7af --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetriedFailureOutcomeTests.cs @@ -0,0 +1,77 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Records what the runner reports when a retried test stops failing without passing. + /// + /// + /// Every CI leg runs with retries and treats the runner's exit code as the verdict, because the + /// results of the earlier attempts are published too and failing on those would cancel out every + /// retry. That is only sound while a zero exit code means every failure was cleared, and this is + /// the case where it does not: the retry counts a test that skipped as no longer failing, so the + /// run exits zero with a real failure inside it. + /// + /// The legs close that gap outside the runner, in + /// build/jobs/scripts/Assert-RetriedFailuresPassed.ps1, which requires every test that + /// failed in an attempt to be recorded as passing in the final one. This test is what would + /// notice if the runner ever started reporting this itself, at which point that script has + /// nothing left to do. + /// + public class RetriedFailureOutcomeTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FailThenSkipOnRetry.FailThenSkipOnRetryTests."; + private const string AttemptFileVariable = "XUNIT_EXT_ASSET_ATTEMPT_FILE"; + + /// + /// Runs two tests that both fail first: one then passes, one then skips. + /// + [Fact] + public void GivenARetriedTestThatSkipsInsteadOfPassing_WhenTheRunFinishes_ThenTheRunnerReportsSuccessAnyway() + { + string attemptFile = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", $"attempt-{Guid.NewGuid():N}.txt"); + Directory.CreateDirectory(Path.GetDirectoryName(attemptFile)); + + try + { + TestAssetRun run = TestAssetRunner.Run( + "FailThenSkipOnRetry", + retryFailedTests: "2", + environment: new Dictionary { [AttemptFileVariable] = attemptFile }); + + // Both tests failed on the first attempt. The one that went on to skip is reported as + // NotExecuted, which is indistinguishable from a test that was always going to skip, + // and the run still exits zero: that is the whole of the exposure. + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "FailsOnTheFirstAttemptAndSkipsOnTheNext"] = "NotExecuted", + [ClassName + "FailsOnTheFirstAttemptAndPassesOnTheNext"] = "Passed", + }); + } + finally + { + foreach (string path in new[] { attemptFile, attemptFile + ".passing" }) + { + try + { + File.Delete(path); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + // A leaked temp file is not worth failing an otherwise good test over. + } + } + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryAssertionClassificationTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryAssertionClassificationTests.cs new file mode 100644 index 0000000000..a4e7481206 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryAssertionClassificationTests.cs @@ -0,0 +1,120 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Xunit; +using Xunit.Sdk; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers how a failure is recognised as an assertion failure when the retry attempt itself has + /// to classify it. + /// + /// + /// RetryOnAssertionFailure is what tells a retrying test not to spend attempts on a failure + /// that will fail the same way every time. It can only do that if an assertion failure is still + /// recognised once something has wrapped it, which is the normal shape rather than an exotic one: + /// anything awaited through , a parallel helper, or a teardown + /// that throws alongside the test arrives as an . Two places make + /// this call - the message bus, from the reported type names, and the attempt itself, from the + /// exception - and they have to agree, or whether the policy is honoured comes down to which one + /// happened to see the failure. + /// + public class RetryAssertionClassificationTests + { + /// + /// The plain case: an assertion failure with nothing around it. + /// + [Fact] + public void GivenABareAssertionFailure_WhenItIsClassified_ThenItIsAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure(new XunitException("assertion"))); + } + + /// + /// One wrapper deep, which is what awaiting a single faulted task produces. + /// + [Fact] + public void GivenAnAssertionFailureWrappedAlone_WhenItIsClassified_ThenItIsAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure( + new AggregateException(new XunitException("assertion")))); + } + + /// + /// The case that unwrapping only single-inner aggregates missed. A test that awaits several + /// tasks, or whose teardown throws alongside the assertion, produces an aggregate holding more + /// than one exception, and the assertion is no less deterministic for having company. + /// + [Fact] + public void GivenAnAssertionFailureAmongOthers_WhenItIsClassified_ThenItIsStillAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure( + new AggregateException( + new InvalidOperationException("the connection dropped"), + new XunitException("assertion")))); + } + + /// + /// Nesting is not always flat: an aggregate can hold an aggregate. + /// + [Fact] + public void GivenAnAssertionFailureNestedDeeply_WhenItIsClassified_ThenItIsStillAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure( + new AggregateException( + new InvalidOperationException("the connection dropped"), + new AggregateException(new XunitException("assertion"))))); + } + + /// + /// An assertion reached through an ordinary inner exception rather than an aggregate. + /// + [Fact] + public void GivenAnAssertionFailureAsAnInnerException_WhenItIsClassified_ThenItIsStillAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure( + new InvalidOperationException("wrapped", new XunitException("assertion")))); + } + + /// + /// The other direction matters just as much. Classifying a transient failure as an assertion + /// would cost it every retry it was given the attribute for. + /// + [Fact] + public void GivenNoAssertionFailureAnywhere_WhenItIsClassified_ThenItIsNotAnAssertionFailure() + { + Assert.False(RetryTestCase.ContainsAssertionFailure( + new AggregateException( + new InvalidOperationException("the connection dropped"), + new TimeoutException("it took too long")))); + } + + /// + /// A timeout needs no carve-out here, unlike in the message bus, which matches on type names + /// and so has to exclude the one whose name contains "Xunit". This classifies by type, and the + /// invariant that makes that safe is pinned here: TestTimeoutException does not derive + /// from , so a timeout stays retryable - which is the whole point, + /// as a test that ran long because a dependency was slow once is what retrying is for. The + /// type cannot be constructed from here, so the relationship is asserted rather than exercised. + /// + [Fact] + public void GivenTheTimeoutType_WhenItIsInspected_ThenItIsNotAnAssertionType() + { + Assert.False(typeof(XunitException).IsAssignableFrom(typeof(TestTimeoutException))); + } + + /// + /// Nothing to classify is not an assertion failure, and must not throw on the way to saying so. + /// + [Fact] + public void GivenNoException_WhenItIsClassified_ThenItIsNotAnAssertionFailure() + { + Assert.False(RetryTestCase.ContainsAssertionFailure(null)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs new file mode 100644 index 0000000000..c60c72fb75 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs @@ -0,0 +1,88 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies how resolves a deferred failure when the run is + /// cancelled part way through, which is where a retrying test can either lose a real failure + /// or invent one that never happened. + /// + /// + /// Both scenarios need their two collections to make progress at the same time: one cancels the + /// run while the other is mid-attempt. The scenarios arrange that themselves by waiting + /// asynchronously for each other, so it holds however many threads the runner uses. Pinning a + /// thread count from here was measured to make it worse rather than better: with two threads and + /// a blocking wait, the collection being waited for did not get a thread until the waiter had + /// already given up. + /// + public class RetryFactCancellationTests + { + private const string RetryDelayPrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay."; + private const string PassingAttemptPrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt."; + + /// + /// What this budget has to separate is a cancelled run from one that sat through its retry + /// delays instead. Those are 30s apart and there are three of them, so a run that was not + /// cancelled cannot finish in less than 30s and in practice takes about 90s. A cancelled run + /// takes seconds. Setting the line at 60s therefore still distinguishes the two, while + /// leaving room for a loaded agent to be slow at starting a child process without turning a + /// correct run red: this assertion is a backstop, and the scenario's own handshake is what + /// makes the cancellation land where it should. + /// + private static readonly TimeSpan CancellationBudget = TimeSpan.FromSeconds(60); + + /// + /// A failing attempt's result is held back in case a later attempt supersedes it. If the + /// run is cancelled during the retry delay there is no later attempt, so the held-back + /// failure is the only record the test ever ran and must still be published. + /// + [Fact] + public void GivenARetryingTest_WhenTheRunIsCancelledDuringTheRetryDelay_ThenTheDeferredFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("CancelledDuringRetryDelay", stopOnFail: true); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [RetryDelayPrefix + "CancellationTriggerTests.FailsToTriggerCancellation"] = "Failed", + [RetryDelayPrefix + "DeferredFailureTests.FailureSurvivesCancellationDuringRetryDelay"] = "Failed", + }); + + // The same failed result would be reported by simply exhausting every retry, so without + // this the scenario would still pass if cancellation never happened. Three attempts + // spaced by the 30s delay cannot complete in anything close to this budget. + string message = $"The run took {run.Duration.TotalSeconds:F1}s, which is long enough to have exhausted " + + $"the retry delays rather than being cancelled during one. The expected results were reported, but " + + $"not for the reason this scenario exists to check.{Environment.NewLine}Actual: {run}"; + + Assert.True(run.Duration < CancellationBudget, message); + } + + /// + /// An attempt can run to completion while cancellation is already requested. It has + /// published its own result, so the earlier attempt's deferred failure must be discarded: + /// republishing it would add a second, unattributable result for a test that passed. + /// + [Fact] + public void GivenARetryingTest_WhenAnAttemptPassesWhileCancellationIsRequested_ThenOnlyThePassIsReported() + { + TestAssetRun run = TestAssetRunner.Run("CancelledDuringPassingAttempt", stopOnFail: true); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [PassingAttemptPrefix + "CancellationTriggerTests.FailsToTriggerCancellation"] = "Failed", + [PassingAttemptPrefix + "PassesWhileCancellingTests.PassingAttemptIsNotOverriddenByEarlierFailure"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs new file mode 100644 index 0000000000..b990dc3b55 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs @@ -0,0 +1,46 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that publishes exactly one result per test, with + /// the outcome of the attempt that decided it. + /// + public class RetryFactOutcomeTests + { + private const string Prefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryOutcomes.RetryOutcomeTests."; + + /// + /// Covers the outcome matrix: a non-retriable failure, an exhausted retry, a flaky test + /// that eventually passes, a non-assertion exception, a plain pass, a run-time skip, and a + /// clamped configuration. Every one of these must appear exactly once, and the failures must + /// not be swallowed by the retry bookkeeping. The skip pins that a test which excuses itself + /// is neither retried nor turned into a pass. + /// + [Fact] + public void GivenRetryingTests_WhenTheRunCompletes_ThenEachTestIsReportedExactlyOnce() + { + TestAssetRun run = TestAssetRunner.Run("RetryOutcomes"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "NonRetriableAssertionFailure_IsReportedOnce"] = "Failed", + [Prefix + "ExhaustedRetries_IsReportedFailedOnce"] = "Failed", + [Prefix + "NonAssertionException_IsReportedFailedOnce"] = "Failed", + [Prefix + "FlakyThenPasses_IsReportedPassedOnce"] = "Passed", + [Prefix + "AlwaysPasses_IsReportedPassed"] = "Passed", + [Prefix + "SkippedAtRunTime_IsReportedSkipped"] = "NotExecuted", + [Prefix + "ClampedRetryConfiguration_RunsExactlyOnce"] = "Passed", + [Prefix + "ClampedNegativeDelay_StillRetries"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs new file mode 100644 index 0000000000..348279ad3f --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins the decision RetryOnAssertionFailure makes, rather than only the outcomes either + /// side of it. + /// + /// + /// The existing retry scenarios use tests that fail on every attempt or on none. Those report + /// the same result whether the retries were spent or not, so a regression that retried + /// everything, or nothing, would leave them green. Both tests here recover on their second + /// attempt, which makes the reported outcome a direct statement about whether that attempt ran. + /// + public class RetryPolicyTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryPolicy.RetryPolicyTests."; + + /// + /// Under the default policy an assertion failure is not retried and a non-assertion + /// exception is, so a test that recovers on its second attempt is reported failed in the + /// first case and passed in the second. + /// + [Fact] + public void GivenTestsThatRecoverOnASecondAttempt_WhenTheDefaultPolicyApplies_ThenOnlyTheNonAssertionFailureIsRetried() + { + TestAssetRun run = TestAssetRunner.Run("RetryPolicy"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "AssertionFailureUnderTheDefaultPolicy_IsNotRetried"] = "Failed", + [ClassName + "NonAssertionExceptionUnderTheDefaultPolicy_IsRetried"] = "Passed", + [ClassName + "WrappedAssertionFailureUnderTheDefaultPolicy_IsNotRetried"] = "Failed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryReconciliationScriptTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryReconciliationScriptTests.cs new file mode 100644 index 0000000000..fd94e4cd99 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryReconciliationScriptTests.cs @@ -0,0 +1,325 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Xml.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers the script the CI legs use to decide whether a retried failure really cleared. + /// + /// + /// Every test leg runs with retries and lets the runner's exit code be the verdict, and the + /// runner exits zero when a failing test stops failing without passing. The script in + /// build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 is the only thing that turns that + /// back into a red leg, so a mistake in it puts a real failure back on the green path it was + /// written to close. These tests run the real script over reports shaped like the ones the + /// runner writes. + /// + public class RetryReconciliationScriptTests + { + private const string TrxNamespace = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + private const string DefaultStorage = @"c:\bin\assembly.dll"; + + private static readonly TimeSpan RunTimeout = TimeSpan.FromMinutes(2); + + /// + /// Two tests whose names differ only in case, where the failing one never passed. + /// + /// + /// Display names are not identities. A theory whose rows differ only in the case of an + /// argument produces exactly this shape, and the repository already has one: + /// VersioningConfigurationTests passes both "Versioned" and the constant whose + /// value is "versioned" to the same theory. Keyed by name in a comparison that + /// ignores case, the row that passed answers for the row that did not, and the failure the + /// script exists to catch leaves through the door it was meant to close. + /// + [Fact] + public void GivenTwoTestsNamedAlikeApartFromCase_WhenOnlyOneOfThemPassed_ThenTheOtherIsStillReportedAsUnresolved() + { + var results = new ResultsDirectory(); + + results.WriteAttempt( + Result("11111111-1111-1111-1111-111111111111", "T.Row(input: \"Versioned\")", "Failed")); + + results.WriteFinal( + Result("11111111-1111-1111-1111-111111111111", "T.Row(input: \"Versioned\")", "NotExecuted"), + Result("22222222-2222-2222-2222-222222222222", "T.Row(input: \"versioned\")", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("T.Row(input: \"Versioned\") -> NotExecuted", run.Output, StringComparison.Ordinal); + } + + /// + /// Two tests sharing one display name exactly, both of which passed in the end. + /// + /// + /// This is the other half of telling tests apart by identity: distinguishing them must not + /// come at the cost of reporting a test that did pass, or the legs learn to ignore this + /// script. + /// + [Fact] + public void GivenTwoTestsSharingADisplayName_WhenBothOfThemPassed_ThenNothingIsReported() + { + var results = new ResultsDirectory(); + + results.WriteAttempt( + Result("11111111-1111-1111-1111-111111111111", "T.Row", "Failed"), + Result("22222222-2222-2222-2222-222222222222", "T.Row", "Failed")); + + results.WriteFinal( + Result("11111111-1111-1111-1111-111111111111", "T.Row", "Passed"), + Result("22222222-2222-2222-2222-222222222222", "T.Row", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("(2 reconciled)", run.Output, StringComparison.Ordinal); + } + + /// + /// A test that failed and then passed, which is what a retry is for. + /// + [Fact] + public void GivenAFailureThatPassedOnTheNextAttempt_WhenReconciling_ThenTheLegIsLeftGreen() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "Failed")); + results.WriteFinal(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("(1 reconciled)", run.Output, StringComparison.Ordinal); + } + + /// + /// A test that failed and then skipped, which the runner reports as a success. + /// + [Fact] + public void GivenAFailureThatSkippedOnTheNextAttempt_WhenReconciling_ThenTheLegIsFailed() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "Failed")); + results.WriteFinal(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "NotExecuted")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("T.Flaky -> NotExecuted", run.Output, StringComparison.Ordinal); + } + + /// + /// A test that failed and then was not recorded in the final attempt at all. + /// + [Fact] + public void GivenAFailureAbsentFromTheFinalAttempt_WhenReconciling_ThenTheLegIsFailed() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result("11111111-1111-1111-1111-111111111111", "T.Vanished", "Failed")); + results.WriteFinal(Result("22222222-2222-2222-2222-222222222222", "T.Other", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("T.Vanished -> not run at all", run.Output, StringComparison.Ordinal); + } + + /// + /// The same test identity reported by two different assemblies. + /// + /// + /// This repository compiles the same shared test files into several assemblies, so the same + /// fully qualified name is a real test in each of them and the reports can give them the + /// same identifier. A pass in one assembly must not answer for a failure in another. + /// + [Fact] + public void GivenOneTestIdentityInTwoAssemblies_WhenOnlyOneOfThemPassed_ThenTheOtherIsStillReportedAsUnresolved() + { + var results = new ResultsDirectory(); + + results.WriteAttempt( + Result("11111111-1111-1111-1111-111111111111", "Shared.T.Case", "Failed", storage: @"c:\bin\r4.dll")); + + // Each assembly reports its own results, so the two reports below are what the same test + // identity coming from two assemblies actually looks like on disk. + results.WriteFinal( + Result("11111111-1111-1111-1111-111111111111", "Shared.T.Case", "NotExecuted", storage: @"c:\bin\r4.dll")); + + results.WriteFinal( + Result("11111111-1111-1111-1111-111111111111", "Shared.T.Case", "Passed", storage: @"c:\bin\r5.dll")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("Shared.T.Case -> NotExecuted", run.Output, StringComparison.Ordinal); + } + + /// + /// A result the report gives no identifier, which nothing can be matched against. + /// + [Fact] + public void GivenAResultWithNoTestId_WhenReconciling_ThenTheLegIsFailedWithAnExplanation() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result(testId: null, "T.Anonymous", "Failed")); + results.WriteFinal(Result("11111111-1111-1111-1111-111111111111", "T.Anonymous", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("with no test id", run.Output, StringComparison.Ordinal); + } + + /// + /// Reports that recorded nothing at all, which is what an empty run writes. + /// + /// + /// The verdict for a leg that ran nothing belongs to the discovery floor, which says so in + /// as many words. This script only has to leave that message intact instead of replacing it + /// with an error about reading XML. + /// + [Fact] + public void GivenReportsThatRecordedNothing_WhenReconciling_ThenTheLegIsLeftGreenWithoutAnError() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(); + results.WriteFinal(); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("nothing to reconcile", run.Output, StringComparison.Ordinal); + } + + /// + /// A run with no retries below it, which is every run where nothing failed. + /// + [Fact] + public void GivenNoRetryAttempts_WhenReconciling_ThenTheLegIsLeftGreen() + { + var results = new ResultsDirectory(); + + results.WriteFinal(Result("11111111-1111-1111-1111-111111111111", "T.Steady", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("nothing to reconcile", run.Output, StringComparison.Ordinal); + } + + /// + /// Retry attempts with no final report beside them, so nothing says how they ended. + /// + [Fact] + public void GivenRetryAttemptsWithNoFinalReport_WhenReconciling_ThenTheLegIsFailed() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "Failed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("no final report", run.Output, StringComparison.Ordinal); + } + + private static TestResult Result(string testId, string testName, string outcome, string storage = DefaultStorage) + => new TestResult(testId, testName, outcome, storage); + + private sealed record TestResult(string TestId, string TestName, string Outcome, string Storage); + + /// + /// A results directory shaped the way the retry extension leaves one behind. + /// + private sealed class ResultsDirectory + { + private readonly string _root = Path.Combine(Path.GetTempPath(), "xunit-ext-reconcile", Guid.NewGuid().ToString("N")); + private int _attempts; + private int _reports; + + public void WriteAttempt(params TestResult[] results) + { + _attempts++; + Write(Path.Combine(_root, "Retries", "run", _attempts.ToString(CultureInfo.InvariantCulture), "report.trx"), results); + } + + /// + /// Writes one final report. Each call writes a separate report, which is how results + /// from more than one assembly reach the same results directory. + /// + public void WriteFinal(params TestResult[] results) + { + _reports++; + Write(Path.Combine(_root, FormattableString.Invariant($"report{_reports}.trx")), results); + } + + public ScriptRun Reconcile() + { + try + { + return ScriptRunner.Run( + ScriptRunner.Resolve("RetryReconciliationScript"), + new Dictionary { ["ResultsDirectory"] = _root }); + } + finally + { + try + { + Directory.Delete(_root, recursive: true); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + // A leaked temp directory is not worth failing an otherwise good test over. + } + } + } + + private static void Write(string path, IReadOnlyList results) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)); + + XNamespace ns = TrxNamespace; + + var document = new XDocument( + new XElement( + ns + "TestRun", + new XAttribute("id", Guid.NewGuid().ToString()), + new XElement( + ns + "Results", + results.Select(r => new XElement( + ns + "UnitTestResult", + r.TestId == null ? null : new XAttribute("testId", r.TestId), + new XAttribute("testName", r.TestName), + new XAttribute("outcome", r.Outcome)))), + new XElement( + ns + "TestDefinitions", + results.Where(r => r.TestId != null).Select(r => new XElement( + ns + "UnitTest", + new XAttribute("id", r.TestId), + new XAttribute("name", r.TestName), + new XAttribute("storage", r.Storage)))))); + + document.Save(path); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs new file mode 100644 index 0000000000..b751dfba40 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs @@ -0,0 +1,218 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Xunit; +using Xunit.Sdk; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers how accounts for an attempt that published no result of + /// its own, which happens when the run is cancelled or aborted underneath it. + /// + /// + /// This decision cannot be reached deterministically from a scenario asset, because it needs the + /// runner to abandon an attempt mid-flight at an exact moment. The decision is therefore made by + /// two pure helpers so that every combination can be pinned directly. Without these tests the + /// branch that stops a never-reported test from counting as a pass is unprotected. + /// + public class RetryTestCaseNoResultTests + { + /// + /// A failure seen by the attempt that was cut short supersedes anything an earlier attempt + /// deferred, so only the current attempt's failure is replayed. Replaying both would publish + /// two results for one test. + /// + [Fact] + public void GivenFailuresOnBothTheCurrentAndAnEarlierAttempt_WhenNothingWasReported_ThenOnlyTheCurrentAttemptIsReplayed() + { + NoResultOutcome outcome = RetryTestCase.DecideNoResultOutcome( + currentAttemptObservedFailure: true, + earlierAttemptObservedFailure: true); + + Assert.Equal(NoResultOutcome.ReplayCurrentAttempt, outcome); + } + + /// + /// A failure seen only by the attempt that was cut short is still a real observed failure. + /// + [Fact] + public void GivenAFailureOnTheCurrentAttemptOnly_WhenNothingWasReported_ThenTheCurrentAttemptIsReplayed() + { + NoResultOutcome outcome = RetryTestCase.DecideNoResultOutcome( + currentAttemptObservedFailure: true, + earlierAttemptObservedFailure: false); + + Assert.Equal(NoResultOutcome.ReplayCurrentAttempt, outcome); + } + + /// + /// A failure deferred while waiting for a retry that then reported nothing must still reach + /// the results, rather than being dropped along with the abandoned attempt. + /// + [Fact] + public void GivenAFailureOnAnEarlierAttemptOnly_WhenNothingWasReported_ThenTheEarlierAttemptIsReplayed() + { + NoResultOutcome outcome = RetryTestCase.DecideNoResultOutcome( + currentAttemptObservedFailure: false, + earlierAttemptObservedFailure: true); + + Assert.Equal(NoResultOutcome.ReplayEarlierAttempt, outcome); + } + + /// + /// When no attempt ever saw a failure and none reported a result, there is nothing to replay. + /// + [Fact] + public void GivenNoObservedFailure_WhenNothingWasReported_ThenNothingIsReplayed() + { + NoResultOutcome outcome = RetryTestCase.DecideNoResultOutcome( + currentAttemptObservedFailure: false, + earlierAttemptObservedFailure: false); + + Assert.Equal(NoResultOutcome.ReportNothing, outcome); + } + + /// + /// A test that no attempt ever reported must contribute nothing to the totals. Reporting one + /// test with no failures is what previously turned an abandoned test into a silent pass. + /// + [Fact] + public void GivenNothingToReplay_WhenTheSummaryIsBuilt_ThenTheTestContributesNothing() + { + RunSummary summary = RetryTestCase.CreateNoResultSummary(NoResultOutcome.ReportNothing); + + Assert.Equal(0, summary.Total); + Assert.Equal(0, summary.Failed); + } + + /// + /// A failure replayed from the attempt that was cut short is a published result, so it must + /// be counted as one failed test. + /// + [Fact] + public void GivenTheCurrentAttemptIsReplayed_WhenTheSummaryIsBuilt_ThenTheTestCountsAsOneFailure() + { + RunSummary summary = RetryTestCase.CreateNoResultSummary(NoResultOutcome.ReplayCurrentAttempt); + + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Failed); + } + + /// + /// A failure replayed from an earlier attempt is equally a published result, so it counts the + /// same way. + /// + [Fact] + public void GivenAnEarlierAttemptIsReplayed_WhenTheSummaryIsBuilt_ThenTheTestCountsAsOneFailure() + { + RunSummary summary = RetryTestCase.CreateNoResultSummary(NoResultOutcome.ReplayEarlierAttempt); + + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Failed); + } + + /// + /// The exception attached to a replayed current-attempt failure has to be the one that + /// attempt captured. Attaching the earlier attempt's exception would report this attempt's + /// failure under the previous attempt's error text. + /// + [Fact] + public void GivenTheCurrentAttemptIsReplayed_WhenSelectingTheException_ThenTheCurrentAttemptsExceptionIsChosen() + { + var current = new InvalidOperationException("current"); + var earlier = new InvalidOperationException("earlier"); + + Exception selected = RetryTestCase.SelectNoResultException(NoResultOutcome.ReplayCurrentAttempt, current, earlier); + + Assert.Same(current, selected); + } + + /// + /// An attempt cut short before it captured anything of its own still has its failure + /// replayed, and nothing is attached rather than reaching back for a stale exception. + /// + [Fact] + public void GivenTheCurrentAttemptIsReplayedWithoutAnException_WhenSelectingTheException_ThenNothingIsChosen() + { + Exception selected = RetryTestCase.SelectNoResultException( + NoResultOutcome.ReplayCurrentAttempt, + currentAttemptException: null, + earlierAttemptException: new InvalidOperationException("earlier")); + + Assert.Null(selected); + } + + /// + /// When the earlier attempt's failure is the one replayed, its exception is the one that + /// describes it. + /// + [Fact] + public void GivenAnEarlierAttemptIsReplayed_WhenSelectingTheException_ThenTheEarlierAttemptsExceptionIsChosen() + { + var current = new InvalidOperationException("current"); + var earlier = new InvalidOperationException("earlier"); + + Exception selected = RetryTestCase.SelectNoResultException(NoResultOutcome.ReplayEarlierAttempt, current, earlier); + + Assert.Same(earlier, selected); + } + + /// + /// Nothing was published for the test, so nothing should be attached to it either. Adding an + /// exception here would turn a test that reported no result into a reported error. + /// + [Fact] + public void GivenNothingIsReplayed_WhenSelectingTheException_ThenNothingIsChosen() + { + Exception selected = RetryTestCase.SelectNoResultException( + NoResultOutcome.ReportNothing, + new InvalidOperationException("current"), + new InvalidOperationException("earlier")); + + Assert.Null(selected); + } + + /// + /// An outcome none of the three helpers was taught to handle must stop the run rather than + /// be absorbed by a default branch. + /// + /// + /// The helpers used to disagree about an unknown value: the two that publish treated it as + /// "report nothing" while the one that counts treated anything but ReportNothing as a + /// failure. Since the runner's verdict comes from the messages and not the summary, an + /// outcome added later and missed in one switch would have counted a failure that was never + /// published - a lost failure, and a green run. The value below is deliberately not a + /// declared member, which is what a future addition looks like to code compiled against + /// today's enum. + /// + [Fact] + public void GivenAnOutcomeNoHelperHandles_WhenItIsAccountedFor_ThenEachHelperRefusesIt() + { + const NoResultOutcome unknown = (NoResultOutcome)int.MaxValue; + + Assert.Throws(() => RetryTestCase.CreateNoResultSummary(unknown)); + Assert.Throws(() => RetryTestCase.SelectNoResultException(unknown, null, null)); + Assert.Throws(() => RetryTestCase.ReportSingleResult(unknown, null, null)); + } + + /// + /// Every declared outcome is handled by all three helpers, so the refusal above cannot be + /// satisfied by a helper that refuses everything. + /// + [Fact] + public void GivenEveryDeclaredOutcome_WhenItIsAccountedFor_ThenNoHelperRefusesIt() + { + foreach (NoResultOutcome outcome in Enum.GetValues()) + { + RetryTestCase.CreateNoResultSummary(outcome); + RetryTestCase.SelectNoResultException(outcome, null, null); + RetryTestCase.ReportSingleResult(outcome, null, null); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs new file mode 100644 index 0000000000..040429c6a2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs @@ -0,0 +1,63 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that publishes exactly one result per data row, + /// with the outcome of the attempt that decided that row. + /// + public class RetryTheoryOutcomeTests + { + private const string Prefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryTheoryOutcomes.RetryTheoryOutcomeTests."; + + /// + /// Retries are applied per row, not per method: two rows recover on their second attempt and + /// must be reported as passed, while the row that fails every attempt must be reported as + /// failed. A discoverer that lost rows, shared a single test case between them, or dropped + /// the retry wrapper would change this result set. + /// + [Fact] + public void GivenARetryingTheory_WhenTheRunCompletes_ThenEachRowIsReportedExactlyOnce() + { + TestAssetRun run = TestAssetRunner.Run("RetryTheoryOutcomes"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "FlakyRow(row: 1, recovers: True)"] = "Passed", + [Prefix + "FlakyRow(row: 2, recovers: False)"] = "Failed", + [Prefix + "FlakyRow(row: 3, recovers: True)"] = "Passed", + }); + } + + /// + /// A theory whose data is resolved at run time rather than at discovery cannot be wrapped for + /// retry, because the wrapper would supply the arguments it was built with -- none -- and every + /// row would be lost to an arity error instead of running. The discoverer leaves such a case + /// alone, so the rows still run, each with its own arguments, and simply do not retry: all + /// three fail here because each one fails its first attempt. Names carrying the argument values + /// are what separates "ran without retrying" from "lost its data". + /// + [Fact] + public void GivenARetryingTheoryResolvedAtRunTime_WhenTheRunCompletes_ThenEveryRowStillRunsWithItsArguments() + { + TestAssetRun run = TestAssetRunner.Run("RetryTheoryOutcomes", preEnumerateTheories: false); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "FlakyRow(row: 1, recovers: True)"] = "Failed", + [Prefix + "FlakyRow(row: 2, recovers: False)"] = "Failed", + [Prefix + "FlakyRow(row: 3, recovers: True)"] = "Failed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs new file mode 100644 index 0000000000..9b5ccd815d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers what the retrying test attributes do with a test that exceeds its timeout. + /// + /// + /// A timeout is the failure a retry harness exists for: a test that runs long because a + /// dependency was slow once is the definition of flaky. It reaches the retry decision as + /// Xunit.Sdk.TestTimeoutException, whose name contains "Xunit" and so matches the + /// substring test that classifies deterministic assertion failures - which are deliberately not + /// retried. A timeout is carved out of that match for exactly this reason, and these tests are + /// what would notice if the carve-out were removed: without it a timeout is denied every attempt + /// it was configured for. + /// + public class RetryTimeoutTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryTimeouts.RetryTimeoutTests."; + + /// + /// A test that times out once and then completes has to be retried into a pass, and one + /// that times out on every attempt has to be reported failed exactly once. + /// + [Fact] + public void GivenTestsThatExceedTheirTimeout_WhenTheRunCompletes_ThenTheFlakyOneIsRetriedIntoAPass() + { + TestAssetRun run = TestAssetRunner.Run("RetryTimeouts"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "FlakyTimeout_IsRetriedAndPasses"] = "Passed", + [ClassName + "AlwaysTimesOut_IsReportedFailedOnce"] = "Failed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryVariantTests.cs new file mode 100644 index 0000000000..718b64528e --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryVariantTests.cs @@ -0,0 +1,115 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a class that is both expanded per data store and uses the retrying test attributes, + /// which is the shape most of this repository's integration tests take. + /// + /// + /// Expansion and retrying were only ever asserted apart. They meet in the test case the retry + /// discoverers build, which is a different type from the one expansion writes to and copies the + /// traits across by hand. A copy that lost or renamed a trait would leave a variant that no leg + /// selecting positively on a data store could see, and such a leg reports success having run + /// nothing. + /// + public class RetryVariantTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryVariants.RetryVariantTests."; + + /// + /// Every retrying test is expanded per data store, theory rows included, and the flaky one + /// still recovers on a later attempt within each variant. + /// + [Fact] + public void GivenRetryingTestsInAnExpandedClass_WhenTheRunCompletes_ThenEveryVariantIsReported() + { + TestAssetRun run = TestAssetRunner.Run("RetryVariants"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "PassingRetryFact (Sql)"] = "Passed", + [ClassName + "PassingRetryFact (Cosmos)"] = "Passed", + [ClassName + "FlakyRetryFact (Sql)"] = "Passed", + [ClassName + "FlakyRetryFact (Cosmos)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 1) (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 2) (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 1) (Cosmos)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 2) (Cosmos)"] = "Passed", + [ClassName + "MalformedRetryTheory (Sql)"] = "Failed", + [ClassName + "EmptyDataRetryTheory (Sql)"] = "Failed", + [ClassName + "MalformedRetryTheory (Cosmos)"] = "Failed", + [ClassName + "EmptyDataRetryTheory (Cosmos)"] = "Failed", + }); + } + + /// + /// The shape the export and E2E legs use. A retrying test has to be selectable by the same + /// compound positive filter as any other, on both the data store the expansion gave it and + /// the category its class declares. + /// + [Fact] + public void GivenRetryingTestsInAnExpandedClass_WhenALegSelectsOnBothTraits_ThenOnlyThatDataStoreRuns() + { + TestAssetRun run = TestAssetRunner.Run( + "RetryVariants", + filterQueryTraits: "(AssetDataStore=Sql)&(Category=RetryVariant)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "PassingRetryFact (Sql)"] = "Passed", + [ClassName + "FlakyRetryFact (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 1) (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 2) (Sql)"] = "Passed", + [ClassName + "MalformedRetryTheory (Sql)"] = "Failed", + [ClassName + "EmptyDataRetryTheory (Sql)"] = "Failed", + }); + } + + /// + /// Trait filtering compares names without regard to case, and a retrying test that is also + /// expanded goes through two separate pieces of trait handling to reach the runner. This + /// pins that it still answers to the same filter either way. + /// + [Fact] + public void GivenRetryingTestsInAnExpandedClass_WhenALegNamesTheTraitInAnotherCase_ThenTheSameTestsRun() + { + TestAssetRun run = TestAssetRunner.Run("RetryVariants", filterTrait: "assetdatastore=sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "PassingRetryFact (Sql)"] = "Passed", + [ClassName + "FlakyRetryFact (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 1) (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 2) (Sql)"] = "Passed", + [ClassName + "MalformedRetryTheory (Sql)"] = "Failed", + [ClassName + "EmptyDataRetryTheory (Sql)"] = "Failed", + }); + } + + /// + /// A theory that declares no data is not a test that failed, it is a test that could not be + /// built, and the reason has to reach the results rather than being replaced by whatever + /// happens when the retrying case calls the method with no arguments. + /// + [Fact] + public void GivenAMalformedRetryingTheory_WhenTheRunCompletes_ThenTheReportedFailureNamesTheMissingData() + { + TestAssetRun run = TestAssetRunner.Run("RetryVariants"); + + Assert.Contains("No data found for", run.Output, System.StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRun.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRun.cs new file mode 100644 index 0000000000..765a2c7ae7 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRun.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// What a script reported. + /// + /// The exit code the leg reads to decide whether to fail. + /// Everything the script wrote, on either stream. + internal sealed record ScriptRun(int ExitCode, string Output); +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRunner.cs new file mode 100644 index 0000000000..09f419ed14 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRunner.cs @@ -0,0 +1,185 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Runs one of the repository's CI guard scripts the way the pipeline runs it. + /// + /// + /// Two of the guards that decide whether a test leg is allowed to report success are PowerShell + /// scripts rather than tests, so the only way to assert on what they do is to run them. They are + /// run from their place in the repository rather than from a copy, so what these tests cover is + /// what the pipeline invokes. + /// + internal static class ScriptRunner + { + private static readonly TimeSpan RunTimeout = TimeSpan.FromMinutes(2); + + /// + /// Runs a script and returns its exit code together with everything it wrote. + /// + /// The script to run. + /// The parameters to pass, by name without the leading dash. + public static ScriptRun Run(string scriptPath, IReadOnlyDictionary parameters) + { + var startInfo = new ProcessStartInfo("pwsh") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + // The pipeline runs these scripts with pwsh set on every task that calls them, and they + // say so themselves with a #requires line. Windows PowerShell cannot parse them, so + // running whatever "powershell" happens to mean here would test something no leg runs. + // + // They are invoked through -Command rather than -File so that a failure message can be + // read back as the script wrote it: PowerShell's default error view draws a box around a + // terminating error and rewraps the text inside it to the console width, which would + // leave assertions matching on how wide a machine's console happens to be. The exit code + // is unaffected by the view, and is what the leg actually reads. + var command = new StringBuilder("$ErrorView='NormalView'; & '").Append(scriptPath).Append('\''); + + foreach (KeyValuePair parameter in parameters) + { + command.Append(" -").Append(parameter.Key).Append(" '").Append(parameter.Value).Append('\''); + } + + command.Append("; exit $LASTEXITCODE"); + + startInfo.ArgumentList.Add("-NoProfile"); + startInfo.ArgumentList.Add("-NonInteractive"); + startInfo.ArgumentList.Add("-Command"); + startInfo.ArgumentList.Add(command.ToString()); + + using var process = new Process { StartInfo = startInfo }; + using var outputComplete = new ManualResetEventSlim(false); + using var errorComplete = new ManualResetEventSlim(false); + + var output = new StringBuilder(); + + // Both streams are drained as they arrive rather than one after the other. Reading one + // stream to the end first deadlocks as soon as the script writes more to the other than + // its pipe holds, which is not a hypothetical: these scripts report what went wrong by + // naming every test or project involved, on the error stream, while the same run is + // writing what it matched to the output stream. The wait below cannot rescue that, + // because it is only reached once the reads have finished. + void Append(DataReceivedEventArgs e, ManualResetEventSlim complete) + { + if (e.Data == null) + { + try + { + complete.Set(); + } + catch (ObjectDisposedException) + { + // The only way this event is disposed while a callback is still in flight is + // that this method has already given up waiting and is unwinding. Letting the + // exception escape would take down the test host from a thread pool thread. + } + } + else + { + lock (output) + { + output.AppendLine(e.Data); + } + } + } + + process.OutputDataReceived += (_, e) => Append(e, outputComplete); + process.ErrorDataReceived += (_, e) => Append(e, errorComplete); + + try + { + process.Start(); + } + catch (System.ComponentModel.Win32Exception e) + { + throw new InvalidOperationException( + "PowerShell 7 ('pwsh') was not found on this machine. The CI legs run this script with it, so these tests need it to run the same thing the legs do.", + e); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + bool exited = process.WaitForExit((int)RunTimeout.TotalMilliseconds); + + // Waiting for the process to exit is not the same as having been handed everything it + // wrote: the timed overload of WaitForExit returns as soon as the process is gone, while + // the handlers above are still being called on other threads. Reading the output without + // waiting for them yields whatever happened to have arrived, which on a loaded machine + // is nothing at all, and turns a passing script into an assertion about an empty string. + if (exited) + { + outputComplete.Wait(RunTimeout); + errorComplete.Wait(RunTimeout); + } + else + { + try + { + process.Kill(entireProcessTree: true); + } + catch (Exception e) when (e is InvalidOperationException or System.ComponentModel.Win32Exception) + { + // The child can exit on its own between the wait giving up and the kill landing. + } + + lock (output) + { + throw new InvalidOperationException($"'{scriptPath}' did not finish within {RunTimeout}. It reported:{Environment.NewLine}{output}"); + } + } + + lock (output) + { + return new ScriptRun(process.ExitCode, output.ToString()); + } + } + + /// + /// Resolves a path the project file recorded in assembly metadata. + /// + /// The metadata key holding the path. + public static string Resolve(string metadataKey) + { + string path = typeof(ScriptRunner).Assembly + .GetCustomAttributes() + .FirstOrDefault(a => a.Key == metadataKey) + ?.Value; + + if (string.IsNullOrEmpty(path)) + { + throw new InvalidOperationException( + $"The {metadataKey} metadata attribute is missing. It is injected by the project file and identifies a file these tests run or read."); + } + + // The project file writes this path with the separator MSBuild used on the machine that + // built it, and a backslash is an ordinary filename character everywhere except Windows. + path = Path.GetFullPath(path.Replace('\\', Path.DirectorySeparatorChar)); + + if (!File.Exists(path)) + { + throw new InvalidOperationException($"The file recorded as {metadataKey} was not found at '{path}'."); + } + + return path; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SerializedVariantCollectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SerializedVariantCollectionTests.cs new file mode 100644 index 0000000000..dcc152414c --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SerializedVariantCollectionTests.cs @@ -0,0 +1,44 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that expanding a test class into fixture argument set variants leaves the class in + /// the collection its author put it in. + /// + public class SerializedVariantCollectionTests + { + private const string FirstClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants.FirstSerializedVariantTests"; + private const string SecondClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants.SecondSerializedVariantTests"; + + /// + /// A collection is how an author says "these must not run at the same time". Giving each + /// variant its own collection would let xUnit run them concurrently, and because the + /// collection is named rather than backed by a [CollectionDefinition] class there is + /// no definition to inspect and notice the grouping. + /// + [Fact] + public void GivenClassesInOneNamedCollection_WhenTheirVariantsRun_ThenNoTwoRunConcurrently() + { + TestAssetRun run = TestAssetRunner.Run("SerializedVariants"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstClass + ".RunsWithoutOverlappingTheOtherVariants (Sql)"] = "Passed", + [FirstClass + ".RunsWithoutOverlappingTheOtherVariants (Cosmos)"] = "Passed", + [SecondClass + ".RunsWithoutOverlappingTheOtherVariants (Sql)"] = "Passed", + [SecondClass + ".RunsWithoutOverlappingTheOtherVariants (Cosmos)"] = "Passed", + [SecondClass + ".ReceivesTheFixtureItsNameClaims (Sql)"] = "Passed", + [SecondClass + ".ReceivesTheFixtureItsNameClaims (Cosmos)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs new file mode 100644 index 0000000000..2dfc8352b8 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs @@ -0,0 +1,217 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Sdk; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins the rule that resolves two attempts competing to report a single test. + /// + /// + /// An intercepting bus replays anything it still holds when it is disposed, so that a failure + /// can never be lost. The cost of that fail-safe is that a caller cannot stay silent about a + /// bus by saying nothing: the loser of the competition has to be discarded explicitly, or it + /// publishes a second result moments after the winner. When the loser is holding an abstention + /// and the winner is a failure, that second result is a skip arriving after a failure - which + /// is how a red test can be read as green. + /// + [Trait("Category", "UnitTests")] + public class SingleResultReportingTests + { + [Fact] + public void GivenAnEarlierFailureAndAnAbstainingCurrentAttempt_WhenTheEarlierAttemptWins_ThenOnlyTheFailureIsPublished() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus earlier = DeferFailure(inner); + RetryTestCase.FailureInterceptingMessageBus current = DeferAbstention(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayEarlierAttempt, current, earlier); + + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + + // Disposal is the fail-safe, and it must have nothing left to fall back on. + current.Dispose(); + earlier.Dispose(); + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + } + + [Fact] + public void GivenBothAttemptsHoldingMessages_WhenTheCurrentAttemptWins_ThenOnlyItsMessagesArePublished() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus earlier = DeferFailure(inner); + RetryTestCase.FailureInterceptingMessageBus current = DeferFailure(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayCurrentAttempt, current, earlier); + + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + + current.Dispose(); + earlier.Dispose(); + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + } + + [Fact] + public void GivenNothingObservedAFailure_WhenThereIsNothingToReport_ThenTheCurrentAbstentionSurvivesToBePublished() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus current = DeferAbstention(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReportNothing, current, earlierAttempt: null); + + // The abstention is the only result this test has, so it is left for disposal to publish + // rather than being dropped. + Assert.Empty(inner.PublishedKinds); + current.Dispose(); + Assert.Equal(new[] { nameof(ITestSkipped) }, inner.PublishedKinds); + } + + [Fact] + public void GivenAnEarlierFailureThatLostToNothing_WhenThereIsNothingToReport_ThenItIsNotPublished() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus earlier = DeferFailure(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReportNothing, currentAttempt: null, earlierAttempt: earlier); + + earlier.Dispose(); + Assert.Empty(inner.PublishedKinds); + } + + [Fact] + public void GivenAMissingBus_WhenTheRuleIsApplied_ThenItReportsThatTheRunMayContinue() + { + Assert.True(RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayEarlierAttempt, currentAttempt: null, earlierAttempt: null)); + Assert.True(RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayCurrentAttempt, currentAttempt: null, earlierAttempt: null)); + } + + [Fact] + public void GivenTheUnderlyingBusAsksToStop_WhenTheWinnerIsReplayed_ThenTheRequestIsPassedOn() + { + var inner = new RecordingMessageBus { ContinueRunning = false }; + RetryTestCase.FailureInterceptingMessageBus earlier = DeferFailure(inner); + + Assert.False(RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayEarlierAttempt, currentAttempt: null, earlierAttempt: earlier)); + } + + [Fact] + public void GivenADeliberateHandoffToDisposal_WhenTheBusIsDisposed_ThenItPublishesWithoutClaimingAnInternalError() + { + var inner = new RecordingMessageBus(); + var diagnostics = new List(); + RetryTestCase.FailureInterceptingMessageBus current = DeferAbstention(inner); + current.DiagnosticLog = diagnostics.Add; + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReportNothing, current, earlierAttempt: null); + current.Dispose(); + + Assert.Equal(new[] { nameof(ITestSkipped) }, inner.PublishedKinds); + Assert.DoesNotContain(diagnostics, d => d.Contains("Internal error", StringComparison.Ordinal)); + } + + [Fact] + public void GivenMessagesNobodyResolved_WhenTheBusIsDisposed_ThenItStillPublishesAndSaysSo() + { + var inner = new RecordingMessageBus(); + var diagnostics = new List(); + RetryTestCase.FailureInterceptingMessageBus orphaned = DeferFailure(inner); + orphaned.DiagnosticLog = diagnostics.Add; + + orphaned.Dispose(); + + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + Assert.Contains(diagnostics, d => d.Contains("Internal error", StringComparison.Ordinal)); + } + + private static RetryTestCase.FailureInterceptingMessageBus DeferFailure(RecordingMessageBus inner) + { + var bus = new RetryTestCase.FailureInterceptingMessageBus(inner, deferFailures: true); + bus.QueueMessage(new StubTestFailed()); + return bus; + } + + private static RetryTestCase.FailureInterceptingMessageBus DeferAbstention(RecordingMessageBus inner) + { + var bus = new RetryTestCase.FailureInterceptingMessageBus(inner, deferFailures: true, deferAbstentions: true); + bus.QueueMessage(new StubTestSkipped()); + return bus; + } + + private sealed class RecordingMessageBus : IMessageBus + { + private readonly List _published = new List(); + + public bool ContinueRunning { get; set; } = true; + + public IReadOnlyList PublishedKinds => _published + .ToList(); + + public bool QueueMessage(IMessageSinkMessage message) + { + _published.Add(message switch + { + ITestFailed => nameof(ITestFailed), + ITestSkipped => nameof(ITestSkipped), + _ => message.GetType().Name, + }); + + return ContinueRunning; + } + + public void Dispose() + { + } + } + + private abstract class StubResultMessage + { + public string AssemblyUniqueID => "assembly"; + + public string TestCollectionUniqueID => "collection"; + + public string TestClassUniqueID => "class"; + + public string TestMethodUniqueID => "method"; + + public string TestCaseUniqueID => "case"; + + public string TestUniqueID => "test"; + + public decimal ExecutionTime => 0m; + + public string Output => string.Empty; + + public string[] Warnings => null; + + public DateTimeOffset FinishTime => DateTimeOffset.MinValue; + + public string ToJson() => "{}"; + } + + private sealed class StubTestFailed : StubResultMessage, ITestFailed + { + public FailureCause Cause => FailureCause.Assertion; + + public int[] ExceptionParentIndices => new[] { -1 }; + + public string[] ExceptionTypes => new[] { typeof(InvalidOperationException).FullName }; + + public string[] Messages => new[] { "stub failure" }; + + public string[] StackTraces => new string[] { null }; + } + + private sealed class StubTestSkipped : StubResultMessage, ITestSkipped + { + public string Reason => "stub skip"; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs new file mode 100644 index 0000000000..152263d5d7 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins what a retrying test reports when an attempt skips after an earlier one failed. + /// + /// + /// A pass supersedes a failure held over from an earlier attempt, because a pass is a claim that + /// the code works. A skip makes no such claim - it says the attempt should not have run - so + /// letting it supersede would erase a failure the test really did show, and report NotExecuted + /// for a test that failed. The held failure therefore wins and the skip is dropped. This is + /// asserted rather than assumed because the opposite reading - that the newest result always + /// wins - is the simpler implementation and would be a silent regression. + /// + public class SkipAfterFailureTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipAfterFailure.SkipAfterFailureTests."; + + /// + /// A test that fails, is retried, and then skips itself is reported failed exactly once, with + /// the abstaining skip discarded rather than reported alongside it. + /// + [Fact] + public void GivenARetryingTestThatSkipsAfterFailing_WhenTheRunCompletes_ThenItIsReportedFailedOnce() + { + TestAssetRun run = TestAssetRunner.Run("SkipAfterFailure"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary { [ClassName + "FailsThenSkips"] = "Failed" }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipShimTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipShimTests.cs new file mode 100644 index 0000000000..e9c80f6b58 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipShimTests.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies the source-compatibility shims that keep the repository's legacy + /// SkippableFact and Skip.If call sites working on xunit.v3. + /// + public class SkipShimTests + { + private const string Prefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipShims.SkipShimTests."; + + /// + /// The shims replace a package that has no xunit.v3 release, so their behaviour is defined + /// entirely by this repository and nothing else checks it. The outcome is what matters: a + /// conditional skip reported as a failure would break every leg that skips tests it cannot + /// run, and one reported as a pass would let a test that never ran count as one that did. + /// + [Fact] + public void GivenTestsUsingTheSkipShims_WhenTheyRun_ThenEachIsReportedWithTheOutcomeItsConditionAsksFor() + { + TestAssetRun run = TestAssetRunner.Run("SkipShims"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "SkipIfTrue_IsSkipped"] = "NotExecuted", + [Prefix + "SkipIfFalse_Runs"] = "Passed", + [Prefix + "SkipIfNotFalse_IsSkipped"] = "NotExecuted", + [Prefix + "SkipIfNotTrue_Runs"] = "Passed", + [Prefix + "SkipWithReason_IsSkipped"] = "NotExecuted", + [Prefix + "SkippableTheory_SkipsPerRow(skip: True)"] = "NotExecuted", + [Prefix + "SkippableTheory_SkipsPerRow(skip: False)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + + /// + /// A skipped test is only actionable if the report says why it was skipped, so the reason + /// given at the call site has to survive as far as the output. + /// + [Fact] + public void GivenATestSkippedWithAReason_WhenItRuns_ThenTheReasonReachesTheOutput() + { + TestAssetRun run = TestAssetRunner.Run("SkipShims"); + + Assert.Contains("a distinctive skip reason", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipThenTeardownThrowsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipThenTeardownThrowsTests.cs new file mode 100644 index 0000000000..bffd02c72d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipThenTeardownThrowsTests.cs @@ -0,0 +1,46 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins what a retrying test reports when an attempt both abstains and fails in its teardown, + /// after an earlier attempt had already failed. + /// + /// + /// Holding a skip so it cannot erase an earlier failure also holds every message that follows it, + /// because a reporter finalizes a test at ITestFinished and anything forwarded afterwards is + /// dropped. That raised the question of whether an attempt could arrive carrying both an + /// abstention and a failure of its own, and be reported twice. + /// + /// It cannot, and this records why rather than leaving the reasoning to be redone. xUnit collects + /// the skip and the teardown exception into one AggregateException and publishes a single + /// ITestFailed; no ITestSkipped is ever raised, so the abstention-holding path is not entered at + /// all. That is xUnit's behaviour rather than this code's, which is exactly why it is worth + /// pinning: were it to change, the retry machinery would start seeing a case it has no branch for. + /// + /// + public class SkipThenTeardownThrowsTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipThenTeardownThrows.SkipThenTeardownThrowsTests."; + + /// + /// A test that fails, is retried, then skips itself while its teardown throws is reported + /// failed exactly once - not skipped, not twice, and not absent. + /// + [Fact] + public void GivenARetryingTestThatSkipsAndThrowsFromTeardown_WhenTheRunCompletes_ThenItIsReportedFailedOnce() + { + TestAssetRun run = TestAssetRunner.Run("SkipThenTeardownThrows"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary { [ClassName + "FailsThenSkipsWithThrowingTeardown"] = "Failed" }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetResult.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetResult.cs new file mode 100644 index 0000000000..86db377a3b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetResult.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// A single test result reported by a test asset run, as recorded in the TRX report. + /// + /// + /// The fully qualified display name the result was published under, or null when the + /// runner published a result it could not attribute to a test. + /// + /// The TRX outcome, for example Passed or Failed. + public sealed record TestAssetResult(string Name, string Outcome); +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs new file mode 100644 index 0000000000..87224de342 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs @@ -0,0 +1,84 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// The outcome of running one test asset scenario in a child process. + /// + public sealed class TestAssetRun + { + /// + /// Initializes a new instance of the class. + /// + /// The exit code of the runner process. + /// The combined standard output and standard error of the runner. + /// The results the runner published to its TRX report. + /// + /// The number of runner-level errors reported, which are counted separately from failed + /// tests and do not appear as results. + /// + /// The wall-clock time the runner process took. + public TestAssetRun(int exitCode, string output, IReadOnlyList results, int errorCount, TimeSpan duration) + { + ExitCode = exitCode; + Output = output; + Results = results; + ErrorCount = errorCount; + Duration = duration; + } + + /// + /// Gets the exit code of the runner process. + /// + public int ExitCode { get; } + + /// + /// Gets the combined standard output and standard error of the runner, for diagnostics. + /// + public string Output { get; } + + /// + /// Gets the results the runner published to its TRX report. + /// + public IReadOnlyList Results { get; } + + /// + /// Gets the number of runner-level errors the TRX report recorded, which are counted + /// separately from failed tests and are never published as results. + /// + /// + /// The Microsoft Testing Platform TRX reporter builds its summary from passed, failed, + /// skipped and timed-out counts only, and writes this counter as a constant zero, so today + /// this is a tripwire for a reporter that starts populating it rather than a check that can + /// currently fire. It is kept because a run that is wrong in this way shows nothing at all + /// in the result list, which is exactly the failure these tests would otherwise miss. + /// + public int ErrorCount { get; } + + /// + /// Gets the wall-clock time the runner process took. A scenario that cancels its own run + /// asserts on this to show the run was actually cut short, rather than reaching the same + /// outcome by running every attempt to completion. + /// + public TimeSpan Duration { get; } + + /// + /// Renders the run for use in assertion messages. + /// + /// The exit code, every result, and the runner output. + public override string ToString() + { + var lines = Results.Select(r => $" {r.Outcome} :: {r.Name ?? ""}"); + return $"exit code {ExitCode}, {Results.Count} result(s), {ErrorCount} runner error(s), took {Duration.TotalSeconds:F1}s:{Environment.NewLine}" + + string.Join(Environment.NewLine, lines) + + $"{Environment.NewLine}--- runner output ---{Environment.NewLine}{Output}"; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs new file mode 100644 index 0000000000..03e2d928d9 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs @@ -0,0 +1,104 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Assertions over what a test asset run published. + /// + public static class TestAssetRunAssertions + { + private const int SuccessExitCode = 0; + private const int FailedTestsExitCode = 2; + + /// + /// Asserts that a run published exactly the expected results and nothing else. + /// + /// + /// Comparing the whole result set, rather than counting outcomes, is what catches the + /// failure modes these tests exist for: a result that was never published, a result + /// published twice, and a result published without a display name because it was + /// attributed to a test the runner had already finished. + /// + /// The run to assert on. + /// + /// The expected results, as display name to outcome. Order is not significant. + /// + public static void PublishedExactly(TestAssetRun run, IReadOnlyDictionary expected) + { + EnsureNoRunnerErrors(run); + EnsureNoOrphanedResults(run); + EnsureExitCodeMatchesResults(run, expected); + + string[] actualEntries = run.Results + .Select(r => $"{r.Outcome} :: {r.Name}") + .OrderBy(e => e, StringComparer.Ordinal) + .ToArray(); + + string[] expectedEntries = expected + .Select(kvp => $"{kvp.Value} :: {kvp.Key}") + .OrderBy(e => e, StringComparer.Ordinal) + .ToArray(); + + string message = $"The run did not publish the expected results.{Environment.NewLine}" + + $"Expected:{Environment.NewLine} {string.Join(Environment.NewLine + " ", expectedEntries)}{Environment.NewLine}" + + $"Actual: {run}"; + + Assert.True(actualEntries.SequenceEqual(expectedEntries, StringComparer.Ordinal), message); + } + + /// + /// Fails when the TRX report recorded runner-level errors. + /// + /// + /// See : the pinned reporter writes this counter as a + /// constant zero, so this cannot fire today and is not what keeps these tests honest. + /// + /// The run to assert on. + private static void EnsureNoRunnerErrors(TestAssetRun run) + { + string message = $"The run reported {run.ErrorCount} runner-level error(s). These are counted separately " + + $"from failed tests and are never published as results, so the result list below can look correct " + + $"even though the run went wrong.{Environment.NewLine}Actual: {run}"; + + Assert.True(run.ErrorCount == 0, message); + } + + private static void EnsureNoOrphanedResults(TestAssetRun run) + { + int orphaned = run.Results.Count(r => string.IsNullOrWhiteSpace(r.Name)); + + string message = $"The run published {orphaned} result(s) with no display name, which means a result was " + + $"attributed to a test the runner had already finished.{Environment.NewLine}Actual: {run}"; + + Assert.True(orphaned == 0, message); + } + + private static void EnsureExitCodeMatchesResults(TestAssetRun run, IReadOnlyDictionary expected) + { + // The published results and the exit code are reported through different paths, so a + // child that writes a correct-looking report and then dies -- in a fixture teardown, + // say -- would otherwise be indistinguishable from a clean run. + // + // A skip is not a failure: the runner exits 0 for a run whose tests all passed or were + // skipped, so only a genuinely failing outcome may expect the failure exit code. + bool expectingFailures = expected.Values.Any(outcome => + !string.Equals(outcome, "Passed", StringComparison.Ordinal) + && !string.Equals(outcome, "NotExecuted", StringComparison.Ordinal)); + int expectedExitCode = expectingFailures ? FailedTestsExitCode : SuccessExitCode; + + string message = $"The run exited with code {run.ExitCode}, but {expectedExitCode} was expected for these " + + $"results. An unexpected code means the run ended for a reason its published results do not " + + $"show.{Environment.NewLine}Actual: {run}"; + + Assert.True(run.ExitCode == expectedExitCode, message); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs new file mode 100644 index 0000000000..5d99aa805b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -0,0 +1,364 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Xml.Linq; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Runs one scenario from the test assets assembly in a child process and reads back the + /// results it published. + /// + /// + /// The behaviours under test -- how many results a run publishes, what they are named, and + /// whether a result is published at all -- are properties of a whole xUnit run, so they cannot + /// be observed from inside the run that is asserting on them. + /// + public static class TestAssetRunner + { + private const string TrxNamespace = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + private const string ScenarioNamespacePrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios."; + + private static readonly TimeSpan RunTimeout = TimeSpan.FromMinutes(2); + + /// + /// Runs a single scenario and returns everything it reported. + /// + /// + /// The scenario folder name, which is also the last segment of the scenario's namespace. + /// + /// + /// Whether to cancel the run as soon as a test fails. Scenarios that exercise cancellation + /// rely on this to cancel the run from inside it. + /// + /// + /// Whether the runner may resolve theory data at discovery time. Turning this off is the + /// supported way to reach the delay-enumerated code path, which a theory also reaches on + /// its own when its data cannot be serialized. + /// + /// + /// A Name=Value trait filter to apply on top of the namespace filter, in the form the + /// runner takes on the command line. CI legs select their tests this way, so this is how a + /// scenario checks that it is still selected when they do. + /// + /// + /// The value to pass as the runner's thread limit, in the form it takes on the command line. + /// Scenarios that observe how much runs at once set this so the expected bound is explicit. + /// + /// + /// A Name=Value trait filter to exclude on, in the form the runner takes on the command + /// line. The repository's integration legs select their tests this way - each excludes the + /// other data store - so this is how a scenario checks that a leg excluding one value still + /// sees what it was meant to run. + /// + /// + /// The trait predicate of a query filter, in the form the runner takes inside the brackets of a + /// query - such as (DataStore=CosmosDb)&(Category=ExportLongRunning). This is the + /// compound form the repository's E2E and export legs select with, where a case runs only if it + /// carries every named trait, so a failure missing any one of them is invisible to the leg that + /// would have run the tests it stands for. The runner scopes the query to the scenario itself, + /// because a query filter cannot be combined with the plain namespace filter used otherwise. + /// + /// + /// How many attempts the runner may make at the tests that failed, in the form it takes on the + /// command line. Every CI leg runs with this on, and it changes what a run reports rather than + /// only how long it takes, so a scenario that describes a retried run has to set it. + /// + /// + /// Extra environment variables for the child process. A scenario whose behaviour differs + /// between attempts needs somewhere to record that it has already run, and the attempts are + /// separate processes. + /// + /// The exit code, output and published results of the run. + public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null, string maxThreads = null, string filterNotTrait = null, string filterQueryTraits = null, string retryFailedTests = null, IReadOnlyDictionary environment = null) + { + // The runner rejects a query filter outright once any plain filter has been added, so the + // two forms cannot be mixed. Saying that here keeps a scenario that tries from failing as an + // unhandled exception inside the asset, which reads as the asset being broken. + if (!string.IsNullOrEmpty(filterQueryTraits) && (!string.IsNullOrEmpty(filterTrait) || !string.IsNullOrEmpty(filterNotTrait))) + { + throw new ArgumentException("A query filter cannot be combined with a trait filter; express the whole selection as a query.", nameof(filterQueryTraits)); + } + + string assetsAssembly = ResolveAssetsAssembly(); + string resultsDirectory = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(resultsDirectory); + + try + { + const string trxFileName = "scenario.trx"; + + // The assets are launched through Microsoft Testing Platform because that is the + // runner CI uses: reporters differ in how they handle a result published for a + // test they have already finished, so asserting against a different runner would + // not say anything about the behaviour that actually ships. + var arguments = new List + { + "exec", + assetsAssembly, + "--parallel", + "collections", + "--report-trx", + "--report-trx-filename", + trxFileName, + "--results-directory", + resultsDirectory, + }; + + if (string.IsNullOrEmpty(filterQueryTraits)) + { + arguments.Add("--filter-namespace"); + arguments.Add(ScenarioNamespacePrefix + scenario); + } + else + { + // A query names each level of the test's identity in turn - assembly, namespace, + // class, method - before the traits, so scoping it to the scenario means matching + // any assembly and any class and method within that one namespace. + arguments.Add("--filter-query"); + arguments.Add(FormattableString.Invariant($"/*/{ScenarioNamespacePrefix}{scenario}/*/*/[{filterQueryTraits}]")); + } + + if (stopOnFail) + { + arguments.Add("--stop-on-fail"); + arguments.Add("on"); + } + + if (!preEnumerateTheories) + { + arguments.Add("--pre-enumerate-theories"); + arguments.Add("off"); + } + + if (!string.IsNullOrEmpty(filterTrait)) + { + arguments.Add("--filter-trait"); + arguments.Add(filterTrait); + } + + if (!string.IsNullOrEmpty(filterNotTrait)) + { + arguments.Add("--filter-not-trait"); + arguments.Add(filterNotTrait); + } + + if (!string.IsNullOrEmpty(maxThreads)) + { + arguments.Add("--max-threads"); + arguments.Add(maxThreads); + } + + if (!string.IsNullOrEmpty(retryFailedTests)) + { + arguments.Add("--retry-failed-tests"); + arguments.Add(retryFailedTests); + } + + (int exitCode, string output, TimeSpan duration) = Execute(arguments, environment); + + string trxPath = Path.Combine(resultsDirectory, trxFileName); + if (!File.Exists(trxPath)) + { + throw new InvalidOperationException( + $"The '{scenario}' test asset run published no TRX report. Exit code {exitCode}.{Environment.NewLine}{output}"); + } + + return ParseTrx(trxPath, exitCode, output, duration); + } + finally + { + try + { + Directory.Delete(resultsDirectory, recursive: true); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + // A leaked temp directory is not worth failing an otherwise good test over. + // Windows in particular reports a file still held open as an access violation + // rather than an IO error, so both have to be tolerated here. + } + } + } + + private static string ResolveAssetsAssembly() + { + string path = typeof(TestAssetRunner).Assembly + .GetCustomAttributes() + .FirstOrDefault(a => a.Key == "TestAssetsAssembly") + ?.Value; + + if (string.IsNullOrEmpty(path)) + { + throw new InvalidOperationException( + "The TestAssetsAssembly metadata attribute is missing. It is injected by the project file and identifies the assembly holding the scenarios."); + } + + // The project file writes this path with the separator MSBuild used on the machine that + // built it, and a backslash is an ordinary filename character everywhere except Windows. + path = Path.GetFullPath(path.Replace('\\', Path.DirectorySeparatorChar)); + + if (!File.Exists(path)) + { + throw new InvalidOperationException( + $"The test assets assembly was not found at '{path}'. It is built by the project reference to Microsoft.Health.Extensions.Xunit.TestAssets."); + } + + return path; + } + + private static (int ExitCode, string Output, TimeSpan Duration) Execute(IReadOnlyList arguments, IReadOnlyDictionary environment) + { + var startInfo = new ProcessStartInfo("dotnet") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + foreach (string argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + if (environment != null) + { + foreach (KeyValuePair variable in environment) + { + startInfo.Environment[variable.Key] = variable.Value; + } + } + + // The pipeline sets this so its own test runs land in a known place. The child would + // inherit it and could write its TRX somewhere other than --results-directory, which + // would look exactly like a run that produced no report at all. + startInfo.Environment.Remove("platformOptions__resultDirectory"); + + var output = new StringBuilder(); + using var outputComplete = new ManualResetEventSlim(false); + using var errorComplete = new ManualResetEventSlim(false); + using var process = new Process { StartInfo = startInfo }; + + void Append(DataReceivedEventArgs e, ManualResetEventSlim complete) + { + if (e.Data == null) + { + try + { + complete.Set(); + } + catch (ObjectDisposedException) + { + // The only way this event is disposed while a callback is still in flight is + // that Execute has already abandoned the wait and is unwinding on the timeout + // path. Nothing is listening for the signal any more, and letting it escape + // would take down the whole test host from a thread pool thread, replacing the + // timeout report with an unrelated crash. + } + } + else + { + lock (output) + { + output.AppendLine(e.Data); + } + } + } + + process.OutputDataReceived += (_, e) => Append(e, outputComplete); + process.ErrorDataReceived += (_, e) => Append(e, errorComplete); + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + long startedAt = Stopwatch.GetTimestamp(); + + if (!process.WaitForExit((int)RunTimeout.TotalMilliseconds)) + { + try + { + process.Kill(entireProcessTree: true); + } + catch (Exception e) when (e is InvalidOperationException or System.ComponentModel.Win32Exception) + { + // The child can exit on its own between the wait giving up and the kill landing. + // Letting that race escape would replace the timeout report -- which carries the + // child's output -- with an unrelated and far less useful exception. + } + + // The child is still writing on its background threads, so the buffer has to be + // snapshotted under the same lock its callbacks take. Reading it directly here + // would race those appends and surface as an obscure StringBuilder crash instead + // of the timeout that actually happened. + string captured; + lock (output) + { + captured = output.ToString(); + } + + throw new TimeoutException( + $"A test asset run did not finish within {RunTimeout}.{Environment.NewLine}{captured}"); + } + + TimeSpan duration = Stopwatch.GetElapsedTime(startedAt); + + // Exit does not imply the redirected streams have been drained. Waiting for the EOF + // signal on each is what makes the captured output complete; if that wait itself times + // out the output is truncated, which would otherwise quietly weaken every diagnostic + // built from it. + bool drained = outputComplete.Wait(RunTimeout) & errorComplete.Wait(RunTimeout); + + lock (output) + { + if (!drained) + { + output.AppendLine("[TestAssetRunner] The child process output was not fully drained before this run was read, so the text above may be truncated."); + } + + return (process.ExitCode, output.ToString(), duration); + } + } + + private static TestAssetRun ParseTrx(string trxPath, int exitCode, string output, TimeSpan duration) + { + XNamespace trx = TrxNamespace; + XDocument document = XDocument.Load(trxPath); + + List results = document + .Descendants(trx + "UnitTestResult") + .Select(e => new TestAssetResult((string)e.Attribute("testName"), (string)e.Attribute("outcome"))) + .ToList(); + + // Runner-level errors are counted here and nowhere else: they never become results, so + // a run that mishandled its own bookkeeping still looks tidy in the result list. + // + // A missing counter is not the same as a count of zero. Defaulting it to zero would + // turn the guard built on it into a no-op the moment the report shape changed, and the + // guard would keep passing while checking nothing. + XAttribute errorAttribute = document + .Descendants(trx + "Counters") + .Attributes("error") + .FirstOrDefault(); + + if (errorAttribute == null) + { + throw new InvalidOperationException( + $"The TRX report at '{trxPath}' has no Counters/@error attribute, so runner-level errors cannot be read from it."); + } + + return new TestAssetRun(exitCode, output, results, (int)errorAttribute, duration); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingArgumentSetsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingArgumentSetsTests.cs new file mode 100644 index 0000000000..04c2de3080 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingArgumentSetsTests.cs @@ -0,0 +1,78 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a class whose argument set attribute cannot be constructed. + /// + /// + /// Reading an attribute runs its constructor, so this is a class the discoverer cannot expand and + /// cannot interrogate the ordinary way either. The failure standing in for its lost tests still + /// has to carry the data store trait those tests would have had: the export and E2E legs select + /// positively on one, and a filter cannot match a trait that is not there, so those legs would + /// run nothing and report success with a whole class missing. + /// + public class ThrowingArgumentSetsTests + { + private const string SqlErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingArgumentSets.ThrowingArgumentSetsTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string CosmosErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingArgumentSets.ThrowingArgumentSetsTests.NeverRuns (fixture argument set discovery: Cosmos)"; + + /// + /// Unfiltered, one failure stands in for the lost tests of each declared data store. + /// + [Fact] + public void GivenAnAttributeThatCannotBeConstructed_WhenItIsDiscovered_ThenTheFaultIsReportedForEachDataStore() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingArgumentSets"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SqlErrorCaseName] = "Failed", + [CosmosErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The shape that matters. A leg naming one data store has to see the failure for that store, + /// which it can only do if the values were taken from metadata rather than from the attribute + /// the discoverer was unable to construct. + /// + [Fact] + public void GivenAnAttributeThatCannotBeConstructed_WhenALegSelectsOneDataStore_ThenThatLegStillSeesTheFailure() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingArgumentSets", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SqlErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The cause has to survive into the reported failure, or whoever reads the results is told a + /// test failed without being told that the class never expanded. + /// + [Fact] + public void GivenAnAttributeThatCannotBeConstructed_WhenItIsDiscovered_ThenTheCauseIsWrittenToTheOutput() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingArgumentSets"); + + Assert.Contains("This argument set attribute cannot be constructed.", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingAttributeConstructorTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingAttributeConstructorTests.cs new file mode 100644 index 0000000000..7b1c03b6a3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingAttributeConstructorTests.cs @@ -0,0 +1,78 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a discovery fault on a class that also carries an attribute whose constructor throws. + /// + /// + /// This is a wider hole than a trait attribute that refuses to produce its traits. A + /// declaration's attributes are constructed together, so one throwing constructor fails the read + /// for the whole declaration - and the attribute that throws need not have anything to do with + /// traits, which is what makes it easy to arrive at without meaning to. The class trait would go + /// with it, and a leg selecting positively on that trait would report success with the class's + /// tests missing. + /// + public class ThrowingAttributeConstructorTests + { + private const string FaultCase = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingAttributeConstructor.ThrowingAttributeConstructorTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string FaultCaseTwoDimensions = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingAttributeConstructor.ThrowingAttributeConstructorTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + + private static Dictionary BothFaultCases() => + new Dictionary + { + [FaultCase] = "Failed", + [FaultCaseTwoDimensions] = "Failed", + }; + + /// + /// The failure standing in for the lost tests is reported even though constructing the + /// class's attributes throws. + /// + [Fact] + public void GivenAClassWithAnAttributeConstructorThatThrows_WhenItsDiscoveryFaults_ThenTheFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingAttributeConstructor"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The class trait still selects the failure, even though the attribute that throws sits on + /// the same class. This is the shape of the export leg's filter, which requires a positive + /// Category, and the reason the sound attributes must survive the broken one. + /// + [Fact] + public void GivenAClassWithAnAttributeConstructorThatThrows_WhenALegSelectsOnTheClassTrait_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingAttributeConstructor", filterQueryTraits: "(Category=ThrowingConstructorProbe)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The argument set trait selects it too. That trait is built from the argument sets rather + /// than read from the declaration, so this pins that the failed read costs the failure + /// nothing beyond the declaration it happened on. + /// + [Fact] + public void GivenAClassWithAnAttributeConstructorThatThrows_WhenALegSelectsOnTheArgumentSet_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingAttributeConstructor", filterQueryTraits: "(AssetDataStore=Sql)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs new file mode 100644 index 0000000000..8056e9397f --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs @@ -0,0 +1,90 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a discovery fault on a method that also carries a trait attribute that throws. + /// + /// + /// The traits of the lost method are read as one dictionary, and xUnit computes them on demand, + /// so a single trait attribute that throws can take every other trait down with it. The failure + /// reported in the method's place would then be missing the very traits a leg selects on, and a + /// leg filtering positively would report success without it - the outcome reporting these + /// failures exists to prevent. + /// + public class ThrowingTraitTests + { + private const string FaultCase = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait.ThrowingTraitTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string FaultCaseTwoDimensions = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait.ThrowingTraitTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + + private static Dictionary BothFaultCases() => + new Dictionary + { + [FaultCase] = "Failed", + [FaultCaseTwoDimensions] = "Failed", + }; + + /// + /// The failure standing in for the lost method is reported even though one of its trait + /// attributes threw. + /// + [Fact] + public void GivenAMethodWithATraitAttributeThatThrows_WhenItsDiscoveryFaults_ThenTheFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingTrait"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The trait the method's own declaration named still selects the failure. This is the + /// shape of the export leg's filter, which requires a positive Category. + /// + [Fact] + public void GivenAMethodWithATraitAttributeThatThrows_WhenALegSelectsOnAnOrdinaryTrait_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingTrait", filterQueryTraits: "(Owner=ThrowingTraitOwner)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The class-level trait selects it too, so the failure is visible to a leg filtering on the + /// category the class declares. + /// + [Fact] + public void GivenAMethodWithATraitAttributeThatThrows_WhenALegSelectsOnTheClassTrait_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingTrait", filterQueryTraits: "(Category=ThrowingTraitProbe)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The argument set trait still selects the failure. This is the trait the E2E and export + /// legs filter positively on, so it is the one whose loss would let such a leg pass green + /// with the method missing - the outcome this whole path exists to prevent. + /// + [Fact] + public void GivenAMethodWithATraitAttributeThatThrows_WhenALegSelectsOnTheArgumentSet_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingTrait", filterQueryTraits: "(AssetDataStore=Sql)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/UnitTestProjectManifestTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/UnitTestProjectManifestTests.cs new file mode 100644 index 0000000000..f9da2100a2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/UnitTestProjectManifestTests.cs @@ -0,0 +1,214 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers the guard that checks the unit test leg still runs the projects it is meant to. + /// + /// + /// The unit test leg names no projects: it runs whatever **/*UnitTests/*.csproj matches. + /// A project that leaves the glob is never run, every project that stayed passes, and the leg + /// reports success. The script in + /// build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 is what turns that into a red + /// leg, by checking the projects that matched against a list of the ones expected. + /// + public class UnitTestProjectManifestTests + { + /// + /// Every project the list names still exists where it says. + /// + /// + /// The leg's own copy of this check runs against the repository the pipeline cloned. This + /// one runs against the working tree, so a rename that was not carried into the list is + /// reported by the tests rather than only by CI. + /// + [Fact] + public void GivenTheExpectedProjectList_WhenReadAgainstTheRepository_ThenEveryProjectListedExists() + { + string manifest = ScriptRunner.Resolve("UnitTestProjectManifest"); + string root = Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(manifest)))); + + string[] missing = ReadManifest(manifest) + .Where(relative => !File.Exists(Path.Combine(root, relative.Replace('/', Path.DirectorySeparatorChar)))) + .ToArray(); + + Assert.Empty(missing); + } + + /// + /// The list is not empty, which is the one shape that would accept a leg running nothing. + /// + [Fact] + public void GivenTheExpectedProjectList_WhenRead_ThenItNamesProjects() + { + Assert.NotEmpty(ReadManifest(ScriptRunner.Resolve("UnitTestProjectManifest"))); + } + + /// + /// A tree holding every expected project. + /// + [Fact] + public void GivenEveryExpectedProject_WhenChecked_ThenTheLegIsLeftGreen() + { + using var tree = new ProjectTree(); + + ScriptRun run = tree.Check(); + + Assert.Equal(0, run.ExitCode); + } + + /// + /// One project renamed and another added, which leaves the number of projects unchanged. + /// + /// + /// This is the case a count cannot see, and it is not a hypothetical one: renaming a project + /// and adding a project are both ordinary changes, and they only have to land in the same + /// pull request. The renamed project stops being tested and the count still adds up. + /// + [Fact] + public void GivenAProjectRenamedWhileAnotherIsAdded_WhenChecked_ThenTheRenamedProjectIsReported() + { + using var tree = new ProjectTree(); + + string renamed = tree.Remove(0); + tree.Add("src/Some.Brand.New.UnitTests/Some.Brand.New.UnitTests.csproj"); + + ScriptRun run = tree.Check(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains(renamed, run.Output, StringComparison.Ordinal); + } + + /// + /// A project added without being listed, which still runs and must not fail the leg. + /// + [Fact] + public void GivenAnUnlistedProject_WhenChecked_ThenTheLegIsLeftGreenAndTheProjectIsNamed() + { + using var tree = new ProjectTree(); + + tree.Add("src/Some.Brand.New.UnitTests/Some.Brand.New.UnitTests.csproj"); + + ScriptRun run = tree.Check(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("Some.Brand.New.UnitTests", run.Output, StringComparison.Ordinal); + } + + /// + /// A project whose directory suffix changed, so the glob no longer matches it. + /// + [Fact] + public void GivenAProjectDirectoryNoLongerEndingInUnitTests_WhenChecked_ThenItIsReported() + { + using var tree = new ProjectTree(); + + string moved = tree.Remove(0); + tree.Add(moved.Replace("UnitTests/", "Tests/", StringComparison.Ordinal)); + + ScriptRun run = tree.Check(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains(moved, run.Output, StringComparison.Ordinal); + } + + /// + /// A list naming nothing, which would otherwise accept any tree at all. + /// + [Fact] + public void GivenAnEmptyExpectedProjectList_WhenChecked_ThenTheLegIsFailed() + { + using var tree = new ProjectTree(); + + tree.OverwriteManifest("# every line here is a comment"); + + ScriptRun run = tree.Check(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("names no projects", run.Output, StringComparison.Ordinal); + } + + private static IReadOnlyList ReadManifest(string path) + => File.ReadAllLines(path) + .Select(line => line.Trim()) + .Where(line => line.Length > 0 && !line.StartsWith('#')) + .ToArray(); + + /// + /// A directory holding a project file for each entry of the real expected project list. + /// + private sealed class ProjectTree : IDisposable + { + private readonly string _root = Path.Combine(Path.GetTempPath(), "xunit-ext-projects", Guid.NewGuid().ToString("N")); + private readonly List _projects; + private string _manifest; + + public ProjectTree() + { + _manifest = ScriptRunner.Resolve("UnitTestProjectManifest"); + _projects = ReadManifest(_manifest).ToList(); + + foreach (string project in _projects) + { + Add(project); + } + } + + public void Add(string relativePath) + { + string full = Path.Combine(_root, relativePath.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(full)); + File.WriteAllText(full, ""); + } + + /// + /// Deletes one of the expected projects and returns the path it was listed under. + /// + public string Remove(int index) + { + string relativePath = _projects[index]; + Directory.Delete(Path.GetDirectoryName(Path.Combine(_root, relativePath.Replace('/', Path.DirectorySeparatorChar))), recursive: true); + return relativePath; + } + + /// + /// Replaces the list the check reads with one written for a single test. + /// + public void OverwriteManifest(string contents) + { + _manifest = Path.Combine(_root, "expected-projects.txt"); + File.WriteAllText(_manifest, contents); + } + + public ScriptRun Check() + => ScriptRunner.Run( + ScriptRunner.Resolve("UnitTestProjectsScript"), + new Dictionary + { + ["SourcesDirectory"] = _root, + ["ManifestPath"] = _manifest, + }); + + public void Dispose() + { + try + { + Directory.Delete(_root, recursive: true); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + // A leaked temp directory is not worth failing an otherwise good test over. + } + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs new file mode 100644 index 0000000000..be8bb1b32b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs @@ -0,0 +1,93 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers the traits a fixture argument set variant puts on the test cases discovered from it. + /// + /// + /// Every CI leg selects by trait, so these decide what a leg runs. A test case that reaches the + /// runner without the injected data store trait cannot be selected by a leg naming one, and a + /// variant that kept only the injected trait would lose the category the export and E2E legs + /// name alongside it. Either way the leg reports success with tests it was meant to run absent, + /// and nothing in its output says so. + /// + public class VariantTraitsTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.VariantTraits.VariantTraitsTests"; + + /// + /// Unfiltered, both tests are present in both variants and the malformed one fails. + /// + [Fact] + public void GivenAClassWithAMalformedTest_WhenItIsRunUnfiltered_ThenEveryVariantOfBothTestsIsReported() + { + TestAssetRun run = TestAssetRunner.Run("VariantTraits"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".MalformedTheoryIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Cosmos)"] = "Failed", + [ScenarioClass + ".MalformedTheoryIsStillReportedToAFilteringLeg (Cosmos)"] = "Failed", + [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", + [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Cosmos)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// xunit builds an error test case for a test it cannot run, and builds it with no traits at + /// all. A leg naming a data store would then select neither variant of the malformed test and + /// report success, so the traits of the variant it was discovered from have to be put back. + /// + [Fact] + public void GivenAMalformedTest_WhenALegSelectsOneDataStore_ThenItsFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("VariantTraits", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".MalformedTheoryIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The compound shape the export and E2E legs select by. A test runs only if it carries every + /// named trait, so injecting the data store trait must add to the test's own rather than + /// replace them - for the error case xunit built as much as for the healthy one, since the + /// error case gets every trait it has this way and would otherwise carry only the injected one. + /// + [Fact] + public void GivenTestsCarryingTheirOwnTrait_WhenALegSelectsOnBothTraits_ThenBothAreStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("VariantTraits", filterQueryTraits: "(AssetDataStore=Sql)&(Category=ExportLongRunning)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".MalformedTheoryIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs deleted file mode 100644 index 1893dfeb76..0000000000 --- a/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs +++ /dev/null @@ -1,27 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using EnsureThat; - -namespace Microsoft.Health.Extensions.Xunit -{ - /// - /// Placed on an assembly to indicate that the given type should be instantiated - /// before any tests are executed and disposed (if it implements IDisposable) - /// at the end of the test run. - /// - [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)] - public sealed class AssemblyFixtureAttribute : Attribute - { - public AssemblyFixtureAttribute(Type fixtureType) - { - EnsureArg.IsNotNull(fixtureType, nameof(fixtureType)); - FixtureType = fixtureType; - } - - public Type FixtureType { get; } - } -} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs index 4c60b395a0..0b5ddcf000 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs @@ -4,46 +4,36 @@ // ------------------------------------------------------------------------------------------------- using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { /// /// An XunitTestFramework implementation that allows parameterizing class fixtures with combinations of argument values. /// To use, decorate the test assembly with - /// [assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] - /// Also allows an assembly to declare one or more , which are created before any tests - /// are executed and disposed at the end of the test run. + /// [assembly: TestFramework(typeof(CustomXunitTestFramework))] + /// Assembly fixtures declared with are created before any tests run and + /// disposed at the end of the run. That is xUnit v3 behaviour inherited from the base framework, not something this + /// class adds: it previously needed a local implementation, which this assembly no longer carries. /// public sealed class CustomXunitTestFramework : XunitTestFramework { /// - /// This type's assembly name. + /// Initializes a new instance of the class. /// - public const string AssemblyName = nameof(Microsoft) + "." + - nameof(Microsoft.Health) + "." + - nameof(Microsoft.Health.Extensions) + "." + - nameof(Microsoft.Health.Extensions.Xunit); - - /// - /// The full name of this type. Intended to be used as an attribute argument. - /// - public const string TypeName = AssemblyName + "." + nameof(CustomXunitTestFramework); - - public CustomXunitTestFramework(IMessageSink messageSink) - : base(messageSink) + public CustomXunitTestFramework() + : base(configFileName: null) { } - protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) + protected override ITestFrameworkDiscoverer CreateDiscoverer(Assembly assembly) { - return new CustomXunitTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); + return new CustomXunitTestFrameworkDiscoverer(assembly); } - protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + protected override ITestFrameworkExecutor CreateExecutor(Assembly assembly) { - return new CustomXunitTestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink); + return new CustomXunitTestFrameworkExecutor(assembly); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 3d5dadde68..a1254e2f3c 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -1,127 +1,1436 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Reflection; +using System.Threading.Tasks; using EnsureThat; -using Xunit.Abstractions; +using Xunit; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { /// /// An implementation of that supports discovering tests with parameterized fixtures. /// - internal sealed class CustomXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer, ITestFrameworkDiscoverer + internal sealed class CustomXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer { - public CustomXunitTestFrameworkDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, IMessageSink diagnosticMessageSink, IXunitTestCollectionFactory collectionFactory = null) - : base(assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory) + private static readonly FieldInfo TestCaseDisplayNameField = typeof(XunitTestCase).GetField("testCaseDisplayName", BindingFlags.Instance | BindingFlags.NonPublic); + private readonly ConcurrentDictionary _variantClassCache = new(StringComparer.Ordinal); + + public CustomXunitTestFrameworkDiscoverer(Assembly assembly, IXunitTestCollectionFactory collectionFactory = null) + : base(new XunitTestAssembly(assembly, configFileName: null, assembly.GetName().Version, UniqueIDGenerator.ForAssembly(assembly.Location, null)), collectionFactory) { } - protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) + protected override async ValueTask FindTestsForType(IXunitTestClass testClass, ITestFrameworkDiscoveryOptions discoveryOptions, Func> callback) { EnsureArg.IsNotNull(testClass, nameof(testClass)); - EnsureArg.IsNotNull(messageBus, nameof(messageBus)); + EnsureArg.IsNotNull(callback, nameof(callback)); EnsureArg.IsNotNull(discoveryOptions, nameof(discoveryOptions)); - var attributeInfo = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute)).SingleOrDefault(); + var tracker = new CallbackTracker(callback); + + try + { + return await FindTestsForTypeCore(testClass, discoveryOptions, tracker); + } + catch (Exception ex) when (ShouldRethrowRatherThanReport(ex, tracker.IsCallbackFailure(ex), IsRunCancelled())) + { + // Either the run was cancelled, or the caller's own sink threw. Neither is a failure + // to expand fixture argument sets, so neither is reported as one. + throw; + } + catch (Exception ex) + { + // An exception thrown out of discovery is reported by xunit.v3 only as a diagnostic + // message, which is suppressed unless the run was started with --xunit-diagnostics. + // The class is dropped and the run still reports success, so a broken expansion would + // look exactly like a class that has no tests. Reporting a test case that fails on + // execution instead puts the fault in the results and in the exit code, where it + // cannot be missed. + // + // Reaching here means the failure was outside the per-method loop, which reports its + // own faults, so it belongs to the class as a whole and no method of it was expanded. + // + // testClass.Methods includes every public method, property accessors among them, so + // the failures are reported against those xunit would treat as tests: a fault + // reported against a property getter names nothing a reader can act on, and the + // getter carries none of the traits a CI leg selects the class's tests by. + // + // One failure is reported per lost method rather than one for the class, because a + // single failure can only carry one set of traits, and the methods of a class need + // not agree on theirs. A failure carrying every method's traits is dropped by a leg + // excluding any one of them; a failure carrying only the first method's is invisible + // to a leg selecting by another's. Either way a leg passes green with tests missing. + MethodInfo[] lostMethods = TryReadMethodsToReportAgainst(testClass); + + if (lostMethods.Length == 0) + { + // Nothing to hang a test case off. Rethrowing keeps the original xunit behaviour, + // which is all that is left. + Console.WriteLine( + $"[FixtureArgumentSets] ERROR: discovery of '{testClass.TestClassName}' failed and it declares no method to report the failure against. {ex}"); + throw; + } + + return await ReportDiscoveryFault( + testClass, + lostMethods, + tracker, + ex, + $"Discovering the fixture argument set variants of '{testClass.TestClassName}' failed, so none of its tests ran."); + } + } + + /// + /// Reads the methods a class's discovery failure should be reported against. + /// + /// + /// This runs inside the handler for a failure whose cause may be the class itself - a type + /// that cannot be loaded, or whose members cannot be reflected over. Reading its methods can + /// therefore throw the same way the discovery did, and an exception escaping a catch clause + /// replaces the original with one raised while handling it, losing the only description of + /// what actually went wrong. Returning nothing instead leaves the caller to rethrow the + /// original, which is the outcome when there is no method to report against anyway. + /// + /// The class whose discovery failed. + /// The methods to report against, empty if they could not be read. + private static MethodInfo[] TryReadMethodsToReportAgainst(IXunitTestClass testClass) + { + try + { + return SelectMethodsToReportAgainst(testClass.Methods); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] ERROR: the methods of '{testClass.TestClassName}' could not be read while reporting a discovery failure against them. {ex}"); + + return Array.Empty(); + } + } + + /// + /// Decides whether an exception thrown while expanding fixture argument sets goes back to + /// xunit unchanged, or is reported as a failing test case standing in for the tests it lost. + /// + /// + /// Reporting a fault as a test is what stops a broken expansion from leaving a run green with + /// its tests silently absent, but it is only right for a failure that belongs to the expansion. + /// Two do not. A cancelled run is not a fault at all, and turning it into a test case would + /// make pressing Ctrl+C, or a runner-imposed timeout, produce a red test that names a class + /// with nothing wrong with it. A failure raised by xunit's own callback is not ours either, + /// and the only way to report it would be to hand that same callback another test case. + /// + /// Cancellation is recognised by asking whether the run was actually cancelled, not by the + /// exception's type alone. The type on its own is not evidence: expansion runs the attributes + /// declared on the class, and one of those is free to throw + /// for reasons of its own - a timeout inside an + /// attribute that reaches out to something, say. Taking that for a cancelled run rethrows it, + /// drops the class, and leaves the run green with its tests missing, which is the exact outcome + /// this mechanism exists to prevent, reached through the one door left open for it. + /// + /// + /// This is a separate function so that the decision can be pinned directly. Reached only + /// through a catch clause, the cancellation case in particular needs a run to be cancelled at + /// the moment a class is being expanded, which no scenario can arrange reliably. + /// + /// + /// The exception that ended the expansion. + /// Whether the exception came from xunit's own callback. + /// Whether the run was actually cancelled. + /// true to rethrow; false to report the failure as a test case. + internal static bool ShouldRethrowRatherThanReport(Exception exception, bool isCallbackFailure, bool isCancellationRequested) => + (exception is OperationCanceledException && isCancellationRequested) || isCallbackFailure; + + /// + /// Reports whether the run has been cancelled, as far as it can be determined here. + /// + /// + /// Discovery is not handed a cancellation token, so this reads the ambient one xunit keeps for + /// the run. If it cannot be read the answer is no, which reports a fault rather than dropping a + /// class: a spurious red test naming a class is visible and can be argued with, while a class + /// silently absent from a green run is neither. + /// + private static bool IsRunCancelled() + { + try + { + return TestContext.Current?.CancellationToken.IsCancellationRequested ?? false; + } + catch (Exception ex) + { + Console.WriteLine($"[FixtureArgumentSets] WARNING: the run's cancellation state could not be read while handling a discovery failure, so it is treated as a fault. {ex}"); + return false; + } + } + + /// + /// Determines whether xunit would treat a method as a test. + /// + /// + /// The test is for , which is what xunit itself discovers by, and + /// not for . An attribute may implement the interface without + /// deriving from that class, and such a method is every bit as much a test: matching only the + /// class would leave it out of the methods a failure is reported against, so the tests it + /// stands for would be missing from a run that still reported success - the one outcome this + /// mechanism exists to prevent. Matching the interface can only widen the set. + /// + /// This is asked while handling a discovery failure, over the very metadata that may have + /// caused it, so a method whose attributes cannot be read answers that its test-ness is + /// unknown rather than throwing, and rather than claiming it is not a test. Those are not the + /// same answer: the attributes that cannot be read are very often the ones that caused the + /// failure being reported, and answering "not a test" there would drop the method from the + /// set a failure stands in for. + /// + /// + /// The method to inspect. + /// + /// true when the method carries a fact or theory attribute, false when it + /// certainly does not, and null when its attributes could not be read. + /// + internal static bool? IsTestMethod(MethodInfo method) + { + try + { + return method.GetCustomAttributes(inherit: true).Any(attribute => attribute is IFactAttribute); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the attributes of '{method.Name}' could not be read while choosing the methods to report a discovery failure against, so it is treated as a test. {ex}"); + return null; + } + } + + /// + /// Chooses the methods a class's discovery failure is reported against. + /// + /// + /// A method whose test-ness could not be determined is kept rather than dropped. The two + /// mistakes are not equal: reporting a failure against a helper is noise a reader can see and + /// dismiss, while dropping a method that really was a test means no failure stands in for it, + /// and a leg selecting by a trait only that method carried passes green with it absent. + /// + /// Only when no method is a test, or may be one, does this fall back to whatever the class + /// declares first, so that a failure is still reported somewhere rather than not at all. A + /// failure reported against a property accessor names nothing a reader can act on, which is + /// why it is the last resort rather than part of the set. + /// + /// + /// The methods the class declares. + /// The methods to report against. + internal static MethodInfo[] SelectMethodsToReportAgainst(IEnumerable methods) + { + EnsureArg.IsNotNull(methods, nameof(methods)); + + MethodInfo[] all = methods as MethodInfo[] ?? methods.ToArray(); + MethodInfo[] testMethods = all.Where(method => IsTestMethod(method) != false).ToArray(); + + return testMethods.Length > 0 ? testMethods : all.Take(1).ToArray(); + } + + /// + /// Reports a discovery failure as a test case that fails when the run reaches it. + /// + /// The class being discovered. + /// The methods the fault lost, each of which gets its own failures. + /// The callback discovered test cases are handed to. + /// The failure being reported. + /// A sentence saying what was lost, used to open the failure message. + /// Whether discovery should continue, as the callback reported it. + /// + /// The console line written here is the record of the fault that does not depend on anything + /// downstream. The test case is what puts the failure in the results and in the exit code, and + /// it does normally carry the cause - see the last paragraph - but only because it is anchored + /// to the variant class whose fixture can still be built. A case travels through the ordinary + /// class runner, which builds the class's fixtures first, so anchoring it anywhere the + /// fixture's argument is unavailable would aggregate a fixture failure ahead of this message + /// and report that instead. Both ways out of that were tried and are worse - anchoring the case + /// to a class of its own takes it out of the namespace and class filters that would have + /// selected the tests it replaces, and discarding the aggregated failure discards this message + /// with it, leaving the fault unreported again. + /// + /// Every failure reported here stands for exactly one thing that would have run: one method, + /// under one combination of argument set values. Nothing is merged, because a case carrying + /// more than one method's traits, or more than one combination's values, is a case a filter can + /// drop for a reason that applies to only part of what it stands for. A trait filter drops a + /// case when any of its values under the named trait matches, so a single case + /// declaring both DataStore=CosmosDb and DataStore=SqlServer is dropped by the + /// SQL leg's --filter-not-trait DataStore=CosmosDb and by the Cosmos leg's + /// --filter-not-trait DataStore=SqlServer alike - reported to nobody while both legs + /// stay green. The same holds of ordinary traits: pooling one method's + /// Category=ExportLongRunning into the failures standing for methods that do not + /// declare it would let the normal leg's Category!=ExportLongRunning drop all of them. + /// + /// + /// A case standing for a combination is anchored to that combination's variant class, the same + /// class the tests it replaces would have run in. That is what lets the failure carry its own + /// message: the executor takes the fixture's argument from the variant class it is running, + /// and only falls back to reading it off the case's traits when there is no variant class to + /// ask - a fallback that cannot tell one combination from another and refuses to guess. With + /// the argument supplied the fixture builds, nothing is aggregated ahead of the case, and the + /// report shows the discovery failure rather than a fixture that could not be constructed. + /// + /// + private async ValueTask ReportDiscoveryFault( + IXunitTestClass testClass, + IReadOnlyList lostMethods, + CallbackTracker tracker, + Exception exception, + string summary) + { + Console.WriteLine($"[FixtureArgumentSets] ERROR: {summary} Each lost test was replaced by a failing test case. {exception}"); - if (attributeInfo == null) + foreach (MethodInfo method in lostMethods) { - return base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions); + if (!await ReportDiscoveryFaultForMethod(testClass, method, tracker, exception, summary)) + { + return false; + } } - // get the class-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) - SingleFlag[][] classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attributeInfo); + return true; + } + + private async ValueTask ReportDiscoveryFaultForMethod( + IXunitTestClass testClass, + MethodInfo method, + CallbackTracker tracker, + Exception exception, + string summary) + { + // Overloads share a name, so a fault reported against each of two of them would otherwise + // give both cases the same unique ID and xunit would keep only one. + string methodKey = BuildFaultMethodKey(method); - // convert these to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) - SingleFlag[][] classLevelClosedParameterSets = CartesianProduct(classLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + IReadOnlyList> combinations = BuildFaultArgumentSetCombinations(ReadFaultArgumentSetDimensions(testClass, method)); - foreach (var method in testClass.Class.GetMethods(true)) + foreach (IReadOnlyList combination in combinations) { - IAttributeInfo fixtureParameterAttributeInfo = method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute)).SingleOrDefault(); + string displaySuffix = combination.Count == 0 + ? string.Empty + : $": {string.Join(", ", combination.Select(flag => flag.EnumValue.ToString()))}"; + string idSuffix = combination.Count == 0 + ? string.Empty + : $"-{string.Join("-", combination.Select(flag => $"{flag.EnumValue.GetType().Name}.{flag.EnumValue}"))}"; + + // The case stays on the class that failed - either the class itself, or the variant of + // it the lost tests would have run in - rather than on a class of its own, so that it + // is still selected by whatever namespace or class filter would have selected them, + // and it carries their traits so that a leg selecting by those cannot pass with the + // tests silently missing. + IXunitTestClass anchorClass = testClass; + if (combination.Count > 0) + { + SingleFlag[] closedVariant = combination.ToArray(); + string classKey = BuildVariantClassKey(BuildVariantKey(testClass.TestCollection, closedVariant), testClass.Class); + anchorClass = _variantClassCache.GetOrAdd( + classKey, + _ => new FixtureArgumentSetTestClass( + testClass.Class, + testClass.TestCollection, + closedVariant, + UniqueIDGenerator.ForTestClass(testClass.TestCollection.UniqueID, classKey))); + } + + var errorTestMethod = new XunitTestMethod( + anchorClass, + method, + Array.Empty(), + uniqueID: UniqueIDGenerator.ForTestMethod(anchorClass.UniqueID, methodKey)); + + // The test case is handed to xunit through the callback, which owns it from then on + // and disposes it with the rest of the discovered cases. +#pragma warning disable CA2000 + var errorTestCase = new DiscoveryErrorTestCase( + errorTestMethod, + $"{testClass.TestClassName}.{method.Name} (fixture argument set discovery{displaySuffix})", + $"{testClass.UniqueID}-{methodKey}-fixture-argument-set-discovery-error{idSuffix}", + errorMessage: $"{summary} {exception}", + traits: BuildFaultTraits(errorTestMethod, combination)); +#pragma warning restore CA2000 + + if (!await tracker.Invoke(errorTestCase)) + { + return false; + } + } + + return true; + } + + /// + /// Closes the argument set values a failed class declared over each other, so that each + /// combination its tests would have run as can be reported as its own failure. + /// + /// + /// The dimensions each declared argument set attribute expands to, one entry per attribute, + /// each holding that attribute's dimensions in the order it declares them. + /// + /// + /// One list per combination, deduplicated. A class declaring no argument set values yields a + /// single empty combination, so a fault is always reported at least once. + /// + /// + /// A dimension is one flags enum, taken by one of the fixture's constructor parameters, and + /// the variants an attribute expands to are the product of the single flags set within each of + /// its dimensions. Reproducing that product means a leg selecting a variant by trait selects + /// the failure standing in for it, and a leg excluding a variant by trait excludes only that + /// one and still sees the rest. + /// + /// Each attribute is closed over separately rather than every value being pooled into one + /// product, because attributes need not agree on which dimensions they use. Pooling would give + /// the failures standing in for a method that declares only a data store a format value it + /// never asked for, and a leg excluding that format would then exclude every failure the class + /// produced - passing green with those tests missing, which is the one outcome this whole + /// mechanism exists to prevent. Closing each attribute separately keeps every combination one + /// that some method really would have run as. + /// + /// + internal static IReadOnlyList> BuildFaultArgumentSetCombinations(IEnumerable argumentSetDimensions) + { + EnsureArg.IsNotNull(argumentSetDimensions, nameof(argumentSetDimensions)); - SingleFlag[][] closedSets = classLevelClosedParameterSets; + var combinations = new List>(); - if (fixtureParameterAttributeInfo != null) + // Two attributes declaring the same values would otherwise be reported twice, and the two + // failures would share a unique ID. + var seen = new HashSet(StringComparer.Ordinal); + + foreach (SingleFlag[][] attributeDimensions in argumentSetDimensions) + { + IEnumerable> dimensions = (attributeDimensions ?? Array.Empty()) + .Where(dimension => dimension != null) + .Select(dimension => (IEnumerable)dimension + .Where(flag => flag.EnumValue != null) + .GroupBy(flag => flag.EnumValue) + .Select(values => values.First()) + .ToArray()) + .Where(dimension => dimension.Any()) + .ToArray(); + + foreach (IEnumerable combination in CartesianProduct(dimensions)) { - // get the method-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) - SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttributeInfo); + SingleFlag[] closed = combination.ToArray(); + string key = string.Join( + "|", + closed + .Select(flag => $"{flag.EnumValue.GetType().FullName}.{flag.EnumValue}") + .OrderBy(text => text, StringComparer.Ordinal)); - bool hasOverride = false; - for (int i = 0; i < methodLevelOpenParameterSets.Length; i++) + if (seen.Add(key)) { - if (methodLevelOpenParameterSets[i]?.Length > 0) - { - hasOverride = true; - } - else - { - // means take the class-level set - methodLevelOpenParameterSets[i] = classLevelOpenParameterSets[i]; - } + combinations.Add(closed); } + } + } + + if (combinations.Count == 0) + { + combinations.Add(Array.Empty()); + } + + return combinations; + } + + /// + /// Reads the argument set dimensions that apply to one lost method, for use in reporting the + /// discovery failure that stands in for it. + /// + /// + /// This runs while handling a discovery failure, and reads the same attributes whose contents + /// may well be what caused it, so each attribute is read and expanded on its own and any + /// further exception costs only that one attribute's values. Reporting the failure with fewer + /// traits is worth more than not reporting it at all. + /// + /// Every attribute the class and the method declare is read, not the single one the expansion + /// would have used, because declaring two is one of the ways the expansion fails: asking for + /// the single attribute throws, and answering that with no values at all leaves the failure + /// carrying none of the traits its own declaration named. A method declaring + /// DataStore=SqlServer twice over would then reach the SQL leg with no + /// DataStore trait at all, and that leg selects positively on one - so it would match + /// nothing, and pass green with the method missing. + /// + /// + /// Only this method's own attributes are read, never a sibling's. The failure stands for this + /// method alone, and a value borrowed from a sibling is a value no test of this method would + /// have carried, which an exclusion filter can then use to drop the failure entirely. + /// + /// + /// A method's own attributes are read alongside its class's rather than merged over them the + /// way the successful path merges them. The merge cannot be trusted here - it is one of the + /// things that may have just failed - and the two directions of error are not equal. Reading + /// both over-reports: a method narrowing a dimension is also reported under the class's wider + /// values, so a leg sees a failure for a variant that would not have existed. Merging and + /// getting it wrong under-reports, and a leg then passes green with tests missing. The + /// over-reported failure is loud and traceable to this same declaration; the under-reported + /// one is silent, so the union is deliberate. + /// + /// + /// Reading an attribute runs its constructor, so an attribute whose constructor throws is both + /// a cause of a fault and, ordinarily, unreadable afterwards. The values it was given are still + /// in the assembly's metadata, which can be read without running anything, so that is the + /// fallback: the failure still carries the traits the lost tests would have had. This matters + /// because the E2E and export legs select positively on an argument set, with + /// (DataStore=SqlServer)&(...), and a positive filter cannot match a trait that is + /// not there - such a leg would match nothing and report success with a class missing. + /// + /// + /// What remains beyond reach is a declaration whose metadata cannot be read either, or whose + /// argument sets are not enum constants in the metadata at all. The failure then carries no + /// argument set trait and only the legs that filter by exclusion report it; the console output + /// above still names the class either way. + /// + /// + /// The class whose expansion failed. + /// The lost method the failure stands in for. + /// The dimensions of each declared attribute; empty when none could be read. + private static List ReadFaultArgumentSetDimensions(IXunitTestClass testClass, MethodInfo method) + { + var dimensions = new List(); + + AddAll( + () => testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false), + () => testClass.Class.GetCustomAttributesData(), + testClass.TestClassName); + AddAll( + () => method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false), + () => method.GetCustomAttributesData(), + $"{testClass.TestClassName}.{method.Name}"); + + return dimensions; - if (hasOverride) + void AddAll(Func read, Func> readMetadata, string owner) + { + object[] attributes; + try + { + attributes = read(); + } + catch (Exception ex) + { + // Reading an attribute runs its constructor, and a constructor that throws is one + // of the ways a class gets here in the first place. The values are still in the + // assembly's metadata, which can be read without running anything, so the failure + // standing in for these tests can still carry the traits a leg selects on. Without + // this the failure would reach the runner bare, and the export and E2E legs select + // positively on a data store, so they would match nothing and report success. + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the argument set attributes of '{owner}' could not be read, so their values are being taken from metadata instead. {ex}"); + + AddAllFromMetadata(readMetadata, owner); + return; + } + + // Each attribute is expanded on its own so that the one whose contents may have caused + // the fault in the first place costs only its own values, rather than leaving the + // failure with no traits for a leg's filter to select it by. + foreach (FixtureArgumentSetsAttribute attribute in attributes.OfType()) + { + try + { + dimensions.Add(ExpandFaultEnumFlagsFromAttributeData(attribute)); + } + catch (Exception ex) { - // convert to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) - closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: an argument set attribute of '{owner}' could not be expanded, so a trait filter may not select the failure standing in for its tests. {ex}"); } } + } - foreach (SingleFlag[] closedVariant in closedSets) + void AddAllFromMetadata(Func> readMetadata, string owner) + { + IEnumerable attributeData; + try { - var closedVariantTestClass = new TestClassWithFixtureArguments(testClass.TestCollection, testClass.Class, closedVariant); - var closedVariantTestMethod = new TestMethod(closedVariantTestClass, method); + attributeData = readMetadata() + .Where(d => typeof(FixtureArgumentSetsAttribute).IsAssignableFrom(d.AttributeType)) + .ToArray(); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the argument set metadata of '{owner}' could not be read either, so a trait filter may not select the failure standing in for its tests. {ex}"); + return; + } - if (!FindTestsForMethod(closedVariantTestMethod, includeSourceInformation, messageBus, discoveryOptions)) + foreach (CustomAttributeData data in attributeData) + { + try { - return false; + dimensions.Add(ExpandEnumFlagsFromMetadata(data)); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: an argument set attribute of '{owner}' could not be expanded from metadata, so a trait filter may not select the failure standing in for its tests. {ex}"); } } } + } - return true; + /// + /// Expands the argument sets an attribute declares by reading the assembly's metadata, without + /// constructing the attribute. + /// + /// + /// This is the fault path's way round an attribute whose constructor throws. It reads the same + /// enum values the constructor would have been handed, so the dimensions it produces are the + /// ones the tests would have carried had the class expanded. + /// + /// The metadata of one argument set attribute. + /// One array of single-valued flags per declared dimension. + private static SingleFlag[][] ExpandEnumFlagsFromMetadata(CustomAttributeData attributeData) + { + var dimensions = new List(); + var enumTypes = new List(); + + foreach (CustomAttributeTypedArgument argument in attributeData.ConstructorArguments) + { + // A derived attribute names its enums one per parameter, while the base takes them as + // a params array, so an argument is either an enum or a collection of them. + if (argument.Value is IReadOnlyCollection elements) + { + foreach (CustomAttributeTypedArgument element in elements) + { + AddDimension(element); + } + } + else + { + AddDimension(argument); + } + } + + // This path only ever runs while reporting a fault, so a declaration naming nothing is + // widened here for the same reason it is on the reflection path. + return WidenFaultDimensionsThatNameNothing(dimensions.ToArray(), enumTypes.ToArray()); + + void AddDimension(CustomAttributeTypedArgument argument) + { + Type argumentType = argument.ArgumentType; + + // Metadata stores an enum as its underlying value, so the declared type is what says + // which enum it was. Anything that is not an enum is not a dimension. + if (argumentType == null || !argumentType.IsEnum || argument.Value == null) + { + return; + } + + dimensions.Add(GetSingleValuedFlags((Enum)Enum.ToObject(argumentType, argument.Value)).ToArray()); + enumTypes.Add(argumentType); + } } - private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(IAttributeInfo attributeInfo) + /// + /// Collects the traits a failure standing in for a method's tests must carry to be selected by + /// the same filters they would have been. + /// + /// + /// + /// Two kinds of trait matter. The ordinary ones the method and its class declare come from the + /// anchoring method itself, and CI legs combine them with argument set values in filters such + /// as (DataStore=CosmosDb)&(Category=ExportLongRunning), so leaving either out lets + /// the leg that would have run these tests pass without them. The argument set values are the + /// ones the expansion never got far enough to produce, so they are supplied by the caller from + /// the attributes rather than read off any discovered variant. + /// + /// + /// Reading the anchor's own traits runs xunit's trait discovery, which evaluates user code and + /// the method's and class's attributes, and so can fail for the same reason the expansion did. + /// That is caught here: a failure reported with only its argument set traits is still reported, + /// whereas letting the read escape would take the whole mechanism down with it and leave the + /// class silently absent from a green run. + /// + /// + /// The method the failure is reported against. + /// The one combination of argument set values this failure stands for. + /// The traits, keyed by trait name. + private static Dictionary> BuildFaultTraits( + XunitTestMethod anchor, + IReadOnlyList argumentSetCombination) { - bool IsPowerOfTwo(long x) + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + void Add(string key, string value) { - return (x != 0) && ((x & (x - 1)) == 0); + if (!traits.TryGetValue(key, out IReadOnlyCollection values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + traits[key] = values; + } + + ((HashSet)values).Add(value); + } + + try + { + foreach (KeyValuePair> trait in anchor.Traits) + { + foreach (string value in trait.Value) + { + Add(trait.Key, value); + } + } + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the ordinary traits of '{anchor.TestClass.TestClassName}.{anchor.MethodName}' could not be read in one go, so they are being read one attribute at a time. {ex}"); + + AddFaultTraitsPerAttribute(anchor, Add); + } + + foreach (SingleFlag flag in argumentSetCombination) + { + Add(flag.EnumValue.GetType().Name, flag.EnumValue.ToString()); + } + + return traits; + } + + /// + /// Reads the ordinary traits of the method a discovery failure stands for one attribute at a + /// time, so that a single attribute refusing to produce its traits costs only its own. + /// + /// + /// xUnit computes a test method's traits on demand and returns them as one dictionary, so + /// asking for them runs every trait attribute the method and its class declare. One that + /// throws - a trait computed from configuration that is not present, say - therefore takes + /// the whole dictionary with it, leaving the failure carrying none of the traits its other + /// attributes named. That is the same silence this fault path exists to break: the export + /// leg selects positively with (DataStore=X)&(Category=Y), and a filter cannot + /// match a trait that is absent, so the leg would report success with the method missing. + /// + /// This is only the fallback. The ordinary read is tried first and is what runs for a + /// healthy class, because it is xUnit's own and needs no agreement with it about how traits + /// are gathered. + /// + /// + /// The method the failure is reported against. + /// Records one trait name and value. + private static void AddFaultTraitsPerAttribute(XunitTestMethod anchor, Action add) + { + // The collection definition is read first, and for the same reason it is read at all: + // xunit v3 gives every member of a collection the traits its definition declares, so a + // class can carry a trait that appears nowhere on the class itself. Leaving it out here + // would make the fallback quieter than the ordinary read it stands in for - the failure + // would lose exactly those traits, and a leg selecting on one would not see it. The order + // matches xunit's own: collection, then class, then method. + foreach (MemberInfo declaration in new MemberInfo[] { ReadCollectionDefinition(anchor), anchor.TestClass?.Class, anchor.Method }) + { + if (declaration == null) + { + continue; + } + + ITraitAttribute[] traitAttributes; + + try + { + traitAttributes = declaration.GetCustomAttributes(inherit: true).OfType().ToArray(); + } + catch (Exception ex) + { + // Reading the attributes constructs all of them, so one constructor that throws + // costs this declaration every trait it named - including traits belonging to + // attributes that are perfectly sound, and including the case where the throwing + // attribute is not a trait attribute at all. Losing the lot is what makes the + // failure invisible to a leg selecting positively on a trait, so fall back to + // constructing them one type at a time and keep whichever ones can be read. + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the trait attributes of '{declaration.Name}' could not all be read together, so they are being read one at a time. {ex}"); + + traitAttributes = ReadTraitAttributesIndividually(declaration); + } + + foreach (ITraitAttribute traitAttribute in traitAttributes) + { + try + { + foreach (KeyValuePair trait in traitAttribute.GetTraits()) + { + add(trait.Key, trait.Value); + } + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the trait attribute '{traitAttribute.GetType().Name}' of '{declaration.Name}' could not produce its traits, so a filter naming those traits may not select the failure standing in for its tests. {ex}"); + } + } + } + } + + /// + /// Reads a declaration's trait attributes one type at a time, keeping the ones that can be + /// constructed. + /// + /// + /// This runs only when reading them together has already failed. The attribute types are + /// taken from metadata, which names them without constructing anything, so a constructor + /// that throws costs only the attributes of its own type rather than every attribute on the + /// declaration. Attributes inherited from a base declaration are not enumerated here: doing + /// so means walking the hierarchy by hand, and the ordinary read - which this only stands in + /// for - already covers them. + /// + /// The collection definition, class, or method being read. + /// The trait attributes that could be constructed, which may be none. + private static ITraitAttribute[] ReadTraitAttributesIndividually(MemberInfo declaration) + { + List traitAttributeTypes; + + try + { + traitAttributeTypes = declaration.CustomAttributes + .Select(attribute => attribute.AttributeType) + .Where(type => typeof(ITraitAttribute).IsAssignableFrom(type)) + .Distinct() + .ToList(); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the trait attributes of '{declaration.Name}' could not be named, so a trait filter may not select the failure standing in for its tests. {ex}"); + return Array.Empty(); + } + + var traitAttributes = new List(); + + foreach (Type traitAttributeType in traitAttributeTypes) + { + try + { + traitAttributes.AddRange(Attribute.GetCustomAttributes(declaration, traitAttributeType, inherit: true).OfType()); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the trait attribute '{traitAttributeType.Name}' of '{declaration.Name}' could not be constructed, so a filter naming the traits it declares may not select the failure standing in for its tests. {ex}"); + } + } + + return traitAttributes.ToArray(); + } + + /// + /// Reads the collection definition class the method's collection was declared by, if any. + /// + /// + /// A collection assembled without a definition class has none, which is the ordinary case for + /// a class that never joined one. Read defensively because this runs while handling a failure + /// that may itself have come from a type that cannot be loaded. + /// + /// The method the failure is reported against. + /// The collection definition class, or null if there is not one. + private static Type ReadCollectionDefinition(XunitTestMethod anchor) + { + try + { + return anchor.TestClass?.TestCollection?.CollectionDefinition; + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the collection definition of '{anchor.TestClass?.TestClassName}' could not be read, so a filter naming a trait it declares may not select the failure standing in for its tests. {ex}"); + return null; + } + } + + /// + /// Builds the part of a discovery failure's unique ID that identifies the method it is + /// reported against. + /// + /// + /// The method's name alone is not unique: a class may overload a test method, and two failures + /// sharing a unique ID leave xunit reporting only one of them - the other silently absent, + /// which is the outcome reporting these failures exists to prevent. The parameter types and + /// the generic arity are included to tell overloads apart - overloads may differ by arity + /// alone, as Run<T>() and Run<T, U>() do, and both take no parameters + /// - and are read defensively because this runs while handling a failure that may itself be a + /// type that cannot be loaded. + /// + /// The method the failure is reported against. + /// A key identifying the method, falling back to its name alone. + internal static string BuildFaultMethodKey(MethodInfo method) + { + try + { + string name = method.IsGenericMethodDefinition + ? $"{method.Name}`{method.GetGenericArguments().Length}" + : method.Name; + + ParameterInfo[] parameters = method.GetParameters(); + if (parameters.Length == 0) + { + return name; + } + + return $"{name}({string.Join(",", parameters.Select(parameter => parameter.ParameterType.FullName ?? parameter.ParameterType.Name))})"; + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the signature of '{method.Name}' could not be read, so its failure is identified by metadata token instead. {ex}"); + + try + { + // The token distinguishes overloads within an assembly without reading any type + // the failure may have been caused by being unable to load. Falling back to the + // name alone would let two overloads share a unique ID, and xunit reports only + // one of a colliding pair - losing the very failure this is reporting. + return $"{method.Name}#{method.MetadataToken}"; + } + catch (Exception tokenException) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the metadata token of '{method.Name}' could not be read either, so a failure reported against an overload of it may be dropped as a duplicate. {tokenException}"); + return method.Name; + } + } + } + + /// + /// Reads the fixture argument set attribute a method declares. + /// + /// The method to inspect. + /// The attribute, or null when the method declares none. + private static FixtureArgumentSetsAttribute GetMethodArgumentSetsAttribute(MethodInfo method) => + method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; + + /// + /// Decides whether a class has to be expanded method by method rather than handed to xunit + /// unchanged. + /// + /// The method to inspect. + /// true when the method declares argument sets, or when reading them failed. + /// + /// A method whose attribute cannot even be read answers true so that the class takes the + /// expansion path, where the same read is repeated inside the per-method loop and its failure + /// is reported against that one method. Swallowing the exception here rather than there is what + /// keeps one misdeclared method from costing the whole class its tests; it is not lost, only + /// deferred to the place that can attribute it. + /// + private static bool RequiresPerMethodExpansion(MethodInfo method) + { + try + { + return GetMethodArgumentSetsAttribute(method) != null; + } + catch (Exception) + { + return true; + } + } + + private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, ITestFrameworkDiscoveryOptions discoveryOptions, CallbackTracker tracker) + { + var attribute = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; + + if (attribute == null && !testClass.Methods.Any(RequiresPerMethodExpansion)) + { + return await base.FindTestsForType(testClass, discoveryOptions, tracker.Invoke); } - IEnumerable GetSingleValuedFlags(Enum e) + SingleFlag[][] classLevelOpenParameterSets = Array.Empty(); + SingleFlag[][] classLevelClosedParameterSets = Array.Empty(); + + if (attribute != null) { - if (e is null) + // get the class-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) + classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attribute); + + // convert these to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) + classLevelClosedParameterSets = CartesianProduct(classLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + } + + foreach (var method in testClass.Methods) + { + // Each method is expanded on its own so that one method's misdeclared argument set + // costs only that method. Letting the failure out of the loop would leave the methods + // already published running while the methods after it disappeared, under a single + // failure claiming the whole class had not run. The method's own attribute is read + // inside the try for the same reason: a method carrying two of them, or one whose + // constructor throws, is that method's fault and not the class's. + bool continueDiscovery; + try + { + continueDiscovery = await FindTestsForMethodCore(testClass, method, attribute, GetMethodArgumentSetsAttribute(method), classLevelOpenParameterSets, classLevelClosedParameterSets, discoveryOptions, tracker); + } + catch (Exception ex) when (!ShouldRethrowRatherThanReport(ex, tracker.IsCallbackFailure(ex), IsRunCancelled())) + { + continueDiscovery = await ReportDiscoveryFault( + testClass, + new[] { method }, + tracker, + ex, + $"Discovering the fixture argument set variants of '{testClass.TestClassName}.{method.Name}' failed, so none of that method's tests ran. Other methods of the class were discovered normally."); + } + + if (!continueDiscovery) { - yield break; + return false; } + } + + return true; + } + + private async ValueTask FindTestsForMethodCore( + IXunitTestClass testClass, + MethodInfo method, + FixtureArgumentSetsAttribute attribute, + FixtureArgumentSetsAttribute fixtureParameterAttribute, + SingleFlag[][] classLevelOpenParameterSets, + SingleFlag[][] classLevelClosedParameterSets, + ITestFrameworkDiscoveryOptions discoveryOptions, + CallbackTracker tracker) + { + if (attribute == null && fixtureParameterAttribute == null) + { + var passthroughTestMethod = new XunitTestMethod(testClass, method, Array.Empty(), uniqueID: UniqueIDGenerator.ForTestMethod(testClass.UniqueID, method.Name)); + return await FindTestsForMethod(passthroughTestMethod, discoveryOptions, tracker.Invoke); + } + + SingleFlag[][] closedSets = classLevelClosedParameterSets; + + if (attribute == null) + { + // Method-level parameter sets with no class-level fallback. + SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); + closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + } + else if (fixtureParameterAttribute != null) + { + // get the method-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) + SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); - var enumAsLong = Convert.ToInt64(e); + // The merge runs over every dimension either side declares, not just the ones the + // method does. A method carrying an attribute with fewer dimensions than its class - + // which happens when the two carry different attribute types - would otherwise have + // the class's trailing dimensions dropped rather than inherited, and its tests would + // then run without the values, and so without the traits, that a CI leg selects them + // by: the leg would pass with those tests silently absent. + int dimensionCount = Math.Max(methodLevelOpenParameterSets.Length, classLevelOpenParameterSets.Length); + var mergedOpenParameterSets = new SingleFlag[dimensionCount][]; - foreach (Enum value in Enum.GetValues(e.GetType())) + bool hasOverride = false; + for (int i = 0; i < dimensionCount; i++) { - var flagAsLong = Convert.ToInt64(value); - if (IsPowerOfTwo(flagAsLong)) + SingleFlag[] methodLevelDimension = i < methodLevelOpenParameterSets.Length ? methodLevelOpenParameterSets[i] : null; + + if (methodLevelDimension?.Length > 0) { - if ((enumAsLong & flagAsLong) != 0) + SingleFlag[] classLevelDimension = i < classLevelOpenParameterSets.Length ? classLevelOpenParameterSets[i] : null; + if (classLevelDimension?.Length > 0) { - yield return new SingleFlag(value); + Type methodLevelType = methodLevelDimension[0].EnumValue?.GetType(); + Type classLevelType = classLevelDimension[0].EnumValue?.GetType(); + + if (methodLevelType != classLevelType) + { + // Dimensions are matched by position, but the fixture's constructor + // arguments are matched by enum type. A method whose attribute takes a + // different enum in this position is therefore not overriding the + // class's dimension but replacing it with an unrelated one, and the + // variants would carry no value - and so no trait - for the dimension + // the class declared. A leg selecting on it would then run none of + // this method's tests and still pass. Only the convention that a class + // and its methods use argument set attributes taking the same enums in + // the same order keeps that from happening, so it is checked rather + // than assumed. + throw new InvalidOperationException( + $"'{testClass.TestClassName}.{method.Name}' declares '{methodLevelType}' where its class declares '{classLevelType}' in argument set dimension {i}. " + + "A method-level fixture argument set attribute must take the same enums, in the same order, as the class-level one it overrides."); + } } + + hasOverride = true; + mergedOpenParameterSets[i] = methodLevelDimension; } + else + { + // Take the class-level set: this position is either one the method left empty + // or one it says nothing about at all. A method that declares more dimensions + // than its class and leaves the extra one empty is asking to inherit a + // dimension that does not exist, so there is nothing to take. Reading past the + // end would throw too, but with a message naming an array index rather than the + // attribute that is wrong, and this exception is what the developer is shown as + // the body of the failing test standing in for the method. + if (i >= classLevelOpenParameterSets.Length) + { + throw new InvalidOperationException( + $"'{testClass.TestClassName}.{method.Name}' declares {methodLevelOpenParameterSets.Length} fixture argument set dimensions where its class declares {classLevelOpenParameterSets.Length}, and names no value in dimension {i}. " + + "A dimension naming no value inherits the class-level one in that position, and the class has no dimension there. " + + "Either name the values that dimension should take, or drop it so the method declares no more dimensions than its class."); + } + + // That throw is caught by the per-method handler in FindTestsForTypeCore and + // reported as a failing test case standing in for this method, rather than + // losing the method quietly. + mergedOpenParameterSets[i] = classLevelOpenParameterSets[i]; + } + } + + if (hasOverride) + { + // convert to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) + closedSets = CartesianProduct(mergedOpenParameterSets).Select(e => e.ToArray()).ToArray(); + } + } + + if (closedSets.Length == 0) + { + // Reaching here means an argument set was declared, so a product of nothing is a + // misconfiguration -- an argument set of zero, a value naming no single flag, or a + // null entry -- and every test on this method would otherwise be absent from a run + // that still reported success. Throwing hands it to the per-method handler in + // FindTestsForTypeCore, which reports it as a failing test case so it shows up in + // the results and in the exit code. Discovery over every test assembly in this + // repository produces no such case, so this only fires on a class that is genuinely + // misdeclared. + throw new InvalidOperationException( + $"'{testClass.TestClassName}.{method.Name}' expanded to no fixture argument sets, so none of its tests would run. " + + "Check that every argument set names at least one single-valued flag of a [Flags] enum."); + } + + foreach (SingleFlag[] closedVariant in closedSets) + { + // Every variant stays in the class's original collection. xUnit's unit of + // parallelization is the collection, so giving each variant its own would let + // classes an explicit [Collection] deliberately grouped together -- and the + // variants of a single class -- start running concurrently. Only the identifiers + // below carry the variant, so the collection keeps its original grouping. + var variantKey = BuildVariantKey(testClass.TestCollection, closedVariant); + var classKey = BuildVariantClassKey(variantKey, testClass.Class); + var closedVariantTestClass = _variantClassCache.GetOrAdd( + classKey, + _ => new FixtureArgumentSetTestClass( + testClass.Class, + testClass.TestCollection, + closedVariant, + UniqueIDGenerator.ForTestClass(testClass.TestCollection.UniqueID, classKey))); + + var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); + + closedVariantTestMethod.ApplyArgumentSetTraits(); + + // xUnit builds the display name from the class and method names only, so every + // variant of a method would otherwise be reported under an identical name and + // a failure could not be attributed to a specific fixture argument set. + var variantArguments = closedVariant; + var variantMethod = closedVariantTestMethod; + Func> variantCallback = testCase => + { + ApplyVariantDisplayName(testCase, variantArguments); + ApplyVariantTraits(testCase, variantMethod); + return tracker.Invoke(testCase); + }; + + if (!await FindTestsForMethod(closedVariantTestMethod, discoveryOptions, variantCallback)) + { + return false; } } - return attributeInfo - .GetConstructorArguments() - .Cast() + return true; + } + + /// + /// Appends the fixture argument set to a discovered test case's display name, so that the + /// variants of a test can be told apart in test results. + /// + /// The discovered test case. + /// The fixture arguments the test case was discovered for. + private static void ApplyVariantDisplayName(ITestCase testCase, SingleFlag[] closedVariant) + { + if (closedVariant.Length == 0) + { + return; + } + + if (testCase is not XunitTestCase xunitTestCase) + { + // Without the suffix every variant of this method reports under the same name, so a + // failure cannot be attributed to a fixture argument set. Every test case type + // xunit.v3 produces derives from XunitTestCase, so this only happens if a custom + // discoverer introduces its own type. Fail rather than carry on: the per-method + // handler turns this into a reported failure per fixture argument set, which is + // visible to a filtering leg, whereas a warning on stdout is not. + throw new InvalidOperationException( + $"Test case '{testCase.TestCaseDisplayName}' is {testCase.GetType().Name}, not {nameof(XunitTestCase)}, " + + "so its fixture argument set can be applied to neither its display name nor its traits. " + + "Variants of this test would be indistinguishable in results and invisible to any CI leg selecting on a trait."); + } + + if (TestCaseDisplayNameField == null) + { + throw new InvalidOperationException( + "Unable to name fixture argument set variants because XunitTestCase.testCaseDisplayName was not found. " + + "This usually means the xunit.v3 version changed; see Microsoft.Health.Extensions.Xunit."); + } + + var suffix = $" ({string.Join(", ", closedVariant.Select(argument => argument.EnumValue))})"; + var displayName = xunitTestCase.TestCaseDisplayName; + + if (displayName.EndsWith(suffix, StringComparison.Ordinal)) + { + return; + } + + TestCaseDisplayNameField.SetValue(xunitTestCase, displayName + suffix); + } + + /// + /// Copies the variant's traits onto a discovered test case that did not get them. + /// + /// + /// A test case built from a method normally carries that method's traits, so for a healthy + /// test this adds nothing. The case that matters is the one xunit builds for a test it cannot + /// run - a fact declaring parameters, a theory with no data - which it reports as an + /// ExecutionErrorTestCase constructed with no traits at all. Without them that failure + /// is invisible to any leg selecting on a trait: this repository's E2E and export legs each + /// require a positive DataStore, and a filter cannot match a trait that is absent. The + /// malformed test would then be missing from those legs and they would report success, which + /// is precisely the outcome the rest of this class exists to prevent. + /// + /// Traits are merged rather than replaced so that anything the discovery already attached - + /// a theory row's own traits, say - survives. + /// + /// + /// The discovered test case. + /// The variant method the test case was discovered from. + private static void ApplyVariantTraits(ITestCase testCase, FixtureArgumentSetTestMethod variantMethod) + { + if (testCase is not XunitTestCase xunitTestCase) + { + // A case whose traits cannot be written is one a filtering leg cannot see, which is + // the failure this class exists to prevent -- so fail discovery rather than let it + // through untagged. ApplyVariantDisplayName rejects the same case types, but it + // returns before that check when the variant closes no argument set, so this guard + // is reached rather than redundant. + throw new InvalidOperationException( + $"The fixture argument set traits of '{testCase.TestCaseDisplayName}' could not be applied because it is " + + $"{testCase.GetType().Name}, not {nameof(XunitTestCase)}. A CI leg selecting on those traits would not run it."); + } + + foreach (KeyValuePair> trait in variantMethod.Traits) + { + if (!xunitTestCase.Traits.TryGetValue(trait.Key, out HashSet values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + xunitTestCase.Traits[trait.Key] = values; + } + + foreach (string value in trait.Value) + { + values.Add(value); + } + } + } + + /// + /// Builds a key that distinguishes one fixture argument set from another within a collection. + /// + /// The collection the test class belongs to. + /// The fixture arguments the variant was closed over. + /// A stable key for the variant. + private static string BuildVariantKey(IXunitTestCollection sourceCollection, IReadOnlyList closedVariant) + { + var variantKey = string.Join( + ",", + closedVariant.Select(argument => $"{argument.EnumValue.GetType().AssemblyQualifiedName}={Convert.ToInt64(argument.EnumValue)}")); + + return $"{sourceCollection.UniqueID}|{variantKey}"; + } + + /// + /// Builds a key that identifies one variant of one test class. + /// + /// The key returned by . + /// The test class the variant was derived from. + /// A stable key for the variant class. + /// + /// The variants of a class share a collection and a class name, so the class name alone no + /// longer separates them and this key is what the variant class cache is keyed on. It is + /// also passed as the seed for the class's unique ID, but only reaches it for a variant that + /// carries no arguments: recomputes the unique ID + /// from its own argument values whenever it has any, which is the usual case. + /// + private static string BuildVariantClassKey(string variantKey, Type testClass) + { + return $"{variantKey}|{testClass.AssemblyQualifiedName}"; + } + + private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(FixtureArgumentSetsAttribute attribute) + { + IReadOnlyList argumentSets = attribute.GetArgumentSets(); + + // Dimensions are declared by position but bound to the fixture's constructor by enum + // type, and xUnit's own fixture cache is keyed by type as well, so two dimensions of the + // same type cannot be told apart. The second one is dropped when the fixture is built, + // yet the variant is still named and traited for both: every combination would be + // reported as having run with values the fixture never received, and the combinations + // that were supposed to differ would all be the same run. That is a test suite reporting + // coverage it does not have, so the declaration is refused here. + Type duplicated = argumentSets + .Where(argument => argument != null) + .GroupBy(argument => argument.GetType()) + .FirstOrDefault(group => group.Count() > 1) + ?.Key; + + if (duplicated != null) + { + throw new InvalidOperationException( + $"'{attribute.GetType().Name}' declares more than one fixture argument set dimension of type '{duplicated}'. " + + "Fixture arguments are matched to a fixture constructor by their enum type, so dimensions sharing a type cannot be told apart: " + + "each variant would be built with the first value alone while being reported under every combination. Give each dimension its own enum type."); + } + + return argumentSets + .Select(e => GetSingleValuedFlags(e).ToArray()) + .ToArray(); + } + + /// + /// Expands an argument set declaration for the fault path, widening a dimension that names no + /// value to every value its type declares. + /// + /// + /// A dimension naming no value expands to nothing, so the product of the dimensions is empty + /// and the failure standing in for the method carries no argument set trait at all - or, when + /// another dimension does name a value, carries only that one and stays silent about this. The + /// E2E and export legs select positively on the data store trait, so such a failure is + /// invisible to them: the leg reports success with the method's tests simply absent, which is + /// the outcome this whole path exists to prevent. The value is unknown but the type is not - a + /// dimension is declared as an enum whether or not it names any of its flags - so the failure + /// is reported once per value that type declares instead. That over-reports, in the same + /// deliberate direction as the rest of this path: a leg seeing a failure for a variant that + /// would not have existed is loud and traceable to this declaration, where a leg seeing + /// nothing is not. + /// + /// The declaration to expand. + /// One dimension per argument set, each holding at least one value where the type is known. + private static SingleFlag[][] ExpandFaultEnumFlagsFromAttributeData(FixtureArgumentSetsAttribute attribute) + { + IReadOnlyList argumentSets = attribute.GetArgumentSets(); + SingleFlag[][] dimensions = argumentSets .Select(e => GetSingleValuedFlags(e).ToArray()) .ToArray(); + + return WidenFaultDimensionsThatNameNothing(dimensions, argumentSets.Select(e => e?.GetType()).ToArray()); + } + + /// + /// Replaces each dimension that names no value with every value its type declares, so the + /// failure standing in for the method can still be selected on that dimension. + /// + /// + /// A dimension naming nothing is widened whether or not the dimensions beside it name + /// anything. Naming nothing in one position may be how a method asks to inherit that + /// position from its class, but the class's own declaration is closed over separately and + /// contributes its combinations regardless, so nothing is lost by also reporting this + /// declaration across the position it left open - and a class that declares nothing leaves + /// the position open with no second declaration to fill it. Not widening it there is what + /// makes the failure carry the dimension that was declared and say nothing about the one + /// that was not, which a leg selecting positively on the silent dimension cannot match. + /// + /// The alternative is the outcome this whole path exists to prevent: the method produces no + /// tests, the leg selects nothing standing in for them, and the run is green and empty. A + /// failure reported under a value the declaration did not name is loud and traceable back to + /// this declaration, which is the direction every trade-off on the fault path is made in. + /// + /// + /// The dimensions as declared. + /// The type of each dimension, positionally, with null where unknown. + /// The dimensions to report the failure against. + private static SingleFlag[][] WidenFaultDimensionsThatNameNothing(SingleFlag[][] dimensions, Type[] enumTypes) + { + if (dimensions.Length == 0 || dimensions.All(dimension => dimension.Length > 0)) + { + return dimensions; + } + + return dimensions + .Select((dimension, index) => dimension.Length > 0 + ? dimension + : AllSingleValuedFlagsOf(index < enumTypes.Length ? enumTypes[index] : null)) + .ToArray(); + } + + /// + /// Lists every single-valued flag an enum type declares, for reporting a failure whose own + /// declaration named none of them. + /// + /// The type to read, which may be null or not an enum. + /// The type's single-valued flags, or nothing if they cannot be read. + private static SingleFlag[] AllSingleValuedFlagsOf(Type enumType) + { + if (enumType == null || !enumType.IsEnum) + { + return Array.Empty(); + } + + try + { + return Enum.GetValues(enumType) + .Cast() + .Where(value => + { + long asLong = Convert.ToInt64(value); + return (asLong != 0) && ((asLong & (asLong - 1)) == 0); + }) + .Select(value => new SingleFlag(value)) + .ToArray(); + } + catch (Exception ex) + { + // Reading the type's values is the only thing that can fail here, and a failure leaves + // the dimension as it was: the failure is still reported, just without this trait. + // Losing the trait is the thing being guarded against, so it is said out loud. + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: '{enumType.FullName}' names no value in this declaration and its values could not be read, so the failure reported in the method's place carries no trait for that dimension and a leg selecting positively on it will not see the failure. {ex}"); + + return Array.Empty(); + } + } + + /// + /// Splits a flags enum value into the single-valued flags it names. + /// + /// The value to split, which may be null. + /// One per flag set in the value. + private static IEnumerable GetSingleValuedFlags(Enum e) + { + bool IsPowerOfTwo(long x) + { + return (x != 0) && ((x & (x - 1)) == 0); + } + + if (e is null) + { + yield break; + } + + var enumAsLong = Convert.ToInt64(e); + + foreach (Enum value in Enum.GetValues(e.GetType())) + { + var flagAsLong = Convert.ToInt64(value); + if (IsPowerOfTwo(flagAsLong) && (enumAsLong & flagAsLong) != 0) + { + yield return new SingleFlag(value); + } + } } /// @@ -140,5 +1449,40 @@ public static IEnumerable> CartesianProduct(IEnume emptyProduct, (accumulator, sequence) => accumulator.SelectMany(a => sequence.Select(s => a.Concat(Enumerable.Repeat(s, 1))))); } + + /// + /// Wraps the callback discovered test cases are handed to, so that a failure inside it can be + /// told apart from a failure to expand fixture argument sets. + /// + /// + /// The callback belongs to xunit rather than to this discoverer. Reporting a failure inside it + /// as a fixture argument set fault would name the wrong culprit, and would try to report that + /// through the very callback that has just thrown. + /// + private sealed class CallbackTracker + { + private readonly Func> _callback; + private Exception _failure; + + public CallbackTracker(Func> callback) + { + _callback = callback; + } + + public async ValueTask Invoke(ITestCase testCase) + { + try + { + return await _callback(testCase); + } + catch (Exception ex) + { + _failure = ex; + throw; + } + } + + public bool IsCallbackFailure(Exception exception) => ReferenceEquals(exception, _failure); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index a00c512da7..d6656d1984 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -1,20 +1,19 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; -using System.Diagnostics; +using System.Globalization; using System.Linq; using System.Reflection; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using EnsureThat; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { @@ -23,228 +22,416 @@ namespace Microsoft.Health.Extensions.Xunit /// internal sealed class CustomXunitTestFrameworkExecutor : XunitTestFrameworkExecutor { - public CustomXunitTestFrameworkExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink) - : base(assemblyName, sourceInformationProvider, diagnosticMessageSink) + /// + /// The environment variable Assert reads --assert-equivalent-max-depth from. + /// + /// + /// xunit keeps these names in an internal class, so they are repeated here rather than + /// referenced. AssertionFormattingEnvironmentVariableTests compares them back against + /// that class, so a rename in a future xunit version fails a test rather than turning these + /// options silently back off. + /// + internal const string AssertEquivalentMaxDepthVariable = "XUNIT_ASSERT_EQUIVALENT_MAX_DEPTH"; + + /// The environment variable Assert reads --print-max-enumerable-length from. + internal const string PrintMaxEnumerableLengthVariable = "XUNIT_PRINT_MAX_ENUMERABLE_LENGTH"; + + /// The environment variable Assert reads --print-max-object-depth from. + internal const string PrintMaxObjectDepthVariable = "XUNIT_PRINT_MAX_OBJECT_DEPTH"; + + /// The environment variable Assert reads --print-max-object-member-count from. + internal const string PrintMaxObjectMemberCountVariable = "XUNIT_PRINT_MAX_OBJECT_MEMBER_COUNT"; + + /// The environment variable Assert reads --print-max-string-length from. + internal const string PrintMaxStringLengthVariable = "XUNIT_PRINT_MAX_STRING_LENGTH"; + + public CustomXunitTestFrameworkExecutor(Assembly assembly) + : base(new XunitTestAssembly(assembly, configFileName: null, assembly.GetName().Version, UniqueIDGenerator.ForAssembly(assembly.Location, null))) { - AssemblyInfo = new CustomAssemblyInfo(AssemblyInfo); } - protected override void RunTestCases(IEnumerable testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + /// + /// Runs the discovered test cases through the fixture argument set assembly runner. + /// + /// The test cases to run. + /// The sink execution messages are reported to. + /// The options the run was started with. + /// Cancels the run. + /// A task that completes when the run does. + /// + /// Substituting the assembly runner means not going through the base implementation, which is + /// also where xunit copies the assertion formatting options into the environment that + /// Assert reads them from. Skipping that step would leave options such as + /// --print-max-object-depth accepted on the command line and then silently ignored, so + /// the same copy is made here before the run starts. + /// + public override async ValueTask RunTestCases(IReadOnlyCollection testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions, CancellationToken cancellationToken) { - using (var assemblyRunner = new AssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions)) + EnsureArg.IsNotNull(executionOptions, nameof(executionOptions)); + + foreach (KeyValuePair variable in BuildAssertionFormattingEnvironment(executionOptions)) { - assemblyRunner.RunAsync().GetAwaiter().GetResult(); + SetEnvironment(variable.Key, variable.Value); } + + var runner = new FixtureArgumentSetAssemblyRunner(); + await runner.Run(TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); } - private sealed class AssemblyRunner : XunitTestAssemblyRunner + /// + /// Pairs each assertion formatting option with the environment variable Assert reads it + /// from. + /// + /// + /// The pairing is the part that can go wrong silently. Every one of these options is an + /// , so two of them swapped still compiles, still sets both variables and + /// still leaves the names correct - it only makes assertion failure messages truncate at the + /// wrong place, which no test asserting on names would notice. Returning the pairs rather than + /// setting them directly is what lets a test state them. + /// + /// The options the run was started with. + /// Each variable name with the value to give it, or null to leave it alone. + internal static IReadOnlyList> BuildAssertionFormattingEnvironment(ITestFrameworkExecutionOptions executionOptions) { - private readonly Dictionary _assemblyFixtureMappings = new Dictionary(); - private ExecutionContext _context; + EnsureArg.IsNotNull(executionOptions, nameof(executionOptions)); - public AssemblyRunner(ITestAssembly testAssembly, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) - : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) + return new[] { - } + new KeyValuePair(AssertEquivalentMaxDepthVariable, executionOptions.AssertEquivalentMaxDepth()), + new KeyValuePair(PrintMaxEnumerableLengthVariable, executionOptions.PrintMaxEnumerableLength()), + new KeyValuePair(PrintMaxObjectDepthVariable, executionOptions.PrintMaxObjectDepth()), + new KeyValuePair(PrintMaxObjectMemberCountVariable, executionOptions.PrintMaxObjectMemberCount()), + new KeyValuePair(PrintMaxStringLengthVariable, executionOptions.PrintMaxStringLength()), + }; + } - protected override async Task AfterTestAssemblyStartingAsync() + private static void SetEnvironment(string environmentVariableName, int? value) + { + if (value.HasValue) { - // Let everything initialize - await base.AfterTestAssemblyStartingAsync(); - - // Find the AssemblyFixtureAttributes placed on the test assembly - Aggregator.Run(() => - { - var fixtureAttributes = ((IReflectionAssemblyInfo)TestAssembly.Assembly).Assembly - .GetCustomAttributes(typeof(AssemblyFixtureAttribute), false) - .Cast(); - - foreach (var fixtureAttr in fixtureAttributes) - { - _assemblyFixtureMappings[fixtureAttr.FixtureType] = Activator.CreateInstance(fixtureAttr.FixtureType); - } - }); + Environment.SetEnvironmentVariable(environmentVariableName, value.Value.ToString(CultureInfo.InvariantCulture)); + } + } - _context = ExecutionContext.Capture(); + /// + /// The assembly runner context, extended to bound how many collections run at once. + /// + /// + /// xunit applies the conservative parallel algorithm by acquiring a semaphore inside its own + /// context's collection dispatch. Substituting a collection runner means not going through + /// that dispatch, which would leave MaxParallelThreads unenforced and start every + /// collection at once. Its semaphore is private, so this context keeps an equivalent one of + /// its own and dispatches through it. + /// + private sealed class FixtureArgumentSetAssemblyRunnerContext : XunitTestAssemblyRunnerContext + { + private SemaphoreSlim _parallelSemaphore; + + public FixtureArgumentSetAssemblyRunnerContext( + IXunitTestAssembly testAssembly, + IReadOnlyCollection testCases, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions, + CancellationToken cancellationToken) + : base(testAssembly, testCases, executionMessageSink, executionOptions, cancellationToken) + { } - protected override async Task BeforeTestAssemblyFinishedAsync() + public override void SetupParallelism() { - foreach (var fixture in _assemblyFixtureMappings.Values) + base.SetupParallelism(); + + // The aggressive algorithm bounds concurrency with a synchronization context, which + // the base call has already installed and which applies to this dispatch too. Only + // the conservative algorithm needs a semaphore here. + if (MaxParallelThreads > 0 && ParallelAlgorithm == ParallelAlgorithm.Conservative) { - switch (fixture) - { - case IAsyncLifetime d: - await d.DisposeAsync(); - break; - case IAsyncDisposable d: - await d.DisposeAsync(); - break; - case IDisposable d: - d.Dispose(); - break; - } + _parallelSemaphore = new SemaphoreSlim(initialCount: MaxParallelThreads); } - - await base.BeforeTestAssemblyFinishedAsync(); } - protected override Task RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) + public async ValueTask RunFixtureArgumentSetCollection( + IXunitTestCollection testCollection, + IReadOnlyCollection testCases, + ITestCaseOrderer testCaseOrderer) { - Task result = null; - ExecutionContext.Run(_context, state => result = new CollectionRunner(_assemblyFixtureMappings, testCollection, testCases, DiagnosticMessageSink, messageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), cancellationTokenSource).RunAsync(), state: null); - return result; + if (_parallelSemaphore != null) + { + await _parallelSemaphore.WaitAsync(CancellationTokenSource.Token); + } + + try + { + var runner = new FixtureArgumentSetCollectionRunner(); + + // ExceptionAggregator is a struct wrapping a mutable list, so passing this + // context's own would give every collection a shared list to append to and + // clear. Collections run concurrently, so one collection's fixture failure could + // then surface against another, or be cleared before it was ever reported. + return await runner.Run( + testCollection, + testCases, + ExplicitOption, + MessageBus, + testCaseOrderer, + Aggregator.Clone(), + CancellationTokenSource, + AssemblyFixtureMappings); + } + finally + { + _parallelSemaphore?.Release(); + } } - public override void Dispose() + public override async ValueTask DisposeAsync() { - _context?.Dispose(); - base.Dispose(); + _parallelSemaphore?.Dispose(); + await base.DisposeAsync(); } } - private sealed class CollectionRunner : XunitTestCollectionRunner + private sealed class FixtureArgumentSetAssemblyRunner : XunitTestAssemblyRunnerBase { - private readonly Dictionary _assemblyFixtureMappings; - - public CollectionRunner(Dictionary assemblyFixtureMappings, ITestCollection testCollection, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ITestCaseOrderer testCaseOrderer, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) - : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource) + public async ValueTask Run( + IXunitTestAssembly testAssembly, + IReadOnlyCollection testCases, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions, + CancellationToken cancellationToken) { - _assemblyFixtureMappings = assemblyFixtureMappings; + await using var context = new FixtureArgumentSetAssemblyRunnerContext(testAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); + await context.InitializeAsync(); + + return await Run(context); } - protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable testCases) + protected override ValueTask RunTestCollection(FixtureArgumentSetAssemblyRunnerContext context, IXunitTestCollection testCollection, IReadOnlyCollection testCases) { - EnsureArg.IsNotNull(testClass, nameof(testClass)); - EnsureArg.IsNotNull(@class, nameof(@class)); - EnsureArg.IsNotNull(testCases, nameof(testCases)); - - Dictionary combinedMappings = null; + var testCaseOrderer = context.AssemblyTestCaseOrderer ?? DefaultTestCaseOrderer.Instance; + return context.RunFixtureArgumentSetCollection(testCollection, testCases, testCaseOrderer); + } + } - if (testClass.Class is TestClassWithFixtureArgumentsTypeInfo classWithFixtureArguments) + private sealed class FixtureArgumentSetCollectionRunner : XunitTestCollectionRunner + { + protected override ValueTask RunTestClass(XunitTestCollectionRunnerContext context, IXunitTestClass testClass, IReadOnlyCollection testCases) + { + if (testClass == null) { - combinedMappings = new Dictionary(CollectionFixtureMappings); - - foreach (var variant in classWithFixtureArguments.FixtureArguments) - { - combinedMappings.Add(variant.EnumValue.GetType(), variant.EnumValue); - } + // xunit reports cases with no class as failures with an explanatory message. + // Substituting a class runner must not turn that into an argument exception. + return base.RunTestClass(context, testClass, testCases); } - if (_assemblyFixtureMappings.Count > 0 && combinedMappings == null) - { - combinedMappings = new Dictionary(CollectionFixtureMappings); - } + var testCaseOrderer = context.TestCaseOrderer ?? DefaultTestCaseOrderer.Instance; + var classRunner = new FixtureArgumentSetClassRunner(); - foreach (var assemblyFixtureMapping in _assemblyFixtureMappings) - { - combinedMappings.Add(assemblyFixtureMapping.Key, assemblyFixtureMapping.Value); - } - - return new ExecutionContextFlowingClassRunner( - testClass, - @class, - testCases, - DiagnosticMessageSink, - MessageBus, - TestCaseOrderer, - new ExceptionAggregator(Aggregator), - CancellationTokenSource, - combinedMappings ?? CollectionFixtureMappings) - .RunAsync(); + // Cloned for the same reason as at the collection boundary: classes of a collection + // must not share one mutable list of exceptions. + return classRunner.Run(testClass, testCases, context.ExplicitOption, context.MessageBus, testCaseOrderer, context.Aggregator.Clone(), context.CancellationTokenSource, context.CollectionFixtureMappings); } } /// - /// An that runs tests in the same as when all fixture constructors ran. - /// This means that s set during a fixture constructor can be read during test method execution. + /// Runs a test class, injecting the fixture argument set values its variant was expanded with. /// - private sealed class ExecutionContextFlowingClassRunner : XunitTestClassRunner + /// + /// Under xunit.v2 this runner also ran test methods in the execution context captured right + /// after each class fixture was constructed, so an written by a + /// fixture constructor could be read by the tests. That is not reproducible on xunit.v3: + /// v2 built fixtures in a synchronous CreateClassFixture override, while v3 builds + /// them inside FixtureMappingManager.GetFixture, an async method whose state machine + /// restores the caller's execution context as it returns. Any such write is therefore + /// already discarded before this runner regains control, and no capture point outside that + /// method can see it. Fixtures that need to share ambient state with their tests have to + /// expose it as a member instead of relying on the execution context. + /// + private sealed class FixtureArgumentSetClassRunner : XunitTestClassRunner { - private ExecutionContext _context; + private static readonly FieldInfo FixtureCacheField = typeof(FixtureMappingManager) + .GetField("fixtureCache", BindingFlags.Instance | BindingFlags.NonPublic); - public ExecutionContextFlowingClassRunner(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ITestCaseOrderer testCaseOrderer, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, IDictionary collectionFixtureMappings) - : base(testClass, @class, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource, collectionFixtureMappings) + private static readonly FieldInfo ParentMappingManagerField = typeof(FixtureMappingManager) + .GetField("parentMappingManager", BindingFlags.Instance | BindingFlags.NonPublic); + + protected override async ValueTask OnTestClassStarting(XunitTestClassRunnerContext context) { + InjectFixtureArguments(context); + return await base.OnTestClassStarting(context); } - protected override void CreateClassFixture(Type fixtureType) + protected override ValueTask GetConstructorArgument(XunitTestClassRunnerContext context, ConstructorInfo constructor, int index, ParameterInfo parameter) { - base.CreateClassFixture(fixtureType); - _context = ExecutionContext.Capture(); + if (context?.TestClass is FixtureArgumentSetTestClass fixtureTestClass) + { + var fixtureArguments = fixtureTestClass.GetFixtureArguments(); + if (fixtureArguments.Count > 0) + { + var enumValue = fixtureArguments + .Select(argument => argument.EnumValue) + .FirstOrDefault(value => value != null && parameter.ParameterType == value.GetType()); + + if (enumValue != null) + { + return new ValueTask(enumValue); + } + } + } + + return base.GetConstructorArgument(context, constructor, index, parameter); } - protected override Task RunTestMethodsAsync() + private static void InjectFixtureArguments(XunitTestClassRunnerContext context) { - if (_context == null) + if (context?.TestClass == null) { - // no class fixtures, so context needed + return; + } - return base.RunTestMethodsAsync(); + if (FixtureCacheField == null) + { + throw new InvalidOperationException("Unable to inject fixture arguments because FixtureMappingManager.fixtureCache was not found."); } - Task result = null; - ExecutionContext.Run(_context, state => result = base.RunTestMethodsAsync(), null); - return result; - } - } + if (ParentMappingManagerField == null) + { + throw new InvalidOperationException("Unable to inject fixture arguments because FixtureMappingManager.parentMappingManager was not found."); + } - /// - /// facts have special optimized serialization. Their deserialization needs to be able to instantiate - /// the synthetic classes that we created of the form Namespace.Class(Arg1, Arg2). For these, we want to create a - /// with Arg1 and Arg2 as the fixture arguments - /// - private sealed class CustomAssemblyInfo : LongLivedMarshalByRefObject, IAssemblyInfo - { - private readonly IAssemblyInfo _assemblyInfoImplementation; - private readonly Regex _argumentsRegex = new Regex(@"\((\s*(?[^, )]+)\s*,?)*\)"); + var cacheOwner = context.ClassFixtureMappings; + if (ParentMappingManagerField.GetValue(cacheOwner) is FixtureMappingManager parentMappingManager) + { + cacheOwner = parentMappingManager; + } - public CustomAssemblyInfo(IAssemblyInfo assemblyInfoImplementation) - { - _assemblyInfoImplementation = assemblyInfoImplementation; - } + var cache = FixtureCacheField.GetValue(cacheOwner) as IDictionary; + if (cache == null) + { + throw new TestPipelineException("Unable to inject fixture arguments because the xUnit fixture cache could not be read."); + } - public string AssemblyPath => _assemblyInfoImplementation.AssemblyPath; + var fixtureArguments = new List(); + if (context.TestClass is FixtureArgumentSetTestClass fixtureTestClass) + { + fixtureArguments.AddRange(fixtureTestClass.GetFixtureArguments().Select(argument => argument.EnumValue)); + } - public string Name => _assemblyInfoImplementation.Name; + var fixtureParameterTypes = new HashSet(); + var classFixtureTypes = context.TestClass.ClassFixtureTypes; + if (classFixtureTypes != null) + { + foreach (var fixtureType in classFixtureTypes) + { + var constructor = fixtureType.GetConstructors() + .SingleOrDefault(ctor => !ctor.IsStatic && ctor.IsPublic); + if (constructor == null) + { + continue; + } + + foreach (var parameterType in constructor.GetParameters() + .Select(parameter => parameter.ParameterType) + .Where(parameterType => parameterType.IsEnum)) + { + fixtureParameterTypes.Add(parameterType); + } + } + } - public IEnumerable GetCustomAttributes(string assemblyQualifiedAttributeTypeName) - { - return _assemblyInfoImplementation.GetCustomAttributes(assemblyQualifiedAttributeTypeName); - } + var resolvedArguments = new Dictionary(); - public ITypeInfo GetType(string typeName) - { - EnsureArg.IsNotNull(typeName, nameof(typeName)); + foreach (var argument in fixtureArguments + .Select(enumValue => (EnumValue: enumValue, ArgumentType: enumValue.GetType()))) + { + foreach (var parameterType in fixtureParameterTypes + .Where(parameterType => string.Equals(parameterType.FullName, argument.ArgumentType.FullName, StringComparison.Ordinal) + && !resolvedArguments.ContainsKey(parameterType))) + { + resolvedArguments[parameterType] = parameterType == argument.ArgumentType + ? argument.EnumValue + : Enum.ToObject(parameterType, Convert.ToInt64(argument.EnumValue)); + } + } - // parse out the (Arg1, Arg2) - var match = _argumentsRegex.Match(typeName); - if (!match.Success) + if (resolvedArguments.Count == 0 && fixtureParameterTypes.Count > 0) { - return _assemblyInfoImplementation.GetType(typeName); + var traits = GetTraits(context.TestCases); + foreach (var parameterType in fixtureParameterTypes + .Where(parameterType => traits.TryGetValue(parameterType.Name, out var values) && values.Count > 0)) + { + var traitKey = parameterType.Name; + var values = traits[traitKey]; + + if (values.Count > 1) + { + throw new TestPipelineException($"Fixture argument '{traitKey}' had multiple values: {string.Join(", ", values)}"); + } + + var value = values.First(); + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + if (!Enum.TryParse(parameterType, value, ignoreCase: true, out var parsedValue)) + { + throw new TestPipelineException($"Fixture argument '{traitKey}' value '{value}' could not be parsed as {parameterType.FullName}."); + } + + resolvedArguments[parameterType] = parsedValue; + } } - // retrieve the real type - var typeInfo = _assemblyInfoImplementation.GetType(typeName.Substring(0, match.Index)); - Debug.Assert(typeInfo != null, $"Could not find type {typeName} in assembly"); + // The cache being written to is scoped to the collection, and the variants of a class + // share a collection so that an explicit [Collection] keeps its classes serialized. + // Every variant therefore writes over the previous one's arguments: adding only what + // is missing would leave the first variant's values in place and hand every later + // variant the wrong fixture. Types this class expects but could not resolve are + // evicted for the same reason -- a stale value is worse than no value, because it + // silently constructs a fixture for the wrong argument set. + foreach (Type parameterType in fixtureParameterTypes) + { + if (resolvedArguments.TryGetValue(parameterType, out var resolvedValue)) + { + cache[parameterType] = resolvedValue; + } + else + { + cache.Remove(parameterType); + } + } + } - // now get the the arguments. We don't know what type they are, so we look at the FixtureArgumentSetsAttribute on the class and look at its arguments - IAttributeInfo attributeInfo = typeInfo.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute)).Single(); + private static Dictionary> GetTraits(IReadOnlyCollection testCases) + { + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); - SingleFlag[] arguments = attributeInfo - .GetConstructorArguments() - .Cast() - .Zip( - match.Groups["VALUE"].Captures, - (e, c) => new SingleFlag((Enum)Enum.Parse(e.GetType(), c.Value))) - .ToArray(); + foreach (var testCase in testCases) + { + // Read through the metadata interface, which XunitTestCase implements as a + // read-only projection of the same dictionary its own Traits property exposes. + MergeTraits(traits, ((ITestCaseMetadata)testCase).Traits); + } - return new TestClassWithFixtureArgumentsTypeInfo(typeInfo, arguments); + return traits.ToDictionary( + kvp => kvp.Key, + kvp => (IReadOnlyCollection)kvp.Value.ToArray(), + StringComparer.OrdinalIgnoreCase); } - public IEnumerable GetTypes(bool includePrivateTypes) + private static void MergeTraits(Dictionary> target, IReadOnlyDictionary> source) { - return _assemblyInfoImplementation.GetTypes(includePrivateTypes); + foreach (var kvp in source) + { + if (!target.TryGetValue(kvp.Key, out var values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + target[kvp.Key] = values; + } + + values.UnionWith(kvp.Value); + } } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs new file mode 100644 index 0000000000..ed137c30fc --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs @@ -0,0 +1,69 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using EnsureThat; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit +{ + /// + /// A test case standing in for one test that a failure to expand fixture argument sets lost, + /// carrying the traits the expansion never got far enough to produce. + /// + /// + /// A fixture argument set class is normally expanded into one variant per argument, and each + /// variant carries a trait naming its argument. Legs that select tests by those traits are the + /// reason this type exists: takes no traits and so would be + /// dropped by a positive trait filter, which puts the run back to passing with the broken class + /// silently missing - the exact outcome reporting the fault as a test was meant to prevent. + /// + /// One case carries one combination of argument set values, never all of them at once. A trait + /// filter matches a case when any value under the named trait matches, and an exclusion + /// filter is the negation of that, so a single case declaring both DataStore=CosmosDb and + /// DataStore=SqlServer would be excluded by both --filter-not-trait + /// DataStore=SqlServer and --filter-not-trait DataStore=CosmosDb - that is, by both of + /// the integration legs this repository runs, which select by exclusion alone. The discoverer + /// therefore reports one of these per combination, so each leg selects exactly the case that + /// stands for the tests it lost. The export and E2E legs instead require a positive + /// DataStore, and a case can only reach them by carrying that leg's value, which is the + /// same reason the combinations are reported separately rather than pooled. + /// + /// + internal sealed class DiscoveryErrorTestCase : ExecutionErrorTestCase + { + public DiscoveryErrorTestCase( + IXunitTestMethod testMethod, + string testCaseDisplayName, + string uniqueID, + string errorMessage, + IReadOnlyDictionary> traits) + : base(testMethod, testCaseDisplayName, uniqueID, sourceFilePath: null, sourceLineNumber: null, errorMessage: errorMessage) + { + EnsureArg.IsNotNull(traits, nameof(traits)); + + // XunitTestCase.Traits is the live dictionary the case is filtered on, and the base + // constructor has already created it empty. Adding to it here is what gives the failure + // the traits of the tests it replaces. + foreach (KeyValuePair> trait in traits) + { + if (!Traits.TryGetValue(trait.Key, out HashSet values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + Traits[trait.Key] = values; + } + + values.UnionWith(trait.Value); + } + } + +#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes + public DiscoveryErrorTestCase() + { + } +#pragma warning restore CS0618 + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs new file mode 100644 index 0000000000..c3c21d1584 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs @@ -0,0 +1,72 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using EnsureThat; +using Xunit.Sdk; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit +{ + /// + /// A test class bound to one fixture argument set, so that a single class type can be discovered + /// once per variant with a distinct identity. + /// + /// + /// The fixture arguments are held in memory only. implements + /// IXunitSerializable non-virtually, so a derived type has no way to add its own state to + /// the serialized form, and a runner that discovered tests in one process and ran them in another + /// would hand the executor a plain rather than this type. Both the + /// Microsoft Testing Platform runner used by CI and the Visual Studio test explorer discover and + /// run in the same process, so the variant survives; a runner that did not would fall back to the + /// unvaried class rather than silently mixing variants up. + /// + internal sealed class FixtureArgumentSetTestClass : XunitTestClass + { + private static readonly FieldInfo UniqueIdField = typeof(XunitTestClass).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + "XunitTestClass.uniqueID was not found, so fixture argument set variants cannot be given distinct identities and " + + "would collapse into a single test class. This usually means the xunit.v3 version changed."); + + private readonly IReadOnlyList _fixtureArguments; + + public FixtureArgumentSetTestClass(Type testClassType, IXunitTestCollection testCollection, IReadOnlyList fixtureArguments, string uniqueId) + : base(testClassType, testCollection, uniqueId) + { + EnsureArg.IsNotNull(testClassType, nameof(testClassType)); + EnsureArg.IsNotNull(testCollection, nameof(testCollection)); + EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); + + _fixtureArguments = fixtureArguments; + UpdateUniqueId(); + } + +#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes + public FixtureArgumentSetTestClass() + { + _fixtureArguments = Array.Empty(); + } +#pragma warning restore CS0618 + + internal IReadOnlyList GetFixtureArguments() + { + return _fixtureArguments; + } + + private void UpdateUniqueId() + { + if (_fixtureArguments.Count == 0) + { + return; + } + + var className = $"{TestClassName}({string.Join(", ", _fixtureArguments.Select(v => v.EnumValue))})"; + UniqueIdField.SetValue(this, UniqueIDGenerator.ForTestClass(TestCollection.UniqueID, className)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs new file mode 100644 index 0000000000..639147086e --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -0,0 +1,137 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using EnsureThat; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit +{ + internal sealed class FixtureArgumentSetTestMethod : XunitTestMethod + { + private static readonly FieldInfo TraitsField = GetRequiredField("traits"); + + private readonly IReadOnlyList _fixtureArguments; + + public FixtureArgumentSetTestMethod(FixtureArgumentSetTestClass testClass, MethodInfo methodInfo, IReadOnlyList fixtureArguments, string uniqueId) + : base(testClass, methodInfo, testMethodArguments: BuildMethodArguments(fixtureArguments), uniqueId) + { + EnsureArg.IsNotNull(testClass, nameof(testClass)); + EnsureArg.IsNotNull(methodInfo, nameof(methodInfo)); + EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); + + _fixtureArguments = fixtureArguments; + } + +#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes + public FixtureArgumentSetTestMethod() + { + // The de-serializer bypasses the constructor that assigns this field, and the argument set + // values are recovered from the base type's method arguments rather than from it. Leaving + // it null would turn any later read into a NullReferenceException reported as a test + // failure with no bearing on the test, so it starts empty as it does on the class. + _fixtureArguments = Array.Empty(); + } +#pragma warning restore CS0618 + + /// + /// Resolves a private field on that this type has to write to. + /// + /// + /// xUnit v3 holds this state in private fields with no public setter and no virtual member + /// that would let a derived type supply it, so reflection is the only way to set it. The + /// members are not virtual either, which is why hiding them with new does not work: + /// xUnit dispatches through and would never see the override. + /// + /// The name of the private instance field. + /// The field. + /// Thrown when the field no longer exists. + private static FieldInfo GetRequiredField(string name) + { + return typeof(XunitTestMethod).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + $"XunitTestMethod.{name} was not found. Fixture argument set traits and arguments cannot be applied, " + + "which would silently change which tests the CI filters select. This usually means the xunit.v3 version changed."); + } + + /// + /// Builds the arguments the base type is constructed with, which are the fixture's argument + /// set values. + /// + /// + /// These are supplied through the base constructor rather than written back over it + /// afterwards, so that the private field holding them is one less thing this type has to + /// reach into and one less way a change to xunit can go unnoticed. + /// + /// The argument set values this variant stands for. + /// The values as an argument array. + private static object[] BuildMethodArguments(IReadOnlyList fixtureArguments) + { + EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); + + if (fixtureArguments.Count == 0) + { + return Array.Empty(); + } + + var result = new object[fixtureArguments.Count]; + for (int i = 0; i < fixtureArguments.Count; i++) + { + result[i] = fixtureArguments[i].EnumValue; + } + + return result; + } + + /// + /// Adds a trait naming each fixture argument set value this variant stands for, so a CI leg + /// selecting on those traits can see it. + /// + /// + /// A value here is never null: the only producer builds every through + /// its constructor, and a degenerate combination is rejected before reaching this point. It is + /// deliberately not guarded against anyway. Skipping a value that somehow was null would drop + /// the trait and leave this variant unselectable by the E2E and export legs, which each require + /// a positive match - the test would vanish from them silently. Throwing instead surfaces as a + /// discovery fault, which this assembly turns into a reported failure per argument set. + /// + internal void ApplyArgumentSetTraits() + { + if (_fixtureArguments.Count == 0) + { + return; + } + + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); +#pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists + foreach (var kvp in base.Traits) + { + traits[kvp.Key] = new HashSet(kvp.Value, StringComparer.OrdinalIgnoreCase); + } +#pragma warning restore SA1100 + + for (int i = 0; i < _fixtureArguments.Count; i++) + { + var enumValue = _fixtureArguments[i].EnumValue; + var enumValueText = enumValue.ToString(); + + string key = enumValue.GetType().Name; + if (!traits.TryGetValue(key, out var values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + traits[key] = values; + } + + values.Add(enumValueText); + } + + var typedTraits = traits.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyCollection)kvp.Value, StringComparer.OrdinalIgnoreCase); + TraitsField.SetValue(this, new Lazy>>(() => typedTraits)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs index 0483db4bc6..853f4af7d9 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using EnsureThat; namespace Microsoft.Health.Extensions.Xunit @@ -15,9 +16,26 @@ namespace Microsoft.Health.Extensions.Xunit [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, Inherited = false)] public abstract class FixtureArgumentSetsAttribute : Attribute { + private readonly Enum[] _argumentSets; + + /// + /// Initializes a new instance of the class. + /// + /// + /// The flags enum values to pass to the class fixture's constructor. Each value passed becomes + /// one dimension, and the test class is expanded into one variant per combination of the flags + /// set across the dimensions. By convention each dimension uses a distinct flags enum type, so + /// that a fixture constructor can tell its parameters apart. + /// protected FixtureArgumentSetsAttribute(params Enum[] argumentSets) { EnsureArg.IsNotNull(argumentSets, nameof(argumentSets)); + _argumentSets = argumentSets; + } + + internal IReadOnlyList GetArgumentSets() + { + return _argumentSets; } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj b/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj index f493041c0b..ffba2980ab 100644 --- a/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj +++ b/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj @@ -12,11 +12,19 @@ $(NoWarn);NU5104;NU5100;CA1812;CA1515 + + + <_Parameter1>Microsoft.Health.Extensions.Xunit.UnitTests + + + - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Extensions.Xunit/NoResultOutcome.cs b/src/Microsoft.Health.Extensions.Xunit/NoResultOutcome.cs new file mode 100644 index 0000000000..caa31d5147 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/NoResultOutcome.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit +{ + /// + /// How a retry attempt that published no result of its own should be accounted for. + /// + /// + /// An attempt reports nothing when the run is cancelled or aborted underneath it. That is not + /// the same as passing, and the choice made here decides whether an already observed failure + /// still reaches the results. + /// + internal enum NoResultOutcome + { + /// + /// No attempt ever published a result, so the test contributes nothing to the totals. + /// + ReportNothing, + + /// + /// This attempt observed a failure before it was cut short, so that failure is replayed. + /// + ReplayCurrentAttempt, + + /// + /// An earlier attempt deferred a failure while waiting to be retried, and the retry never + /// reported anything, so the earlier failure is replayed. + /// + ReplayEarlierAttempt, + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs index 60f6e32b2d..e09440b554 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs @@ -4,8 +4,9 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Runtime.CompilerServices; using Xunit; -using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { @@ -13,12 +14,23 @@ namespace Microsoft.Health.Extensions.Xunit /// Attribute that marks a test method to be retried a specified number of times if it fails. /// Useful for handling transient failures in integration and end-to-end tests. /// - [XunitTestCaseDiscoverer("Microsoft.Health.Extensions.Xunit.RetryFactDiscoverer", "Microsoft.Health.Extensions.Xunit")] + [XunitTestCaseDiscoverer(typeof(RetryFactDiscoverer))] [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class RetryFactAttribute : FactAttribute { /// - /// Gets or sets the maximum number of retry attempts (default is 3). + /// Initializes a new instance of the class. + /// + /// The source file containing the test method. Supplied by the compiler. + /// The line number of the test method. Supplied by the compiler. + public RetryFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) + { + } + + /// + /// Gets or sets the maximum number of attempts, including the first run (default is 3). + /// A value of 3 therefore runs a failing test three times in total, not four. /// public int MaxRetries { get; set; } = 3; @@ -28,9 +40,16 @@ public sealed class RetryFactAttribute : FactAttribute public int DelayBetweenRetriesMs { get; set; } = 5000; /// - /// Gets or sets whether to retry on assertion failures (XunitException). + /// Gets or sets whether to retry on assertion failures. /// Default is false - assertion failures usually indicate test bugs, not transient issues. /// Set to true for tests that validate eventually-consistent systems (e.g., cache refresh, reindex operations). + /// + /// What counts as an assertion failure is decided from the reported exception's type name - + /// one containing "Xunit" or "Assert", so that assertion libraries used alongside xUnit are + /// recognised too - rather than from the exception type itself, which the message bus does + /// not receive. Timeouts are deliberately excluded: they match that name test but are the + /// canonical transient failure, so they are retried whatever this is set to. + /// /// public bool RetryOnAssertionFailure { get; set; } = false; } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs index f45ab9690f..f3e800d049 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs @@ -3,52 +3,75 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; -using Xunit.Abstractions; +using System.Linq; +using System.Threading.Tasks; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { /// /// Test case discoverer for . /// - public class RetryFactDiscoverer : IXunitTestCaseDiscoverer + public sealed class RetryFactDiscoverer : IXunitTestCaseDiscoverer { - private readonly IMessageSink _diagnosticMessageSink; - - public RetryFactDiscoverer(IMessageSink diagnosticMessageSink) - { - _diagnosticMessageSink = diagnosticMessageSink; - } - - public IEnumerable Discover( + /// + /// Discovers the test case for a method marked with . + /// + /// The discovery options for the test assembly. + /// The test method being discovered. + /// The applied to the method. + /// A single wrapping the method. + public ValueTask> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo factAttribute) + IXunitTestMethod testMethod, + IFactAttribute factAttribute) { - var maxRetries = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.MaxRetries)); - var delayMs = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.DelayBetweenRetriesMs)); - var retryOnAssertionFailure = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.RetryOnAssertionFailure)); + var attribute = (RetryFactAttribute)factAttribute; - // Use default values if not specified - if (maxRetries == 0) - { - maxRetries = 3; - } + // Let xUnit compute the test case details the same way it does for a plain [Fact], + // so that the display name honors the configured MethodDisplay setting instead of + // being reported as a bare, unqualified method name. + var details = TestIntrospectionHelper.GetTestCaseDetails( + discoveryOptions, + testMethod, + factAttribute, + testMethodArguments: null, + timeout: null, + baseDisplayName: null); - if (delayMs == 0) - { - delayMs = 5000; - } + // xunit builds a test case's traits with an OrdinalIgnoreCase comparer, and the argument + // set traits are merged into them on the same terms, so the copy is made on those terms + // too. Filtering does not depend on it - the runner compares trait names itself - but a + // copy that compared differently from the dictionary it came from would let the same + // trait be held twice under two spellings. + var traits = testMethod.Traits.ToDictionary( + kvp => kvp.Key, + kvp => new HashSet(kvp.Value, StringComparer.OrdinalIgnoreCase), + StringComparer.OrdinalIgnoreCase); - yield return new RetryTestCase( - _diagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), - testMethod, - maxRetries, - delayMs, - retryOnAssertionFailure); + var testCase = new RetryTestCase( + details.ResolvedTestMethod, + details.TestCaseDisplayName, + details.UniqueID, + @explicit: details.Explicit, + skipExceptions: details.SkipExceptions, + skipReason: details.SkipReason, + skipType: details.SkipType, + skipUnless: details.SkipUnless, + skipWhen: details.SkipWhen, + traits: traits, + testMethodArguments: null, + sourceFile: details.SourceFilePath, + sourceLine: details.SourceLineNumber, + timeout: details.Timeout, + maxRetries: attribute.MaxRetries, + delayMs: attribute.DelayBetweenRetriesMs, + retryOnAssertionFailure: attribute.RetryOnAssertionFailure); + + return new ValueTask>(new[] { testCase }); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 1afc54c07f..95df5d1f41 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -4,102 +4,255 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; -using Xunit.Abstractions; +using Xunit; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { /// /// Test case that implements retry logic. /// - public class RetryTestCase : XunitTestCase + public sealed class RetryTestCase : XunitTestCase, ISelfExecutingXunitTestCase { private int _maxRetries; private int _delayMs; private bool _retryOnAssertionFailure; + /// + /// Initializes a new instance of the class. Used only by the deserializer. + /// [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] public RetryTestCase() { } + /// + /// Initializes a new instance of the class. + /// + /// The test method this test case belongs to. + /// The display name reported for the test case. + /// The unique ID of the test case. + /// Whether the test case is only run when explicitly selected. + /// Exception types that cause the test to be reported as skipped rather than failed. + /// The static reason the test is skipped, or null if it is not skipped. + /// The type containing the member named by or . + /// The name of a property that must be true for the test to run. + /// The name of a property that must be false for the test to run. + /// The traits associated with the test case. + /// The arguments passed to the test method. + /// The source file containing the test method. + /// The line number of the test method. + /// The per-attempt timeout in milliseconds, or null for none. + /// The maximum number of attempts. Values below one are clamped to one. + /// The delay in milliseconds between attempts. + /// Whether assertion failures should be retried, rather than only non-assertion exceptions. public RetryTestCase( - IMessageSink diagnosticMessageSink, - TestMethodDisplay defaultMethodDisplay, - TestMethodDisplayOptions defaultMethodDisplayOptions, - ITestMethod testMethod, + IXunitTestMethod testMethod, + string displayName, + string uniqueId, + bool @explicit, + Type[] skipExceptions, + string skipReason, + Type skipType, + string skipUnless, + string skipWhen, + Dictionary> traits, + object[] testMethodArguments, + string sourceFile, + int? sourceLine, + int? timeout, int maxRetries, int delayMs, - bool retryOnAssertionFailure = false, - object[] testMethodArguments = null) - : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) + bool retryOnAssertionFailure) + : base(testMethod, displayName, uniqueId, @explicit, skipExceptions, skipReason, skipType, skipUnless, skipWhen, traits, testMethodArguments, sourceFile, sourceLine, timeout) { - _maxRetries = maxRetries; - _delayMs = delayMs; + // A retry count below one would skip the attempt loop entirely and report no result + // at all, silently dropping the test. Always run at least one attempt. + _maxRetries = Math.Max(1, maxRetries); + + // A negative delay below -1 makes Task.Delay throw, turning a configuration slip into + // an unrelated crash inside the retry loop. + _delayMs = Math.Max(0, delayMs); _retryOnAssertionFailure = retryOnAssertionFailure; } - public override async Task RunAsync( - IMessageSink diagnosticMessageSink, + /// + /// Names the test in diagnostics. + /// + /// + /// The display name is used rather than the class and method names because those are not + /// unique: a method expanded over fixture argument sets, or a theory with several rows, + /// produces many test cases sharing them. Retry diagnostics are read when something has + /// already gone wrong, and messages that cannot say which variant retried, or failed after + /// exhausting its attempts, are worth little at that point. The display name carries the + /// variant and the row, and falls back to the class and method if it is not set. + /// + private string TestDescription => + !string.IsNullOrEmpty(TestCaseDisplayName) + ? TestCaseDisplayName + : $"{TestMethod?.TestClass?.TestClassName}.{TestMethod?.MethodName}"; + + /// + /// Runs the test case, retrying a failed attempt up to the configured number of attempts. + /// + /// + /// Nothing here decides whether a failure was transient. Every failure is retried except one + /// classified as an assertion failure, which is retried only when the test asks for it with + /// RetryOnAssertionFailure - see . A deterministic failure is + /// therefore retried too, and fails every attempt, which costs time but reports the same + /// result. Read a passing retry as "this attempt passed", not as "the failure was transient". + /// + /// Whether explicit tests should be run. + /// The message bus that test results are reported to. + /// The arguments to pass to the test class constructor. + /// The exception aggregator for the containing test class. + /// The cancellation token source for the test run. + /// A summary of the run. Reporters derive their results from + /// rather than from this summary, so every outcome must also be reported as a message. + public async ValueTask Run( + ExplicitOption explicitOption, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { - // Use System.Diagnostics.Trace for ADO visibility - Trace.WriteLine($"##vso[task.logdetail]RetryFact starting test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' with MaxRetries={_maxRetries}, DelayMs={_delayMs}, RetryOnAssertionFailure={_retryOnAssertionFailure}"); + // Trace output is only visible to an attached debugger, never in CI logs. Anything + // that needs to show up in a pipeline log has to go to Console instead. + Trace.WriteLine($"RetryFact starting test '{TestDescription}' with MaxRetries={_maxRetries}, DelayMs={_delayMs}, RetryOnAssertionFailure={_retryOnAssertionFailure}"); var runSummary = new RunSummary { Total = 1 }; Exception lastException = null; + // Holds the deferred failure of an attempt that is about to be retried. It is discarded + // only once a later attempt has actually reported a result, so that a run cancelled + // between attempts still reports the failure that was already observed instead of + // silently dropping the test. + FailureInterceptingMessageBus pendingFailureBus = null; + for (int attempt = 1; attempt <= _maxRetries; attempt++) { var isLastAttempt = attempt == _maxRetries; - Trace.WriteLine($"##vso[task.logdetail]RetryFact attempt {attempt}/{_maxRetries} for test '{TestMethod.Method.Name}'"); + Trace.WriteLine($"RetryFact attempt {attempt}/{_maxRetries} for test '{TestDescription}'"); // Create a fresh aggregator for each attempt var attemptAggregator = new ExceptionAggregator(); - // Only intercept failure messages on non-final attempts - // On the final attempt, let everything go through (both success and failure) - IMessageBus busToUse; FailureInterceptingMessageBus interceptingBus = null; - if (isLastAttempt) - { - busToUse = messageBus; - } - else - { - interceptingBus = new FailureInterceptingMessageBus(messageBus); - busToUse = interceptingBus; - } - try { - var summary = await base.RunAsync( - diagnosticMessageSink, - busToUse, - constructorArguments, + // Always wrap the bus so the failure details are available for the CI log, but only + // defer (hide) a failure while a further attempt could still supersede it. + interceptingBus = new FailureInterceptingMessageBus( + messageBus, + deferFailures: !isLastAttempt, + deferAbstentions: pendingFailureBus?.HasDeferredFailure ?? false); + + var summary = await XunitRunnerHelper.RunXunitTestCase( + this, + interceptingBus, + cancellationTokenSource, attemptAggregator, - cancellationTokenSource); + explicitOption, + constructorArguments); runSummary.Time = summary.Time; + // A cancelled run short-circuits and reports nothing, which is not the same as + // passing. Treating an empty summary as success would drop the test from the + // results entirely, taking any failure already seen on an earlier attempt with it. + // + // An empty summary is the only reliable signal that the attempt reported + // nothing. Cancellation on its own is not: an attempt that ran to completion + // while cancellation was requested has already reported its own result through + // the bus, and replaying an earlier attempt's deferred failure on top of that + // adds a second, orphaned result for a test that actually finished. + if (summary.Total == 0) + { + NoResultOutcome outcome = DecideNoResultOutcome( + currentAttemptObservedFailure: interceptingBus.HasObservedFailure, + earlierAttemptObservedFailure: pendingFailureBus?.HasDeferredFailure ?? false); + + Console.WriteLine( + outcome == NoResultOutcome.ReportNothing + ? $"[RetryFact] Test '{TestDescription}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted), and no earlier attempt observed a failure, so there is nothing to report for it." + : $"[RetryFact] Test '{TestDescription}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted). Reporting the last observed failure rather than a pass."); + + // At most one bus is replayed. A failure this attempt saw supersedes anything held + // over from an earlier one, and replaying both would publish two results for a + // single test. + bool continueRunning = ReportSingleResult(outcome, interceptingBus, pendingFailureBus); + + StopRunIfRequested(continueRunning, cancellationTokenSource); + + Exception noResultException = SelectNoResultException( + outcome, + currentAttemptException: attemptAggregator.ToException(), + earlierAttemptException: lastException); + + if (noResultException != null) + { + aggregator.Add(noResultException); + } + + RunSummary noResultSummary = CreateNoResultSummary(outcome); + noResultSummary.Time = runSummary.Time; + return noResultSummary; + } + + // A skip is not a pass. An attempt that abstains has not shown the test to be + // sound, so it must not erase a failure an earlier attempt already demonstrated: + // that failure is real, and discarding it here reports the test as skipped or + // not run, which no CI leg treats as a failure. + if (summary.Failed == 0 && + (summary.Skipped > 0 || summary.NotRun > 0) && + (pendingFailureBus?.HasDeferredFailure ?? false)) + { + Console.WriteLine( + $"[RetryFact] Test '{TestDescription}' skipped itself on attempt {attempt}/{_maxRetries} after failing earlier. Reporting the failure rather than the skip."); + + // The abstention was held rather than published, so dropping it now leaves + // the replayed failure as the test's only result. + interceptingBus.DiscardDeferredMessages(); + + bool replayed = pendingFailureBus.ReplayDeferredMessages(); + pendingFailureBus = null; + StopRunIfRequested(replayed, cancellationTokenSource); + + runSummary.Failed = 1; + return runSummary; + } + + // This attempt reported a real result, so a failure held over from an earlier + // attempt has been superseded and must not also be reported. + pendingFailureBus?.DiscardDeferredMessages(); + if (summary.Failed == 0) { // Test passed - success message already went through to Test Explorer - Trace.WriteLine($"##vso[task.logdetail]RetryFact test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' passed on attempt {attempt}/{_maxRetries}"); - diagnosticMessageSink.OnMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' passed on attempt {attempt}/{_maxRetries}")); + Trace.WriteLine($"RetryFact test '{TestDescription}' passed on attempt {attempt}/{_maxRetries}"); + messageBus.QueueMessage( + new DiagnosticMessage($"[RetryFact] Test '{TestDescription}' passed on attempt {attempt}/{_maxRetries}")); runSummary.Failed = 0; + + // A test that was skipped or left unrun did not pass, and the counts saying so + // live on the attempt's summary. Returning only Total and Failed would report + // it to anything reading this summary as a test that ran and passed. The + // Microsoft Testing Platform runner is not such a reader - it counts the + // messages instead, and reports the skip correctly either way - so this keeps + // the summary contract rather than fixing anything visible in a CI report. + runSummary.Skipped = summary.Skipped; + runSummary.NotRun = summary.NotRun; return runSummary; } @@ -109,9 +262,9 @@ public override async Task RunAsync( // If no exception was captured but test failed, create an exception using captured failure details if (lastException == null && summary.Failed > 0) { - string failureMsg = interceptingBus?.LastFailureMessage ?? "Test failed but no exception was captured."; - string stackTrace = interceptingBus?.LastFailureStackTrace; - bool isAssertionFailure = interceptingBus?.IsAssertionFailure ?? false; + string failureMsg = interceptingBus.LastFailureMessage ?? "Test failed but no exception was captured."; + string stackTrace = interceptingBus.LastFailureStackTrace; + bool isAssertionFailure = interceptingBus.IsAssertionFailure; string fullMessage = failureMsg + (stackTrace != null ? Environment.NewLine + "Stack Trace:" + Environment.NewLine + stackTrace : string.Empty); @@ -127,22 +280,29 @@ public override async Task RunAsync( lastException = new InvalidOperationException(fullMessage); } - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Test failed but exception is null, created placeholder exception (IsAssertion={isAssertionFailure})"); + Trace.WriteLine($"RetryFact: Test failed but exception is null, created placeholder exception (IsAssertion={isAssertionFailure})"); } - Trace.WriteLine($"##vso[task.logdetail]RetryFact test failed on attempt {attempt} with exception type: {lastException?.GetType().FullName ?? "null"}, Message: {lastException?.Message ?? "null"}"); + Trace.WriteLine($"RetryFact test failed on attempt {attempt} with exception type: {lastException?.GetType().FullName ?? "null"}, Message: {lastException?.Message ?? "null"}"); if (!isLastAttempt) { // Check if we should retry this exception (now handles null) var shouldRetry = ShouldRetry(lastException); - Trace.WriteLine($"##vso[task.logdetail]RetryFact ShouldRetry={shouldRetry} for exception type {lastException?.GetType().FullName ?? "null"}"); + Trace.WriteLine($"RetryFact ShouldRetry={shouldRetry} for exception type {lastException?.GetType().FullName ?? "null"}"); if (!shouldRetry) { - Trace.WriteLine($"##vso[task.logissue type=warning]Test '{TestMethod.Method.Name}' failed with non-retriable exception. Skipping retries."); - diagnosticMessageSink.OnMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.Method.Name}' failed with non-retriable exception. Skipping retries.")); + Console.WriteLine($"[RetryFact] Test '{TestDescription}' failed with a non-retriable exception on attempt {attempt}/{_maxRetries}. Skipping remaining retries."); + messageBus.QueueMessage( + new DiagnosticMessage($"[RetryFact] Test '{TestDescription}' failed with non-retriable exception. Skipping retries.")); + + // This attempt ran on the intercepting bus, which deferred the + // ITestFailed message so the test could be retried. We are not + // retrying, so replay it now: the reporters derive their results + // solely from the message bus, and without this the failure would + // disappear from the test results entirely and the run would pass. + StopRunIfRequested(interceptingBus.ReplayDeferredMessages(), cancellationTokenSource); if (lastException != null) { @@ -153,19 +313,49 @@ public override async Task RunAsync( return runSummary; } - // Not the last attempt - the failure was intercepted, so retry - Trace.WriteLine($"##vso[task.logissue type=warning]Test '{TestMethod.Method.Name}' failed on attempt {attempt}/{_maxRetries}, will retry after {_delayMs}ms"); - diagnosticMessageSink.OnMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' failed on attempt {attempt}/{_maxRetries}. Retrying after {_delayMs}ms delay. Error: {lastException?.Message ?? "No exception message"}")); + // Not the last attempt - the failure was intercepted, so retry. + // Carry the deferred failure forward rather than discarding it now: if the + // run is cancelled during the delay, or the next attempt never reports a + // result, this is the only remaining record that the test failed. + pendingFailureBus = interceptingBus; + + // Ownership moves to pendingFailureBus. Clear the local so this iteration's + // finally does not dispose it, which would replay the failure immediately. + interceptingBus = null; + + Console.WriteLine($"[RetryFact] Test '{TestDescription}' failed on attempt {attempt}/{_maxRetries}, retrying after {_delayMs}ms. Error: {lastException?.Message ?? "No exception message"}"); + messageBus.QueueMessage( + new DiagnosticMessage($"[RetryFact] Test '{TestDescription}' failed on attempt {attempt}/{_maxRetries}. Retrying after {_delayMs}ms delay. Error: {lastException?.Message ?? "No exception message"}")); + + try + { + await Task.Delay(_delayMs, cancellationTokenSource.Token); + } + catch (OperationCanceledException) + { + // Cancelled while waiting to retry. The failure from this attempt is + // still deferred, so report it instead of letting the test vanish. + Console.WriteLine($"[RetryFact] Test '{TestDescription}' was cancelled during the retry delay after attempt {attempt}/{_maxRetries}. Reporting the failure from that attempt."); + + pendingFailureBus.ReplayDeferredMessages(); - await Task.Delay(_delayMs); + // The run is already cancelling, so there is nothing further to stop and + // the replay result is deliberately ignored here. + if (lastException != null) + { + aggregator.Add(lastException); + } + + runSummary.Failed = 1; + return runSummary; + } } else { // Last attempt - failure message already went through to Test Explorer - Trace.WriteLine($"##vso[task.logissue type=error]Test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' failed after {_maxRetries} attempts. Final error: {lastException?.Message ?? "No exception captured"}"); - diagnosticMessageSink.OnMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' failed after {_maxRetries} attempts. Last exception: {lastException?.Message ?? "No exception message"}")); + Console.WriteLine($"[RetryFact] Test '{TestDescription}' failed after all {_maxRetries} attempts. Final error: {lastException?.Message ?? "No exception captured"}"); + messageBus.QueueMessage( + new DiagnosticMessage($"[RetryFact] Test '{TestDescription}' failed after {_maxRetries} attempts. Last exception: {lastException?.Message ?? "No exception message"}")); if (lastException != null) { @@ -173,7 +363,10 @@ public override async Task RunAsync( } else if (summary.Failed > 0) { - // Add an exception with captured failure details if test failed but no exception was captured + // Defensive only: a failing attempt always has an exception synthesized + // for it above, so reaching this means that invariant has been broken. + // Reporting something generic is still better than reporting a failure + // with nothing attached to explain it. aggregator.Add(new InvalidOperationException($"Test failed after {_maxRetries} attempts but no exception was captured")); } @@ -183,133 +376,530 @@ public override async Task RunAsync( } catch (Exception ex) { - Trace.WriteLine($"##vso[task.logissue type=error]RetryFact unexpected exception: {ex.GetType().FullName}: {ex.Message}"); + Console.WriteLine($"[RetryFact] Unexpected exception while running '{TestDescription}': {ex.GetType().FullName}: {ex.Message}"); + + // A crash must not swallow a failure an earlier attempt already produced, but at + // most one attempt's result may reach the bus: replaying both would publish two + // results for a single test. That is the same supersession rule the no-result + // path applies, so it is decided - and carried out - the same way here. + NoResultOutcome disposition = DecideNoResultOutcome( + currentAttemptObservedFailure: interceptingBus?.HasObservedFailure ?? false, + earlierAttemptObservedFailure: pendingFailureBus?.HasDeferredFailure ?? false); + + // The run is already unwinding, so a request to stop has nothing left to stop. + ReportSingleResult(disposition, interceptingBus, pendingFailureBus); + throw; } finally { - // Dispose the intercepting bus if we created one interceptingBus?.Dispose(); } } // Should never reach here - Trace.WriteLine($"##vso[task.logissue type=error]RetryFact WARNING: Reached end of retry loop unexpectedly"); + Console.WriteLine($"[RetryFact] Reached the end of the retry loop unexpectedly for '{TestDescription}'. Reporting the test as failed."); runSummary.Failed = 1; return runSummary; } - public override void Serialize(IXunitSerializationInfo data) + /// + /// Cancels the run when the message bus asked for it to stop. + /// + /// The result of replaying deferred messages. + /// The run's cancellation source. + /// + /// A deferred failure answers true to the runner when it is queued, because at that + /// point the test may still be retried. Replaying it is therefore the first and only chance + /// to honour a request to stop, such as the one --stop-on-fail makes. + /// + private static void StopRunIfRequested(bool continueRunning, CancellationTokenSource cancellationTokenSource) { - base.Serialize(data); - data.AddValue(nameof(_maxRetries), _maxRetries); - data.AddValue(nameof(_delayMs), _delayMs); - data.AddValue(nameof(_retryOnAssertionFailure), _retryOnAssertionFailure); + if (!continueRunning) + { + cancellationTokenSource.Cancel(); + } } - public override void Deserialize(IXunitSerializationInfo data) + /// + /// Decides how an attempt that published no result of its own should be accounted for. + /// + /// Whether this attempt saw a failure before it was cut short. + /// Whether an earlier attempt deferred a failure that has not been superseded. + /// Which deferred failure, if any, should reach the results. + /// + /// This rule governs both places an attempt can end without reporting: the empty-summary + /// path, and the crash path where an unexpected exception unwinds the attempt. Both have to + /// let at most one attempt's failure reach the bus, because replaying two would publish two + /// results for a single test, so both decide it here rather than each reimplementing it. + /// + internal static NoResultOutcome DecideNoResultOutcome(bool currentAttemptObservedFailure, bool earlierAttemptObservedFailure) { - base.Deserialize(data); - _maxRetries = data.GetValue(nameof(_maxRetries)); - _delayMs = data.GetValue(nameof(_delayMs)); - _retryOnAssertionFailure = data.GetValue(nameof(_retryOnAssertionFailure)); + if (currentAttemptObservedFailure) + { + return NoResultOutcome.ReplayCurrentAttempt; + } + + return earlierAttemptObservedFailure + ? NoResultOutcome.ReplayEarlierAttempt + : NoResultOutcome.ReportNothing; + } + + /// + /// Publishes at most one attempt's held-back messages, and drops the other attempt's. + /// + /// The outcome chosen by . + /// The bus this attempt ran on, or null once its ownership has moved. + /// The bus carrying a failure held over from an earlier attempt, if any. + /// false when the underlying bus asked for the run to stop. + /// + /// Deciding which attempt wins is not enough on its own: the loser's messages have to be + /// dropped as well, because a bus still holding messages replays them when it is disposed. + /// That fail-safe exists so a failure can never be lost, but it makes silence the one thing + /// a caller cannot express by omission - leaving the loser alone publishes it anyway, a + /// moment after the winner. An abstention held by an attempt that then crashed would arrive + /// as a second result on top of the earlier attempt's failure, and a skip landing after a + /// failure is exactly the reordering that turns a red test green. + /// + internal static bool ReportSingleResult( + NoResultOutcome outcome, + FailureInterceptingMessageBus currentAttempt, + FailureInterceptingMessageBus earlierAttempt) + { + switch (outcome) + { + case NoResultOutcome.ReplayCurrentAttempt: + earlierAttempt?.DiscardDeferredMessages(); + return currentAttempt?.ReplayDeferredMessages() ?? true; + + case NoResultOutcome.ReplayEarlierAttempt: + currentAttempt?.DiscardDeferredMessages(); + return earlierAttempt?.ReplayDeferredMessages() ?? true; + + case NoResultOutcome.ReportNothing: + // Nothing observed a failure, so there is no competition to resolve. Whatever the + // current attempt is still holding - an abstention, typically - is the only + // result the test has, and is left for its disposal to publish. + earlierAttempt?.DiscardDeferredMessages(); + currentAttempt?.PublishDeferredMessagesOnDispose(); + return true; + + default: + // An outcome added later and not handled here would publish nothing while + // CreateNoResultSummary counted a failure, and the runner's verdict comes from + // the messages rather than the summary, so the failure would be lost and the run + // would pass. Refusing an outcome nobody taught this how to report is the only + // answer that cannot end in a green run hiding one. + throw new ArgumentOutOfRangeException(nameof(outcome), outcome, "No handling is defined for this outcome, so a failure it stands for would not reach the results."); + } + } + + /// + /// Chooses the exception that belongs with the failure being reported for an attempt that + /// published no result of its own. + /// + /// The outcome chosen by . + /// The exception this attempt captured, if any. + /// The exception captured by the most recent attempt that failed, if any. + /// The exception to attach to the reported failure, or null when nothing should be attached. + /// + /// The exception has to describe the same failure that was replayed to the bus, otherwise + /// the run reports one attempt's failure alongside a different attempt's error text. So the + /// choice mirrors the bus exactly: the current attempt's failure carries the current + /// attempt's exception, an earlier attempt's failure carries the earlier one's, and an + /// attempt that reported nothing at all attaches nothing. Attaching the earlier exception to + /// a current-attempt failure is the specific mistake this guards against, because + /// lastException still holds the previous attempt's error at that point. + /// + internal static Exception SelectNoResultException(NoResultOutcome outcome, Exception currentAttemptException, Exception earlierAttemptException) + { + switch (outcome) + { + case NoResultOutcome.ReplayCurrentAttempt: + return currentAttemptException; + + case NoResultOutcome.ReplayEarlierAttempt: + return earlierAttemptException; + + case NoResultOutcome.ReportNothing: + return null; + + default: + throw new ArgumentOutOfRangeException(nameof(outcome), outcome, "No handling is defined for this outcome, so the exception reported with it would not describe the failure being replayed."); + } + } + + /// + /// Builds the summary for an attempt that published no result of its own. + /// + /// The outcome chosen by . + /// The totals this test contributes to the run. + /// + /// A replayed failure is a published result, so it counts as one failed test. When nothing + /// was ever published the test contributes nothing at all: counting it would claim an + /// outcome that no attempt reported, and a total of one with no failures reads as a pass. + /// + internal static RunSummary CreateNoResultSummary(NoResultOutcome outcome) + { + switch (outcome) + { + case NoResultOutcome.ReportNothing: + return new RunSummary { Total = 0, Failed = 0 }; + + case NoResultOutcome.ReplayCurrentAttempt: + case NoResultOutcome.ReplayEarlierAttempt: + return new RunSummary { Total = 1, Failed = 1 }; + + default: + throw new ArgumentOutOfRangeException(nameof(outcome), outcome, "No handling is defined for this outcome, so the totals it contributes would not match what was published."); + } + } + + protected override void Serialize(IXunitSerializationInfo info) + { + base.Serialize(info); + info.AddValue(nameof(_maxRetries), _maxRetries); + info.AddValue(nameof(_delayMs), _delayMs); + info.AddValue(nameof(_retryOnAssertionFailure), _retryOnAssertionFailure); + } + + protected override void Deserialize(IXunitSerializationInfo info) + { + base.Deserialize(info); + _maxRetries = Math.Max(1, info.GetValue(nameof(_maxRetries))); + _delayMs = Math.Max(0, info.GetValue(nameof(_delayMs))); + _retryOnAssertionFailure = info.GetValue(nameof(_retryOnAssertionFailure)); } /// /// Determines if an exception should trigger a retry. /// + /// + /// The whole exception tree is searched, not just the outermost exception or a chain of + /// single-inner aggregates. An assertion that fails inside Task.WhenAll, a parallel + /// helper, or any teardown that runs alongside it arrives wrapped in an + /// that may hold several inner exceptions, and stopping at + /// the first fork would classify it as an ordinary exception and retry it - the very thing + /// was set to prevent. This matches how the message bus + /// classifies the same failure, which reads the full chain of reported type names; the two + /// have to agree, or whether the policy is honoured would depend on which of them happened to + /// see the failure first. + /// + /// A timeout needs no carve-out here. The bus classifies by type name and so has to exclude + /// Xunit.Sdk.TestTimeoutException explicitly, but that type derives from + /// rather than , so it never matches the + /// test below and is retried, which is what a timeout deserves. + /// + /// private bool ShouldRetry(Exception ex) { // If exception is null, we should retry (something went wrong with exception capture) if (ex == null) { - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Exception is null, will retry"); + Trace.WriteLine($"RetryFact: Exception is null, will retry"); return true; // Retry when we can't determine the exception type } - // Unwrap aggregate exceptions - while (ex is AggregateException aggEx && aggEx.InnerExceptions.Count == 1) + // Don't retry assertion failures unless explicitly configured + if (ContainsAssertionFailure(ex)) { - ex = aggEx.InnerException; + Trace.WriteLine($"RetryFact: {(_retryOnAssertionFailure ? "Retrying" : "Not retrying")} XunitException because _retryOnAssertionFailure is {_retryOnAssertionFailure.ToString().ToLowerInvariant()}"); + return _retryOnAssertionFailure; } - // Don't retry assertion failures unless explicitly configured - if (ex is XunitException) + // Retry everything else (network, timeout, SQL transient, etc.) + Trace.WriteLine($"RetryFact: Retrying non-assertion exception of type {ex.GetType().FullName}"); + return true; + } + + /// + /// Reports whether an exception is, or contains anywhere within it, an assertion failure. + /// + /// + /// Separate and internal so the decision can be pinned directly. In a real run the message bus + /// almost always classifies the failure first, because xunit reports failures through the bus + /// rather than letting them out of the attempt, which leaves this path reachable but rarely + /// taken - and a rarely taken path that disagrees with the common one is exactly the kind of + /// difference that only shows up as an unexplained retry long after the fact. + /// + /// The exception to classify. + /// true if an assertion failure appears anywhere in the exception tree. + internal static bool ContainsAssertionFailure(Exception exception) => + exception != null && EnumerateExceptionTree(exception).Any(inner => inner is XunitException); + + /// + /// Walks an exception and everything nested inside it, including every branch of an + /// . + /// + /// The exception to walk. + /// The exception followed by each exception nested within it. + private static IEnumerable EnumerateExceptionTree(Exception exception) + { + var pending = new Stack(); + pending.Push(exception); + + while (pending.Count > 0) { - if (!_retryOnAssertionFailure) + Exception current = pending.Pop(); + yield return current; + + if (current is AggregateException aggregate) { - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Not retrying XunitException because _retryOnAssertionFailure is false"); - return false; + foreach (Exception inner in aggregate.InnerExceptions) + { + if (inner != null) + { + pending.Push(inner); + } + } } - else + else if (current.InnerException != null) { - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Retrying XunitException because _retryOnAssertionFailure is true"); - return true; + pending.Push(current.InnerException); } } - - // Retry everything else (network, timeout, SQL transient, etc.) - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Retrying non-assertion exception of type {ex.GetType().FullName}"); - return true; } /// - /// Message bus that intercepts ONLY failure messages (ITestFailed). - /// Used on non-final retry attempts to suppress intermediate failures. - /// Success messages and all other messages always pass through. - /// Also captures failure details (messages and stack traces) for diagnostic purposes. + /// Message bus that defers failure messages (ITestFailed) so that an attempt which is + /// about to be retried leaves no trace in the test results. /// - private class FailureInterceptingMessageBus : IMessageBus + /// + /// Once a failure is seen, every subsequent message for that attempt is buffered too. + /// Reporters finalize a test when they see ITestFinished, so forwarding a failure after + /// its ITestFinished has already gone through has no effect - the failure would be + /// dropped and the test would disappear from the run entirely. Buffering the tail of the + /// attempt keeps the messages in their original order, so the caller can either discard + /// them (retrying) or replay them intact (reporting the failure). + /// + internal class FailureInterceptingMessageBus : IMessageBus { private readonly IMessageBus _innerBus; + private readonly bool _deferFailures; + private readonly bool _deferAbstentions; + private readonly List _deferredMessages = new List(); + private bool _deferring; + private bool _replayOnDisposeIsIntended; - public FailureInterceptingMessageBus(IMessageBus innerBus) + public FailureInterceptingMessageBus(IMessageBus innerBus, bool deferFailures, bool deferAbstentions = false) { _innerBus = innerBus; + _deferFailures = deferFailures; + _deferAbstentions = deferAbstentions; } public string LastFailureMessage { get; private set; } + /// + /// Gets or sets where this bus writes its own diagnostics, defaulting to the console. + /// + /// + /// Exists so a test can read what was reported without redirecting the console, which is + /// process-wide state and would make any test doing it unsafe to run alongside others. + /// + public Action DiagnosticLog { get; set; } = Console.WriteLine; + public string LastFailureStackTrace { get; private set; } public bool IsAssertionFailure { get; private set; } + /// + /// Gets a value indicating whether a failure was intercepted and is waiting to be + /// either discarded or replayed. + /// + public bool HasDeferredFailure => _deferredMessages.Count > 0; + + /// + /// Gets a value indicating whether this attempt saw a failure at all, whether or not it + /// was deferred. + /// + /// + /// This is not the same question as , and the difference + /// falls exactly on the last attempt: that one is constructed with + /// deferFailures: false, so its failures go straight through to the inner bus and + /// it never has one deferred. Asking whether the + /// current attempt failed therefore always answers no on the last attempt, and an + /// earlier attempt's held-over failure would then be replayed on top of the failure this + /// one already published - two results for a single test. + /// + public bool HasObservedFailure => LastFailureMessage != null; + public bool QueueMessage(IMessageSinkMessage message) { - // Intercept ONLY failure messages - suppress them for non-final attempts + if (_deferring) + { + // This attempt is already holding messages back - either a failure it means to + // retry, or an abstention that has to compete with an earlier failure - so + // everything after it is held too, and is neither published now nor examined + // below. The usual thing arriving here is the ITestFinished that follows a + // deferred failure, and it must not reach the inner bus while the result it + // belongs to is still being decided. + // + // A failure arriving here therefore never sets LastFailureMessage. On the retry + // path that costs nothing, because the failure that started the deferral set it + // already. On the abstention path it is deliberate: the outcome that follows + // replays the earlier attempt's failure, so a failure is published either way, + // and treating this attempt as a failure in its own right would publish two + // results for one test. + _deferredMessages.Add(message); + return true; + } + + if (_deferAbstentions && (message is ITestSkipped || message is ITestNotRun)) + { + // An attempt that skips itself has abstained rather than shown the test to be + // sound, and this attempt follows one that failed. Hold the abstention so the + // caller can choose between it and that earlier failure; publishing it here + // would leave the failure no way to be reported without a second result. + _deferring = true; + _deferredMessages.Add(message); + return true; + } + if (message is ITestFailed failed) { // Capture failure details for diagnostics - LastFailureMessage = failed.Messages != null && failed.Messages.Length > 0 - ? string.Join(Environment.NewLine, failed.Messages) - : failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 - ? string.Join(", ", failed.ExceptionTypes) - : "Unknown failure"; + if (failed.Messages != null && failed.Messages.Length > 0) + { + LastFailureMessage = string.Join(Environment.NewLine, failed.Messages); + } + else if (failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0) + { + LastFailureMessage = string.Join(", ", failed.ExceptionTypes); + } + else + { + LastFailureMessage = "Unknown failure"; + } LastFailureStackTrace = failed.StackTraces != null && failed.StackTraces.Length > 0 ? string.Join(Environment.NewLine, failed.StackTraces) : null; - // Detect if this is an assertion failure by checking exception types - // XUnit assertion exceptions typically have types containing "Xunit" or "Assert" - IsAssertionFailure = failed.ExceptionTypes != null && - failed.ExceptionTypes.Length > 0 && - (failed.ExceptionTypes[0].Contains("Xunit", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("Assert", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("EqualException", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("TrueException", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("FalseException", StringComparison.Ordinal)); - - return true; // Swallow the failure - we're going to retry + // Classify the failure as an assertion failure so RetryOnAssertionFailure can decide + // whether it is worth another attempt. This is a substring match, not a namespace + // check: xUnit's own assertion exceptions live under Xunit.Sdk, but assertion + // libraries used alongside it do not, and matching "Assert" anywhere in the type + // name catches those too. The trade is that an unrelated type whose name happens to + // contain either word is classified the same way, and with RetryOnAssertionFailure + // left at its default of false that costs the test its remaining attempts rather + // than spending one: an assertion failure is taken to be deterministic, so it is + // not retried. The failure is still reported red either way, so the cost is a + // flaky test going unretried, never a failure going unseen. + // + // A timeout is excluded because it is the one failure that trade cannot be made + // for. Xunit.Sdk.TestTimeoutException contains "Xunit" and so matched here, but a + // test that ran long because a dependency was slow once is the definition of the + // flakiness these attributes exist to absorb -- and unlike the cases above, it is + // not an assertion at all. It derives from System.Exception rather than + // XunitException, so it is only ever mistaken for one by way of this match. + // A wrapped assertion failure is still an assertion failure. Xunit reports the + // whole exception chain here, outer type first, so reading only the first entry + // classifies AggregateException(XunitException) -- what an assertion failing + // inside Task.WhenAll or a parallel helper produces -- as an ordinary exception + // and retries it, which is exactly what the policy was set to prevent. + string[] failedExceptionTypes = failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 + ? failed.ExceptionTypes + : new[] { string.Empty }; + + // The timeout carve-out is applied to the whole chain rather than one entry: a + // timeout wrapped in anything is still a timeout, and still the failure that most + // deserves another attempt. + IsAssertionFailure = + !failedExceptionTypes.Any(type => type != null && type.Contains("Timeout", StringComparison.Ordinal)) && + failedExceptionTypes.Any(type => type != null && + (type.Contains("Xunit", StringComparison.Ordinal) || + type.Contains("Assert", StringComparison.Ordinal))); + + if (_deferFailures) + { + _deferring = true; + _deferredMessages.Add(message); + return true; + } } // All other messages (ITestPassed, ITestStarting, ITestFinished, etc.) pass through return _innerBus.QueueMessage(message); } + /// + /// Replays the deferred failure and everything that followed it to the underlying bus, + /// in their original order. Call this when the attempt will not be retried, otherwise + /// the failure is never reported and the test silently vanishes from the results. + /// + /// + /// false when the underlying bus asked for the run to stop. Deferring a failure + /// forces a true back to the runner at the time it is queued, so this is the only + /// point at which a request to stop -- from --stop-on-fail, for instance -- can still be + /// honoured. + /// + public bool ReplayDeferredMessages() + { + bool continueRunning = true; + + foreach (IMessageSinkMessage message in _deferredMessages) + { + continueRunning &= _innerBus.QueueMessage(message); + } + + _deferredMessages.Clear(); + _deferring = false; + + return continueRunning; + } + + /// + /// Drops the deferred failure without reporting it. Call this only when a later attempt + /// has actually reported a result that supersedes it. + /// + public void DiscardDeferredMessages() + { + _deferredMessages.Clear(); + _deferring = false; + } + + /// + /// Records that whatever is still held is meant to be published by disposal rather than + /// by an explicit replay, so that disposal does not report it as a leak. + /// + /// + /// Disposal replaying held messages is both the fail-safe for a caller that forgot them + /// and the ordinary way the last surviving result reaches the bus. Without a way to tell + /// those apart, the fail-safe announces an internal error on a routine path, and a reader + /// who sees that line on every run has no reason to believe it the one time it means + /// something. This marks the deliberate case so the warning stays rare enough to trust. + /// + public void PublishDeferredMessagesOnDispose() + { + _replayOnDisposeIsIntended = true; + } + public void Dispose() { - // Don't dispose the inner bus - it's owned by the caller + // Don't dispose the inner bus - it's owned by the caller. + // Anything still deferred is reported either way, because dropping it would remove + // the test from the run. Only an unannounced one is a defect worth saying so about. + if (_deferredMessages.Count > 0) + { + if (!_replayOnDisposeIsIntended) + { + DiagnosticLog($"[RetryFact] Internal error: {_deferredMessages.Count} deferred test message(s) were neither replayed nor discarded. Replaying them so the failure is not lost."); + } + + try + { + ReplayDeferredMessages(); + } + catch (Exception e) + { + // Dispose runs on the way out of the attempt, including while an exception is + // already unwinding. Throwing from here would replace whatever actually went + // wrong with a reporting error, so the failure to replay is logged and the + // original exception is left to propagate. + DiagnosticLog($"[RetryFact] Failed to replay {_deferredMessages.Count} deferred test message(s): {e.GetType().FullName}: {e.Message}"); + } + } } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs index 88a65b6b54..97e97fb759 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs @@ -4,8 +4,9 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Runtime.CompilerServices; using Xunit; -using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { @@ -13,12 +14,23 @@ namespace Microsoft.Health.Extensions.Xunit /// Attribute that marks a theory method to be retried a specified number of times if it fails. /// Useful for handling transient failures in integration and end-to-end tests with parameterized data. /// - [XunitTestCaseDiscoverer("Microsoft.Health.Extensions.Xunit.RetryTheoryDiscoverer", "Microsoft.Health.Extensions.Xunit")] + [XunitTestCaseDiscoverer(typeof(RetryTheoryDiscoverer))] [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class RetryTheoryAttribute : TheoryAttribute { /// - /// Gets or sets the maximum number of retry attempts (default is 3). + /// Initializes a new instance of the class. + /// + /// The source file containing the test method. Supplied by the compiler. + /// The line number of the test method. Supplied by the compiler. + public RetryTheoryAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) + { + } + + /// + /// Gets or sets the maximum number of attempts, including the first run (default is 3). + /// A value of 3 therefore runs a failing test three times in total, not four. /// public int MaxRetries { get; set; } = 3; @@ -28,9 +40,16 @@ public sealed class RetryTheoryAttribute : TheoryAttribute public int DelayBetweenRetriesMs { get; set; } = 5000; /// - /// Gets or sets whether to retry on assertion failures (XunitException). + /// Gets or sets whether to retry on assertion failures. /// Default is false - assertion failures usually indicate test bugs, not transient issues. /// Set to true for tests that validate eventually-consistent systems (e.g., cache refresh, reindex operations). + /// + /// What counts as an assertion failure is decided from the reported exception's type name - + /// one containing "Xunit" or "Assert", so that assertion libraries used alongside xUnit are + /// recognised too - rather than from the exception type itself, which the message bus does + /// not receive. Timeouts are deliberately excluded: they match that name test but are the + /// canonical transient failure, so they are retried whatever this is set to. + /// /// public bool RetryOnAssertionFailure { get; set; } = false; } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index 9539e3c1db..c5954fbda3 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -3,10 +3,13 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using System.Linq; -using Xunit.Abstractions; +using System.Threading.Tasks; +using Xunit; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { @@ -15,77 +18,92 @@ namespace Microsoft.Health.Extensions.Xunit /// For Theory tests, we need to let xUnit discover the data-driven test cases first, /// then wrap each one with retry logic. /// - public class RetryTheoryDiscoverer : TheoryDiscoverer + public sealed class RetryTheoryDiscoverer : TheoryDiscoverer { - public RetryTheoryDiscoverer(IMessageSink diagnosticMessageSink) - : base(diagnosticMessageSink) + protected override async ValueTask> CreateTestCasesForDataRow( + ITestFrameworkDiscoveryOptions discoveryOptions, + IXunitTestMethod testMethod, + ITheoryAttribute theoryAttribute, + ITheoryDataRow dataRow, + object[] testMethodArguments) { + return WrapAll( + await base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments), + theoryAttribute); } - protected override IEnumerable CreateTestCasesForDataRow( + protected override async ValueTask> CreateTestCasesForTheory( ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo theoryAttribute, - object[] dataRow) + IXunitTestMethod testMethod, + ITheoryAttribute theoryAttribute) { - var maxRetries = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.MaxRetries)); - var delayMs = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); - var retryOnAssertionFailure = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.RetryOnAssertionFailure)); - - // Use default values if not specified - if (maxRetries == 0) - { - maxRetries = 3; - } - - if (delayMs == 0) - { - delayMs = 5000; - } - - // Create a RetryTestCase for each data row - yield return new RetryTestCase( - DiagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), - testMethod, - maxRetries, - delayMs, - retryOnAssertionFailure, - dataRow); + return WrapAll( + await base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute), + theoryAttribute); } - protected override IEnumerable CreateTestCasesForTheory( - ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo theoryAttribute) + private static IXunitTestCase[] WrapAll( + IReadOnlyCollection baseCases, + ITheoryAttribute theoryAttribute) { - var maxRetries = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.MaxRetries)); - var delayMs = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); - var retryOnAssertionFailure = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.RetryOnAssertionFailure)); + var attribute = (RetryTheoryAttribute)theoryAttribute; + return baseCases + .Select(testCase => WrapTestCase(testCase, attribute)) + .ToArray(); + } - // Use default values if not specified - if (maxRetries == 0) + private static IXunitTestCase WrapTestCase(IXunitTestCase testCase, RetryTheoryAttribute attribute) + { + if (testCase is IXunitDelayEnumeratedTestCase) { - maxRetries = 3; + // A delay-enumerated case has not resolved its data rows yet, and it resolves them + // itself at run time. RetryTestCase cannot stand in for that: it would run the + // method with the arguments it was constructed with -- none -- and every row of the + // theory would be lost with an arity error rather than run. Returning the case + // unwrapped costs the retries but keeps the tests, which is the better trade. + // Theories reach this path when their data cannot be pre-enumerated, or when + // pre-enumeration is turned off. + Console.WriteLine( + $"[RetryTheory] WARNING: Test case '{testCase.TestCaseDisplayName}' resolves its data at run time, so retry logic will NOT be applied to it. " + + "Use pre-enumerable theory data (for example [InlineData], or [MemberData] whose values are serializable) if the rows need to be retried."); + return testCase; } - if (delayMs == 0) + if (testCase is not XunitTestCase xunitTestCase) { - delayMs = 5000; + // Trace output only reaches an attached debugger, so a Trace-only warning here + // means retries are silently not applied in CI. Every test case type xunit.v3 + // produces derives from XunitTestCase, so this only happens if a custom + // discoverer introduces its own type. + Console.WriteLine( + $"[RetryTheory] WARNING: Test case '{testCase.TestCaseDisplayName}' is {testCase.GetType().Name}, not {nameof(XunitTestCase)}. Retry logic will NOT be applied to it."); + return testCase; } - // For theories without data (will be skipped), wrap in retry - return base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute) - .Select(testCase => new RetryTestCase( - DiagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), - testMethod, - maxRetries, - delayMs, - retryOnAssertionFailure, - testCase.TestMethodArguments)); + // The base case carries the method xUnit resolved for this row, including any generic + // arguments closed from the row's data. The method passed to the discoverer is still + // open, so using it here would discard that resolution. + return new RetryTestCase( + xunitTestCase.TestMethod, + xunitTestCase.TestCaseDisplayName, + xunitTestCase.UniqueID, + xunitTestCase.Explicit, + xunitTestCase.SkipExceptions, + xunitTestCase.SkipReason, + xunitTestCase.SkipType, + xunitTestCase.SkipUnless, + xunitTestCase.SkipWhen, + xunitTestCase.Traits.ToDictionary( + kvp => kvp.Key, + kvp => new HashSet(kvp.Value, StringComparer.OrdinalIgnoreCase), + StringComparer.OrdinalIgnoreCase), + xunitTestCase.TestMethodArguments, + xunitTestCase.SourceFilePath, + xunitTestCase.SourceLineNumber, + xunitTestCase.Timeout == 0 ? null : xunitTestCase.Timeout, + attribute.MaxRetries, + attribute.DelayBetweenRetriesMs, + attribute.RetryOnAssertionFailure); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/SingleFlag.cs b/src/Microsoft.Health.Extensions.Xunit/SingleFlag.cs index b52088ef5f..9dbb201fb2 100644 --- a/src/Microsoft.Health.Extensions.Xunit/SingleFlag.cs +++ b/src/Microsoft.Health.Extensions.Xunit/SingleFlag.cs @@ -39,7 +39,10 @@ public SingleFlag(Enum enumValue) public bool Equals(SingleFlag other) { - return other.EnumValue.Equals(EnumValue); + // Not other.EnumValue.Equals(...): this is a struct, so default(SingleFlag) reaches here + // with a null EnumValue without ever passing through the constructor. GetHashCode already + // tolerates that, and an Equals that throws on the same value contradicts it. + return Equals(other.EnumValue, EnumValue); } public override bool Equals(object obj) diff --git a/src/Microsoft.Health.Extensions.Xunit/Skip.cs b/src/Microsoft.Health.Extensions.Xunit/Skip.cs new file mode 100644 index 0000000000..5e2ce9997b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/Skip.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit.Sdk; + +namespace Xunit +{ + /// + /// Provides legacy SkippableFact-compatible helpers for conditional skips. + /// + public static class Skip + { + /// + /// Skips the current test when the condition is true. + /// + /// The condition that triggers the skip. + /// The reason for the skip. + public static void If(bool condition, string reason = null) + { + if (condition) + { + throw SkipException.ForSkip(reason ?? "Skipped"); + } + } + + /// + /// Skips the current test when the condition is false. + /// + /// The condition that prevents the skip. + /// The reason for the skip. + public static void IfNot(bool condition, string reason = null) + { + if (!condition) + { + throw SkipException.ForSkip(reason ?? "Skipped"); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs new file mode 100644 index 0000000000..b327ae1a35 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Runtime.CompilerServices; + +namespace Xunit +{ + /// + /// Compatibility attribute for legacy SkippableFact usage. In xUnit v3 a dynamically skipped test is expressed + /// through Assert.Skip rather than a separate attribute, so this is a source-compatibility alias for the plain + /// fact attribute and exists only so existing call sites keep compiling. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public sealed class SkippableFactAttribute : FactAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The source file containing the test method. Supplied by the compiler. + /// The line number of the test method. Supplied by the compiler. + public SkippableFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs new file mode 100644 index 0000000000..2d22200042 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Runtime.CompilerServices; + +namespace Xunit +{ + /// + /// Compatibility attribute for legacy SkippableTheory usage. In xUnit v3 a dynamically skipped test is expressed + /// through Assert.Skip rather than a separate attribute, so this is a source-compatibility alias for the plain + /// theory attribute and exists only so existing call sites keep compiling. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public sealed class SkippableTheoryAttribute : TheoryAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The source file containing the test method. Supplied by the compiler. + /// The line number of the test method. Supplied by the compiler. + public SkippableTheoryAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArguments.cs b/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArguments.cs deleted file mode 100644 index b7c609b0ae..0000000000 --- a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArguments.cs +++ /dev/null @@ -1,91 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using EnsureThat; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Microsoft.Health.Extensions.Xunit -{ - /// - /// A where the class' is a . - /// This test class uses a fixture with a single set of constructor arguments. - /// - internal class TestClassWithFixtureArguments : LongLivedMarshalByRefObject, ITestClass - { - private ITypeInfo _underlyingClass; - - [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] - public TestClassWithFixtureArguments() - { - } - - public TestClassWithFixtureArguments(ITestCollection testCollection, ITypeInfo underlyingClass, SingleFlag[] fixtureArguments) - { - EnsureArg.IsNotNull(testCollection, nameof(testCollection)); - EnsureArg.IsNotNull(underlyingClass, nameof(underlyingClass)); - EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - - _underlyingClass = underlyingClass; - Class = new TestClassWithFixtureArgumentsTypeInfo(underlyingClass, fixtureArguments); - FixtureArguments = fixtureArguments; - TestCollection = testCollection; - } - - /// - public ITypeInfo Class { get; set; } - - public IReadOnlyList FixtureArguments { get; set; } - - /// - public ITestCollection TestCollection { get; set; } - - /// - public void Serialize(IXunitSerializationInfo info) - { - info.AddValue("TestCollection", TestCollection); - info.AddValue("ClassAssemblyName", _underlyingClass.Assembly.Name); - info.AddValue("ClassTypeName", _underlyingClass.Name); - info.AddValue("FixtureArguments", FixtureArguments.Select(a => a.EnumValue).ToArray()); - } - - /// - public void Deserialize(IXunitSerializationInfo info) - { - TestCollection = info.GetValue("TestCollection"); - - var assemblyName = info.GetValue("ClassAssemblyName"); - var typeName = info.GetValue("ClassTypeName"); - - _underlyingClass = Reflector.Wrap(GetType(assemblyName, typeName)); - - Enum[] rawFixtureArguments = info.GetValue("FixtureArguments"); - - FixtureArguments = Array.ConvertAll(rawFixtureArguments, e => new SingleFlag(e)); - - Class = new TestClassWithFixtureArgumentsTypeInfo(_underlyingClass, FixtureArguments); - } - - /// - /// Converts an assembly name + type name into a object. - /// - /// The assembly name. - /// The type name. - /// The instance of the , if available; null, otherwise. - public static Type GetType(string assemblyName, string typeName) - { - // Make sure we only use the short form - var an = new AssemblyName(assemblyName); - var assembly = Assembly.Load(new AssemblyName { Name = an.Name, Version = an.Version }); - - return assembly == null ? null : assembly.GetType(typeName); - } - } -} diff --git a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs b/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs deleted file mode 100644 index 58c5052f3a..0000000000 --- a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs +++ /dev/null @@ -1,76 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using EnsureThat; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Microsoft.Health.Extensions.Xunit -{ - /// - /// A special for a test class that will use a fixture instantiated with with a single set of constructor arguments. - /// The property is customized to be of the form Namespace.Class(Arg1, Arg2) - /// - internal sealed class TestClassWithFixtureArgumentsTypeInfo : LongLivedMarshalByRefObject, IReflectionTypeInfo - { - private readonly ITypeInfo _typeInfoImplementation; - - public TestClassWithFixtureArgumentsTypeInfo(ITypeInfo typeInfoImplementation, IReadOnlyList fixtureArguments) - { - EnsureArg.IsNotNull(typeInfoImplementation, nameof(typeInfoImplementation)); - EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - - _typeInfoImplementation = typeInfoImplementation; - FixtureArguments = fixtureArguments; - Name = $"{typeInfoImplementation.Name}({string.Join(", ", fixtureArguments.Select(v => $"{v.EnumValue}"))})"; - } - - public IReadOnlyList FixtureArguments { get; } - - public IAssemblyInfo Assembly => _typeInfoImplementation.Assembly; - - public ITypeInfo BaseType => _typeInfoImplementation.BaseType; - - public IEnumerable Interfaces => _typeInfoImplementation.Interfaces; - - public bool IsAbstract => _typeInfoImplementation.IsAbstract; - - public bool IsGenericParameter => _typeInfoImplementation.IsGenericParameter; - - public bool IsGenericType => _typeInfoImplementation.IsGenericType; - - public bool IsSealed => _typeInfoImplementation.IsSealed; - - public bool IsValueType => _typeInfoImplementation.IsValueType; - - public string Name { get; } - - public Type Type => ((ReflectionTypeInfo)_typeInfoImplementation).Type; - - public IEnumerable GetCustomAttributes(string assemblyQualifiedAttributeTypeName) - { - return _typeInfoImplementation.GetCustomAttributes(assemblyQualifiedAttributeTypeName); - } - - public IEnumerable GetGenericArguments() - { - return _typeInfoImplementation.GetGenericArguments(); - } - - public IMethodInfo GetMethod(string methodName, bool includePrivateMethod) - { - return _typeInfoImplementation.GetMethod(methodName, includePrivateMethod); - } - - public IEnumerable GetMethods(bool includePrivateMethods) - { - return _typeInfoImplementation.GetMethods(includePrivateMethods); - } - } -} diff --git a/src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj b/src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj index 0d1e3c17ba..694edc9106 100644 --- a/src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.Azure.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Azure.UnitTests/AssemblyValidationsTests.cs index 17db94a612..0e86beda26 100644 --- a/src/Microsoft.Health.Fhir.Azure.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Azure.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj b/src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj index 6f119c7bae..5b5df19d76 100644 --- a/src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj @@ -6,12 +6,12 @@ - + - + diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/AssemblyValidationsTests.cs index 3129ce935e..94143674ee 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XUnitLogger.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XUnitLogger.cs index 0bf0debf5e..19a8d60f94 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XUnitLogger.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XUnitLogger.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Health.Extensions.DependencyInjection; using NSubstitute; -using Xunit.Abstractions; +using Xunit; namespace Microsoft.Health.Fhir.Core.UnitTests.Extensions; diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XunitAttributeSourceLocationTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XunitAttributeSourceLocationTests.cs new file mode 100644 index 0000000000..5c9c902752 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XunitAttributeSourceLocationTests.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Extensions +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Web)] + public sealed class XunitAttributeSourceLocationTests + { + [Fact] + public void GivenCustomFactAttributes_WhenConstructedWithSourceLocation_ThenBaseSourceLocationIsPopulated() + { + const string sourceFilePath = "test-file.cs"; + const int sourceLineNumber = 123; + + var retryFact = new RetryFactAttribute(sourceFilePath, sourceLineNumber); + var skippableFact = new SkippableFactAttribute(sourceFilePath, sourceLineNumber); + + Assert.Equal(sourceFilePath, retryFact.SourceFilePath); + Assert.Equal(sourceLineNumber, retryFact.SourceLineNumber); + Assert.Equal(sourceFilePath, skippableFact.SourceFilePath); + Assert.Equal(sourceLineNumber, skippableFact.SourceLineNumber); + } + + [Fact] + public void GivenCustomTheoryAttributes_WhenConstructedWithSourceLocation_ThenBaseSourceLocationIsPopulated() + { + const string sourceFilePath = "test-file.cs"; + const int sourceLineNumber = 123; + + var retryTheory = new RetryTheoryAttribute(sourceFilePath, sourceLineNumber); + var skippableTheory = new SkippableTheoryAttribute(sourceFilePath, sourceLineNumber); + + Assert.Equal(sourceFilePath, retryTheory.SourceFilePath); + Assert.Equal(sourceLineNumber, retryTheory.SourceLineNumber); + Assert.Equal(sourceFilePath, skippableTheory.SourceFilePath); + Assert.Equal(sourceLineNumber, skippableTheory.SourceLineNumber); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj index 5b9bc2a7af..51fc2861bb 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj @@ -14,7 +14,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj index 57eb3e5ddb..cd8ab85abd 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj @@ -2,7 +2,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyValidationsTests.cs index 51fccee340..217879340f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj index 4363686f48..51736adb61 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj @@ -3,7 +3,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj index 2550df59fe..202c89c855 100644 --- a/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj @@ -18,9 +18,8 @@ - + - diff --git a/src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj index 0c6a754cd0..9ad126019d 100644 --- a/src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj index 9a2b9e7804..96ffb90696 100644 --- a/src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj index c566ec573f..428517577d 100644 --- a/src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj @@ -12,9 +12,8 @@ - + - diff --git a/src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj index 50b6b8b4df..5f29c3dfd9 100644 --- a/src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj b/src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj index bafbc979f9..3408dc3016 100644 --- a/src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj index e5201cb0c7..e61a37fd94 100644 --- a/src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj @@ -11,9 +11,8 @@ - + - diff --git a/src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj b/src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj index bd10b85ab1..829b474b36 100644 --- a/src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/AssemblyValidationsTests.cs index 00e30ede2a..591690d2c7 100644 --- a/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj b/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj index 7674c947ea..8f0cc39410 100644 --- a/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj @@ -10,8 +10,8 @@ - - + + diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyValidationsTests.cs index 65f75a5b47..cec3ebce3d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Throttling/ThrottlingMiddlewareTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Throttling/ThrottlingMiddlewareTests.cs index 34bcd7ada0..7fcccddd3d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Throttling/ThrottlingMiddlewareTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Throttling/ThrottlingMiddlewareTests.cs @@ -38,7 +38,7 @@ namespace Microsoft.Health.Fhir.Shared.Api.UnitTests.Features.Throttling [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Throttling)] [Trait(Traits.Category, Categories.Web)] - public class ThrottlingMiddlewareTests : IAsyncLifetime + public class ThrottlingMiddlewareTests : IAsyncLifetime, IAsyncDisposable { private HttpContext _httpContext = new DefaultHttpContext(); private Lazy _middleware; @@ -316,8 +316,8 @@ public async Task GivenARequest_ThatResultsInRequestRateExceeded_ReturnsValidFhi return output; } - public Task InitializeAsync() => Task.CompletedTask; + public ValueTask InitializeAsync() => ValueTask.CompletedTask; - async Task IAsyncLifetime.DisposeAsync() => await _middleware.Value.DisposeAsync(); + public ValueTask DisposeAsync() => _middleware.Value.DisposeAsync(); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyValidationsTests.cs index 0dc8b17c31..31ecc88237 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs index a1b838eeba..bd59fc91e9 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -20,7 +20,6 @@ using Newtonsoft.Json; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterComparerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterComparerTests.cs index 6bb613b4f1..8061b5dbd3 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterComparerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterComparerTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -20,7 +20,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Shared.Core.UnitTests.Features.Search.SearchParameters { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs index 29df1f9c93..96284b5780 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; @@ -162,12 +163,12 @@ public SearchParameterDefinitionManagerTests() NullLogger.Instance); } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _searchParameterDefinitionManager.EnsureInitializedAsync(CancellationToken.None); } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; [Fact] public async Task GivenSupportedParams_WhenGettingSupported_ThenSupportedParamsReturned() diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs index 969ab8d328..ffb73da7fe 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs @@ -20,13 +20,13 @@ public class SearchParameterSupportResolverTests : IAsyncLifetime { private SearchParameterSupportResolver _resolver; - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { _resolver = new SearchParameterSupportResolver( await SearchParameterFixtureData.GetFhirTypedElementToSearchValueConverterManagerAsync()); } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; [Fact] public void GivenASupportedSearchParameter_WhenResolvingSupport_ThenTrueIsReturned() diff --git a/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs index c1e8771303..43fc83f87d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -10,6 +10,19 @@ namespace Microsoft.Health.Fhir.Shared.Tests { + /// + /// Assembly-wide checks, from the shared project the unit, integration, and E2E assemblies all + /// compile. + /// + /// + /// This copy deliberately carries no DataStore trait. It compiles into the integration + /// assemblies, whose legs select by excluding the other store, and an exclusion drops a + /// case when any of its values under that trait matches - so naming both stores here would hide + /// these checks from both integration legs rather than reveal them. The consequence is that in + /// an E2E assembly this copy is invisible to the positively-filtered legs, and a check added + /// only here would not run there. E2E-visible checks belong in the copy under + /// Microsoft.Health.Fhir.Shared.Tests.E2E, which is traited for exactly that reason. + /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.AssemblyValidation)] public sealed class AssemblyValidationsTests @@ -27,5 +40,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/AssemblyValidationsTests.cs index 87eb36697f..6d9525446e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesTestsCollection.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesTestsCollection.cs new file mode 100644 index 0000000000..369f10ccc7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesTestsCollection.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search +{ + /// + /// Disables parallelization for CustomQueries tests due to shared static state. + /// + [CollectionDefinition("CustomQueriesTests", DisableParallelization = true)] + public sealed class CustomQueriesTestsCollection + { + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs index 5e6066b0d8..6819dd96b2 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs @@ -16,14 +16,20 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Search)] + [Collection("CustomQueriesTests")] public class CustomQueriesUnitTests { [Fact] public async Task GivenCustomQueryClass_WithGivenWaitTime_QueryToDBWillOccurOnlyAfterWaitPeriod() { - // set wait time to 2 seconds; + // set wait time to 1 second CustomQueries.WaitTime = 1; + // CheckQueryHash only reaches the database once the wait period has elapsed since the + // last query, and that timestamp is static state shared with the sibling test. Waiting + // it out here means the first call below always queries, whichever test ran first. + await Task.Delay(1100); + var connection = Substitute.For(); var command = Substitute.For(); var reader = Substitute.For(); diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs index 4dfb489fc8..2d13f925c2 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs @@ -39,12 +39,12 @@ public IncludeRewriterTests(IncludeRewriterFixture fixture) _searchParameterDefinitionManager = fixture.SearchParameterDefinitionManager; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _fixture.Start(); } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; // Basic Queries with 0-2 include search parameters with all the pair combinations diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs index 2f85f02419..adc94404ac 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs @@ -20,6 +20,7 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Search)] + [Collection(ModelInfoProviderCollection.Name)] public class ScalarTemporalEqualityRewriterTests : IClassFixture { private static readonly DateTimeOffset StartOfDay = new DateTimeOffset(2016, 7, 6, 0, 0, 0, TimeSpan.Zero); diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/QueryGenerators/CompartmentQueryGeneratorTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/QueryGenerators/CompartmentQueryGeneratorTests.cs index 2826d89509..3d39b5aa44 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/QueryGenerators/CompartmentQueryGeneratorTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/QueryGenerators/CompartmentQueryGeneratorTests.cs @@ -28,6 +28,7 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions. /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Search)] + [Collection(ModelInfoProviderCollection.Name)] public class CompartmentQueryGeneratorTests : IClassFixture { private readonly ISqlServerFhirModel _model; diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs index b96f069f66..27b38fbcf0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs @@ -46,6 +46,7 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Storage { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.DataSourceValidation)] + [Collection(ModelInfoProviderCollection.Name)] public class SqlServerFhirDataStoreUnitTests { public static IEnumerable RemoveTrailingZerosTestCases() diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj index ed90df8d67..501d312522 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj @@ -1,7 +1,7 @@  - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/ModelInfoProviderCollection.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/ModelInfoProviderCollection.cs new file mode 100644 index 0000000000..3af5cbb53c --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/ModelInfoProviderCollection.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests +{ + /// + /// Serializes the test classes that read or write the process-global ModelInfoProvider. + /// + /// + /// ModelInfoProvider is a plain static, so a class that replaces it races any class that + /// is relying on it. Concretely, SqlServerFhirDataStoreUnitTests installs a provider that + /// knows only the Group resource type and no compartments, while + /// CompartmentQueryGeneratorTests and ScalarTemporalEqualityRewriterTests need the + /// compartment-aware provider their class fixture installs. When those run concurrently the + /// compartment tests fail with "Expected an expression that evaluates to true", and which tests + /// fail varies from run to run. + /// + /// A class fixture cannot fix this, because it only orders the classes that share it and does + /// nothing about a third class overwriting the same static. Sharing one collection is what + /// actually serializes them: xUnit runs the classes within a collection one after another. + /// DisableParallelization additionally keeps the collection from overlapping other + /// collections, so a future class that touches the provider cannot reintroduce the race without + /// also being added here. + /// + /// The race is latent rather than new, but xUnit v3 schedules these classes concurrently where + /// v2 did not, so the migration is what makes it reproduce. + /// + [CollectionDefinition(Name, DisableParallelization = true)] + public sealed class ModelInfoProviderCollection + { + /// + /// The collection name to put on every test class that reads or writes the global provider. + /// + public const string Name = "ModelInfoProviderTests"; + } +} diff --git a/src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj b/src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj index ae19182c03..ebe6a7f3e5 100644 --- a/src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj @@ -10,7 +10,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj index 8864a3ee60..a1ccc295c8 100644 --- a/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj @@ -15,12 +15,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj b/src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj index 28437cd92a..0aeaa9dc69 100644 --- a/src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs index f4e2e82517..5ddb607e8f 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs @@ -9,6 +9,7 @@ using System.Reflection; using EnsureThat; using Xunit; +using Xunit.v3; namespace Microsoft.Health.Fhir.Tests.Common { @@ -52,6 +53,209 @@ public static IEnumerable ScanTestsLookingForMissingTrait(Assembly assembl return notComplianceTestClass; } + /// + /// Looks for collection definitions that carry traits and are shared with other classes. + /// + /// + /// A trait on a [CollectionDefinition] class is applied to every class that joins that + /// collection, which is not obvious at either end: the trait is written in one file and takes + /// effect in others, and nothing at the joining class says so. This repository's CI legs select + /// on traits, so a collection that quietly adds a Category can move its members out of a + /// leg that excludes that category - and a leg that runs fewer tests than it should still + /// passes. That is not hypothetical; it is what this check was written after. + /// + /// A definition that carries traits but that no other class joins is left alone: it applies + /// them only to itself, which is the same as writing them on the class. The fix when this does + /// fire is the pattern already used elsewhere here - move the collection name onto a separate, + /// traitless definition class, and leave the traits on the test class that declared them. + /// + /// + /// Assembly under analysis. + /// The offending definition types, each with the classes that join its collection. + public static IReadOnlyList<(Type Definition, IReadOnlyList Members)> ScanForTraitCarryingSharedCollectionDefinitions(Assembly assembly) + { + EnsureArg.IsNotNull(assembly, nameof(assembly)); + + Type[] classes = assembly.GetTypes().Where(t => t.IsClass).ToArray(); + + var offenders = new List<(Type, IReadOnlyList)>(); + + foreach (Type definition in classes) + { + CustomAttributeData definitionAttribute = definition.CustomAttributes + .FirstOrDefault(a => a.AttributeType == typeof(CollectionDefinitionAttribute)); + + // Any ITraitAttribute propagates, not just the [Trait] spelling of one, so ask the + // question xUnit asks rather than naming a single attribute type. The scan above is + // deliberately narrower: it reads a trait's name out of the first constructor + // argument, which is a shape only TraitAttribute guarantees. + if (definitionAttribute == null || + !definition.CustomAttributes.Any(a => typeof(ITraitAttribute).IsAssignableFrom(a.AttributeType))) + { + continue; + } + + // A collection definition can name its collection with a string or, since v3, be + // referenced by its own type. Both spellings have to be matched to find the members. + string collectionName = definitionAttribute.ConstructorArguments.Count > 0 + ? definitionAttribute.ConstructorArguments[0].Value as string + : null; + + List members = classes + .Where(candidate => candidate != definition && JoinsCollection(candidate, collectionName, definition)) + .ToList(); + + if (members.Count > 0) + { + offenders.Add((definition, members)); + } + } + + return offenders; + } + + private static bool JoinsCollection(Type candidate, string collectionName, Type definition) + { + foreach (CustomAttributeData attribute in candidate.CustomAttributes + .Where(a => typeof(ICollectionAttribute).IsAssignableFrom(a.AttributeType))) + { + // A class joins a collection by name, by naming the definition type, or - the v3 + // spelling this repository is moving to - as [Collection], which carries + // the definition in its generic argument and takes no constructor argument at all. + if (attribute.AttributeType.IsGenericType && + attribute.AttributeType.GetGenericTypeDefinition() == typeof(CollectionAttribute<>)) + { + if (attribute.AttributeType.GetGenericArguments()[0] == definition) + { + return true; + } + + continue; + } + + if (attribute.ConstructorArguments.Count == 0) + { + continue; + } + + object value = attribute.ConstructorArguments[0].Value; + + if (value is string name && collectionName != null && string.Equals(name, collectionName, StringComparison.Ordinal)) + { + return true; + } + + if (value is Type referenced && referenced == definition) + { + return true; + } + } + + return false; + } + + /// + /// Looks for test classes that no data-store-filtered leg can select. + /// + /// + /// Every E2E and export leg selects positively on the DataStore trait. A class gets + /// that trait one of two ways: the custom framework expands a + /// into one variant per store and traits each, or + /// the class writes the trait out by hand. A class that does neither is not skipped and does + /// not error - it simply matches no leg's filter, so it is never constructed, never run, and + /// never counted, while every leg reports success. Passing a store through + /// [InlineData] looks like it should help and does not: inline arguments are values, + /// not traits, and the filter never sees them. + /// + /// Assembly under analysis. + /// The trait the legs select on. + /// + /// The attribute the framework expands into per-store variants. Passed in because it is + /// declared alongside the tests rather than here. + /// + /// The test classes no leg selects. + public static IReadOnlyList ScanForTestClassesNoStoreFilteredLegSelects(Assembly assembly, string traitName, Type argumentSetAttributeType) + { + EnsureArg.IsNotNull(assembly, nameof(assembly)); + EnsureArg.IsNotNullOrWhiteSpace(traitName, nameof(traitName)); + EnsureArg.IsNotNull(argumentSetAttributeType, nameof(argumentSetAttributeType)); + + return assembly.GetTypes() + .Where(t => t.IsClass && IsTestClass(t)) + .Where(t => !IsExpandedIntoStoreVariants(t, argumentSetAttributeType) && !DeclaresTrait(t, traitName)) + .ToList(); + } + + /// + /// Looks for test methods that no data-store-filtered leg can select, in classes the class + /// level check exempts. + /// + /// + /// A class is exempt from when any + /// one of its methods carries argument sets, because that is enough for the framework to + /// expand that class. It expands only that method, though: a method carrying nothing of its + /// own, in a class declaring nothing of its own, is discovered as an ordinary test case with + /// no DataStore trait, so every leg filters it out. One decorated method is therefore + /// enough to hide all of its siblings from the class level check while they run nowhere and + /// every leg reports success. + /// + /// Assembly under analysis. + /// The trait the legs select on. + /// + /// The attribute the framework expands into per-store variants. + /// + /// + /// The unreachable methods, named as Namespace.Class.Method. Classes no leg selects at + /// all are left to the class level check rather than reported a method at a time. + /// + public static IReadOnlyList ScanForTestMethodsNoStoreFilteredLegSelects(Assembly assembly, string traitName, Type argumentSetAttributeType) + { + EnsureArg.IsNotNull(assembly, nameof(assembly)); + EnsureArg.IsNotNullOrWhiteSpace(traitName, nameof(traitName)); + EnsureArg.IsNotNull(argumentSetAttributeType, nameof(argumentSetAttributeType)); + + return assembly.GetTypes() + .Where(t => t.IsClass && IsTestClass(t)) + .Where(t => !CarriesArgumentSets(t.CustomAttributes, argumentSetAttributeType) && !DeclaresTrait(t.CustomAttributes, traitName)) + .Where(t => t.GetMethods().Any(m => CarriesArgumentSets(m.CustomAttributes, argumentSetAttributeType))) + .SelectMany(t => t.GetMethods() + .Where(IsTestMethod) + .Where(m => !CarriesArgumentSets(m.CustomAttributes, argumentSetAttributeType) && !DeclaresTrait(m.CustomAttributes, traitName)) + .Select(m => $"{t.FullName}.{m.Name}")) + .Distinct(StringComparer.Ordinal) + .ToList(); + } + + private static bool IsExpandedIntoStoreVariants(Type type, Type argumentSetAttributeType) + { + // The attribute is declared on the class or on individual methods, and either is enough + // for the framework to expand that class into per-store variants. + return CarriesArgumentSets(type.CustomAttributes, argumentSetAttributeType) || + type.GetMethods().Any(m => CarriesArgumentSets(m.CustomAttributes, argumentSetAttributeType)); + } + + private static bool CarriesArgumentSets(IEnumerable attributes, Type argumentSetAttributeType) + { + return attributes.Any(a => argumentSetAttributeType.IsAssignableFrom(a.AttributeType)); + } + + private static bool IsTestMethod(MethodInfo method) + { + return method.GetCustomAttributes().Any(a => a is FactAttribute || a is TheoryAttribute); + } + + private static bool DeclaresTrait(Type type, string traitName) + { + return DeclaresTrait(type.CustomAttributes, traitName); + } + + private static bool DeclaresTrait(IEnumerable attributes, string traitName) + { + return attributes + .Where(a => a.AttributeType == typeof(TraitAttribute) && a.ConstructorArguments.Count > 0) + .Any(a => string.Equals(a.ConstructorArguments[0].Value as string, traitName, StringComparison.Ordinal)); + } + private static bool IsTestClass(Type type) { if (!type.IsClass) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs index e54bda668d..d5fb8f562a 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs @@ -27,6 +27,163 @@ public static void EnsureAllTestsHaveOwningTeamTrait(Assembly assembly) AssertTestClasses(assembly, Traits.OwningTeam, types); } + /// + /// Fails when a collection definition carrying traits is shared with other classes, because + /// those classes silently acquire the traits and the CI legs select on them. + /// + /// Assembly under analysis. + public static void EnsureNoSharedCollectionDefinitionCarriesTraits(Assembly assembly) + { + IReadOnlyList<(Type Definition, IReadOnlyList Members)> offenders = + AssemblyChecker.ScanForTraitCarryingSharedCollectionDefinitions(assembly); + + if (offenders.Count == 0) + { + return; + } + + var stringBuilder = new StringBuilder(); + foreach ((Type definition, IReadOnlyList members) in offenders) + { + stringBuilder.AppendLine($"{definition} applies its traits to: {string.Join(", ", members.Select(m => m.ToString()))}"); + } + + Assert.Fail( + $"Assembly '{assembly}' has collection definitions whose traits are applied to other classes. Those classes gain traits nothing at their declaration mentions, " + + $"and a CI leg selecting on traits will then run a different set of tests than it appears to - without failing. Move the collection name onto a separate traitless " + + $"definition class and leave the traits where they were declared.{Environment.NewLine}{stringBuilder}"); + } + + /// + /// Fails when the assembly does not declare the test framework its tests are written for. + /// + /// Assembly under analysis. + /// The test framework the assembly must declare. + /// + /// The custom framework is what expands fixture argument sets into variants, and the variants + /// are what carry the DataStore trait every E2E and export leg selects positively on. + /// Lose the declaration and nothing throws: the classes are discovered as ordinary xUnit + /// tests, carry no data store trait, and are filtered out of every leg. A class that is + /// filtered out never constructs its fixture, so no connection is attempted and nothing + /// errors - the leg runs whatever handful of tests declared their traits by hand, reports + /// success, and the entire suite is missing. This check turns that into a failure. + /// + public static void EnsureTestFrameworkIsDeclared(Assembly assembly, Type expectedFrameworkType) + { + TestFrameworkAttribute[] declared = assembly.GetCustomAttributes().ToArray(); + + if (declared.Length == 1 && declared[0].FrameworkType == expectedFrameworkType) + { + return; + } + + string found = declared.Length == 0 + ? "none" + : string.Join(", ", declared.Select(attribute => attribute.FrameworkType?.FullName ?? "")); + + Assert.Fail( + $"Assembly '{assembly}' must declare exactly one test framework, '{expectedFrameworkType.FullName}', but declares: {found}. " + + $"Without it the fixture argument sets are never expanded, so the tests carry no data store trait, every leg that selects positively on one " + + $"filters them out, and the leg passes with the suite missing rather than failing."); + } + + /// + /// Fails when a test class is invisible to every leg that selects on a data store. + /// + /// Assembly under analysis. + /// The trait the legs select on. + /// The attribute the framework expands into per-store variants. + /// + /// Classes already known to run in no leg, named in full. Each one is a test nobody runs; the + /// list exists so that state is written down and cannot grow without someone saying so. + /// + public static void EnsureEveryTestClassIsSelectedBySomeLeg( + Assembly assembly, + string traitName, + Type argumentSetAttributeType, + IReadOnlyCollection knownUnreachableClasses) + { + IReadOnlyList unreachable = AssemblyChecker.ScanForTestClassesNoStoreFilteredLegSelects(assembly, traitName, argumentSetAttributeType); + + string[] unexpected = unreachable + .Select(type => type.FullName) + .Where(name => !knownUnreachableClasses.Contains(name)) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + string[] staleEntries = knownUnreachableClasses + .Where(name => !unreachable.Any(type => string.Equals(type.FullName, name, StringComparison.Ordinal))) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + if (unexpected.Length > 0) + { + Assert.Fail( + $"Assembly '{assembly}' has test classes that no leg selects, because they neither declare a '{traitName}' trait nor carry the fixture argument sets " + + $"the framework expands into one variant per store. Nothing skips or errors for these: every leg filters them out, runs without them, and passes. " + + $"Give each one argument sets or an explicit '{traitName}' trait, or add it to the known list with a reason.{Environment.NewLine}" + + string.Join(Environment.NewLine, unexpected)); + } + + if (staleEntries.Length > 0) + { + Assert.Fail( + $"Assembly '{assembly}' names classes as unreachable that a leg now selects. Remove them from the known list so it keeps describing what is " + + $"actually unrun.{Environment.NewLine}{string.Join(Environment.NewLine, staleEntries)}"); + } + } + + /// + /// Fails when a test method is invisible to every leg that selects on a data store, in a + /// class the class level check exempts. + /// + /// Assembly under analysis. + /// The trait the legs select on. + /// The attribute the framework expands into per-store variants. + /// + /// Methods already known to run in no leg, named as Namespace.Class.Method. + /// + /// + /// One decorated method exempts its whole class from the class level check, so without this + /// its undecorated siblings are hidden twice over: they carry no trait, so no leg selects + /// them, and the check that would have said so passes. + /// + public static void EnsureEveryTestMethodIsSelectedBySomeLeg( + Assembly assembly, + string traitName, + Type argumentSetAttributeType, + IReadOnlyCollection knownUnreachableMethods) + { + IReadOnlyList unreachable = AssemblyChecker.ScanForTestMethodsNoStoreFilteredLegSelects(assembly, traitName, argumentSetAttributeType); + + string[] unexpected = unreachable + .Where(name => !knownUnreachableMethods.Contains(name)) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + string[] staleEntries = knownUnreachableMethods + .Where(name => !unreachable.Contains(name, StringComparer.Ordinal)) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + if (unexpected.Length > 0) + { + Assert.Fail( + $"Assembly '{assembly}' has test methods that no leg selects. Their class carries neither a '{traitName}' trait nor fixture argument sets, and " + + $"neither do they - but a sibling method does, which is enough to exempt the class from the class-level check. Nothing skips or errors for these: " + + $"every leg filters them out, runs without them, and passes. Give the class argument sets, or give each method its own, or add it to the known list " + + $"with a reason.{Environment.NewLine}" + + string.Join(Environment.NewLine, unexpected)); + } + + if (staleEntries.Length > 0) + { + Assert.Fail( + $"Assembly '{assembly}' names methods as unreachable that a leg now selects. Remove them from the known list so it keeps describing what is " + + $"actually unrun.{Environment.NewLine}{string.Join(Environment.NewLine, staleEntries)}"); + } + } + private static void AssertTestClasses(Assembly currentAssembly, string traitName, IEnumerable types) { if (types == null || !types.Any()) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj index 97395818cc..3d47044590 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj +++ b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj @@ -627,7 +627,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.TaskManagement.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.TaskManagement.UnitTests/AssemblyValidationsTests.cs index 919877d101..28b001a685 100644 --- a/src/Microsoft.Health.TaskManagement.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.TaskManagement.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj b/src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj index 4aa9d4b638..6ce557dc80 100644 --- a/src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj +++ b/src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj index 28462cbe94..8a3473b5cb 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj @@ -32,13 +32,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/CreateAllFhirResourcesTestsFixture.cs b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/CreateAllFhirResourcesTestsFixture.cs index 40b53d06e3..f484c6e960 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/CreateAllFhirResourcesTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/CreateAllFhirResourcesTestsFixture.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -25,7 +26,7 @@ public CreateAllFhirResourcesTestsFixture( public List ResourcesToCleanup { get; } = new List(); - protected override async Task OnDisposedAsync() + protected override async ValueTask OnDisposedAsync() { await base.OnDisposedAsync(); diff --git a/test/Microsoft.Health.Fhir.R4.Tests.Integration/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.R4.Tests.Integration/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.Integration/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.R4.Tests.Integration/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/test/Microsoft.Health.Fhir.R4.Tests.Integration/Microsoft.Health.Fhir.R4.Tests.Integration.csproj b/test/Microsoft.Health.Fhir.R4.Tests.Integration/Microsoft.Health.Fhir.R4.Tests.Integration.csproj index 0296252158..5ad4086fbd 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.Integration/Microsoft.Health.Fhir.R4.Tests.Integration.csproj +++ b/test/Microsoft.Health.Fhir.R4.Tests.Integration/Microsoft.Health.Fhir.R4.Tests.Integration.csproj @@ -10,12 +10,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/test/Microsoft.Health.Fhir.R4B.Tests.E2E/Microsoft.Health.Fhir.R4B.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.R4B.Tests.E2E/Microsoft.Health.Fhir.R4B.Tests.E2E.csproj index 4b85049f96..f07875a543 100644 --- a/test/Microsoft.Health.Fhir.R4B.Tests.E2E/Microsoft.Health.Fhir.R4B.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.R4B.Tests.E2E/Microsoft.Health.Fhir.R4B.Tests.E2E.csproj @@ -20,13 +20,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/test/Microsoft.Health.Fhir.R4B.Tests.Integration/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.R4B.Tests.Integration/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/test/Microsoft.Health.Fhir.R4B.Tests.Integration/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.R4B.Tests.Integration/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/test/Microsoft.Health.Fhir.R4B.Tests.Integration/Microsoft.Health.Fhir.R4B.Tests.Integration.csproj b/test/Microsoft.Health.Fhir.R4B.Tests.Integration/Microsoft.Health.Fhir.R4B.Tests.Integration.csproj index c1c40ac931..cb29974b67 100644 --- a/test/Microsoft.Health.Fhir.R4B.Tests.Integration/Microsoft.Health.Fhir.R4B.Tests.Integration.csproj +++ b/test/Microsoft.Health.Fhir.R4B.Tests.Integration/Microsoft.Health.Fhir.R4B.Tests.Integration.csproj @@ -10,12 +10,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/test/Microsoft.Health.Fhir.R5.Tests.E2E/Microsoft.Health.Fhir.R5.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.R5.Tests.E2E/Microsoft.Health.Fhir.R5.Tests.E2E.csproj index 37b6c8e25d..124773d96e 100644 --- a/test/Microsoft.Health.Fhir.R5.Tests.E2E/Microsoft.Health.Fhir.R5.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.R5.Tests.E2E/Microsoft.Health.Fhir.R5.Tests.E2E.csproj @@ -18,13 +18,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/test/Microsoft.Health.Fhir.R5.Tests.Integration/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.R5.Tests.Integration/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/test/Microsoft.Health.Fhir.R5.Tests.Integration/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.R5.Tests.Integration/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/test/Microsoft.Health.Fhir.R5.Tests.Integration/Microsoft.Health.Fhir.R5.Tests.Integration.csproj b/test/Microsoft.Health.Fhir.R5.Tests.Integration/Microsoft.Health.Fhir.R5.Tests.Integration.csproj index cd5d5b9b3c..7d33ec9902 100644 --- a/test/Microsoft.Health.Fhir.R5.Tests.Integration/Microsoft.Health.Fhir.R5.Tests.Integration.csproj +++ b/test/Microsoft.Health.Fhir.R5.Tests.Integration/Microsoft.Health.Fhir.R5.Tests.Integration.csproj @@ -10,12 +10,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Crucible/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Crucible/AssemblyValidationsTests.cs index 822c474a15..5b227b2141 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Crucible/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Crucible/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs index 742acc5ab9..acfaadbdce 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs @@ -1,21 +1,68 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; +using System.Collections.Generic; using System.Reflection; +using Microsoft.Health.Extensions.Xunit; using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; namespace Microsoft.Health.Fhir.Shared.Tests.E2E.Common { + /// + /// Assembly-wide checks over the E2E tests, from the shared project the E2E assemblies alone + /// compile. + /// + /// + /// This project is imported by the four E2E assemblies and nothing else, and every E2E leg + /// selects positively on a data store, so without the traits below this copy is filtered out of + /// every leg and never runs. It would still compile, and a check added to it would still look + /// like it was covering the E2E assemblies while running nowhere. Naming both stores is safe + /// here precisely because only positively-filtered legs compile it: a leg that selected by + /// excluding a store would drop a class naming both, which is why the copy shared with + /// the integration assemblies deliberately carries neither. + /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.AssemblyValidation)] + [Trait(nameof(DataStore), nameof(DataStore.CosmosDb))] + [Trait(nameof(DataStore), nameof(DataStore.SqlServer))] public sealed class AssemblyValidationsTests { private static Assembly _currentAssembly = Assembly.GetAssembly(typeof(AssemblyValidationsTests)); + private static readonly HashSet KnownUnreachableTestClasses = new HashSet(StringComparer.Ordinal) + { + // Untraited on purpose. This copy is also compiled into the integration assemblies, whose + // legs filter by excluding a store rather than naming one, and an exclusion drops a class + // that names both. Traiting it here would silence it there, so the E2E copy above carries + // the traits instead and this one runs in the integration legs only. + "Microsoft.Health.Fhir.Shared.Tests.AssemblyValidationsTests", + + // Pre-existing, and unrun before this PR as well: the legs used to select by name + // substring, which these matched no better than they match the trait filters. They need a + // container registry and a convert-data configuration that no leg sets up, so traiting + // them would move them from never running to reliably failing. + "Microsoft.Health.Fhir.Tests.E2E.Rest.ContainerRegistryTemplateUploaderTests", + "Microsoft.Health.Fhir.Tests.E2E.Rest.ConvertDataTestModeTests", + + // Takes its data store as an [InlineData] argument, which is a value and not a trait, so + // no filter can see it. Traiting the class would run every row in both legs, including + // the row written for the other store, so this needs per-row traits rather than a + // one-line fix. + "Microsoft.Health.Fhir.Tests.E2E.Rest.Search.SearchParameterInitializationTests", + }; + + /// + /// Methods in a class that only a sibling's argument sets make visible to the class-level + /// check. Empty, and meant to stay so: an entry here is a test nobody runs. + /// + private static readonly HashSet KnownUnreachableTestMethods = new HashSet(StringComparer.Ordinal); + [Fact] public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveCategoryTrait() { @@ -27,5 +74,41 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } + + /// + /// Every leg selects positively on , so a class that carries no such + /// trait runs nowhere while every leg stays green. The classes below are in that state + /// already, from before the legs filtered on traits at all; they are listed so the situation + /// is visible and cannot quietly grow. + /// + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureEveryTestClassIsSelectedBySomeLeg() + { + AssemblyValidationsTester.EnsureEveryTestClassIsSelectedBySomeLeg( + _currentAssembly, + nameof(DataStore), + typeof(FixtureArgumentSetsAttribute), + KnownUnreachableTestClasses); + } + + /// + /// A single decorated method exempts its whole class from the check above, so its + /// undecorated siblings would be selected by no leg and reported by nothing. + /// + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureEveryTestMethodIsSelectedBySomeLeg() + { + AssemblyValidationsTester.EnsureEveryTestMethodIsSelectedBySomeLeg( + _currentAssembly, + nameof(DataStore), + typeof(FixtureArgumentSetsAttribute), + KnownUnreachableTestMethods); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs index 5591b40d4b..3af3a48b34 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs @@ -8,6 +8,6 @@ using Microsoft.Health.Fhir.Tests.E2E.Rest; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] -[assembly: AssemblyFixture(typeof(TestFhirServerFactory))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(TestFhirServerFactory))] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs index 7d5192bc53..074d01ac86 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs @@ -1,17 +1,34 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System.Reflection; +using Microsoft.Health.Extensions.Xunit; using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; namespace Microsoft.Health.Fhir.Tests.E2E { + /// + /// Assembly-wide checks over the E2E tests. + /// + /// + /// Every E2E leg selects positively on a data store - /[(DataStore=CosmosDb)&...] - and + /// a positive match cannot select a test case that carries no DataStore trait at all. Real + /// E2E classes get theirs from HttpIntegrationFixtureArgumentSets, but this class has no + /// server fixture and wants none: it only reads the assembly's metadata. Without the traits below + /// it is filtered out of every leg and never runs anywhere, which would leave these checks - one + /// of which exists to catch the collection-trait propagation that hid twelve E2E cases behind a + /// green run - quietly inert, in the one assembly the incident came from. Both stores are named + /// so the checks run whichever leg is executing. + /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.AssemblyValidation)] + [Trait(nameof(DataStore), nameof(DataStore.CosmosDb))] + [Trait(nameof(DataStore), nameof(DataStore.SqlServer))] public sealed class AssemblyValidationsTests { private static Assembly _currentAssembly = Assembly.GetAssembly(typeof(AssemblyValidationsTests)); @@ -27,5 +44,17 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureTheCustomTestFrameworkIsDeclared() + { + AssemblyValidationsTester.EnsureTestFrameworkIsDeclared(_currentAssembly, typeof(CustomXunitTestFramework)); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems index bff981c26b..39f4f03907 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems @@ -82,6 +82,7 @@ + @@ -157,7 +158,14 @@ + xunit.runner.json PreserveNewest + + Never diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs index 9b93a74a0b..78f43aa6f5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs @@ -26,7 +26,6 @@ using Microsoft.Health.Test.Utilities; using Newtonsoft.Json; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; using static Hl7.Fhir.Model.Encounter; using Task = System.Threading.Tasks.Task; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs index 3ab1816e4f..7eb5c4d694 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs @@ -1,8 +1,9 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -27,7 +28,7 @@ public CompartmentTestFixture(DataStore dataStore, Format format, TestFhirServer public Condition Condition { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Create various resources. Device = await TestFhirClient.CreateAsync(Samples.GetJsonSample("Device-d1")); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs index 50ac5ea977..54d0e1268a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -63,7 +64,7 @@ public IReadOnlyList GetDocumentReferences(string patientId) return new List(); } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await base.OnInitializedAsync(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs index b47bb8e40a..41118b1850 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -36,7 +37,7 @@ public ExpandOperationTestFixture( public IReadOnlyList ValueSets => _valueSets; - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await base.OnInitializedAsync(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/EverythingOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/EverythingOperationTestFixture.cs index 6f71f8ce28..9c88b9e09b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/EverythingOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/EverythingOperationTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -78,7 +78,7 @@ internal async Task UpdatePatient(Patient patientToUpdate) await TestFhirClient.UpdateAsync(patientToUpdate); } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Test case #1 // Create resources that directly referenced by the Patient resource diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index 97b15097f5..a6fe6aefd1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; +using System.Threading.Tasks; using Antlr4.Runtime.Atn; using Hl7.Fhir.Model; using Medino; @@ -77,7 +78,7 @@ internal string ExportTestFilterQueryParameters(params string[] uniqueResourceTy return $"_type={string.Join(',', uniqueResourceTypes)}&_typeFilter={typeFilterPart}&_since={TestDataInsertionTime:O}"; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await SaveTestResourcesToServer(); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index 8edabe954c..34a935b04e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -14,7 +14,6 @@ using Microsoft.Health.Fhir.Tests.E2E.Common; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs index 4a9d51dba8..fa60995d87 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs @@ -15,7 +15,6 @@ using Microsoft.Health.Fhir.Tests.E2E.Common; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using FhirGroup = Hl7.Fhir.Model.Group; using Task = System.Threading.Tasks.Task; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs index db1464980f..ee87fb7afb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs @@ -17,7 +17,7 @@ using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; using Microsoft.Health.Fhir.Tests.E2E.Common; using Newtonsoft.Json; -using Xunit.Abstractions; +using Xunit; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs index caa67fbb10..842e725c60 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs @@ -54,12 +54,12 @@ public HistoryTests(HttpIntegrationTestFixture fixture) protected HttpIntegrationTestFixture Fixture { get; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { _createdResource = await _client.CreateByUpdateAsync(Samples.GetDefaultObservation().ToPoco()); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_createdResource?.Resource != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HttpIntegrationTestFixture{TStartup}.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HttpIntegrationTestFixture{TStartup}.cs index c836d38d28..d275f9d6d8 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HttpIntegrationTestFixture{TStartup}.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HttpIntegrationTestFixture{TStartup}.cs @@ -57,7 +57,7 @@ public HttpIntegrationTestFixture(DataStore dataStore, Format format, TestFhirSe protected internal TestFhirServer TestFhirServer { get; private set; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { Exception lastException = null; @@ -102,7 +102,7 @@ public async Task InitializeAsync() lastException); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await OnDisposedAsync(); } @@ -112,8 +112,8 @@ public string GenerateFullUrl(string relativeUrl) return $"{TestFhirServer.BaseAddress}{relativeUrl}"; } - protected virtual Task OnInitializedAsync() => Task.CompletedTask; + protected virtual ValueTask OnInitializedAsync() => ValueTask.CompletedTask; - protected virtual Task OnDisposedAsync() => Task.CompletedTask; + protected virtual ValueTask OnDisposedAsync() => ValueTask.CompletedTask; } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs index 9ed90a041f..bd442252a5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Task = System.Threading.Tasks.Task; @@ -46,7 +47,7 @@ public ImportBasicSearchTestFixture(DataStore dataStore, Format format, TestFhir public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs index 48059bdbc8..d960d8dec5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -44,7 +45,7 @@ public ImportCompositeSearchTestFixture(DataStore dataStore, Format format, Test public IReadOnlyDictionary DocumentReferences { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Observations = CreateResultDictionary(ObservationTestFileNames); DocumentReferences = CreateResultDictionary(DocumentReferenceTestFiles); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs index 0283a338de..d8d1aab03a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Task = System.Threading.Tasks.Task; @@ -22,7 +23,7 @@ public ImportDateSearchTestFixture(DataStore dataStore, Format format, TestFhirS public IReadOnlyList Observations { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Observations = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportHistorySoftDeleteTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportHistorySoftDeleteTestFixture.cs index 833699ba6e..ec4260c770 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportHistorySoftDeleteTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportHistorySoftDeleteTestFixture.cs @@ -26,7 +26,7 @@ public ImportHistorySoftDeleteTestFixture(DataStore dataStore, Format format, Te public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { TestResources.Add("NewImplicitVersionId", GetNewImplicitVersionIdResources()); TestResources.Add("ImportOverExistingVersionId", GetImportOverExistingVersionIdResources()); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs index 2a6ec0f631..6c505280aa 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Task = System.Threading.Tasks.Task; @@ -22,7 +23,7 @@ public ImportNumberSearchTestFixture(DataStore dataStore, Format format, TestFhi public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { RiskAssessments = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs index 4c9f3ac154..78492a19c0 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Task = System.Threading.Tasks.Task; @@ -22,7 +23,7 @@ public ImportQuantitySearchTestFixture(DataStore dataStore, Format format, TestF public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Observations = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs index a69e3c26a4..2676e5bf11 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Task = System.Threading.Tasks.Task; @@ -22,7 +23,7 @@ public ImportReferenceSearchTestFixture(DataStore dataStore, Format format, Test public IReadOnlyList Patients { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Patients = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs index 880f988519..9a3392f138 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Task = System.Threading.Tasks.Task; @@ -24,7 +25,7 @@ public ImportStringSearchTestFixture(DataStore dataStore, Format format, TestFhi public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Patients = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs index 7560ebf05a..504e583473 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -23,7 +24,7 @@ public ImportTokenSearchTestFixture(DataStore dataStore, Format format, TestFhir public IReadOnlyList Observations { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await TestFhirClient.CreateResourcesAsync(p => { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs index 20b63003ea..2dfced61be 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Task = System.Threading.Tasks.Task; @@ -22,7 +23,7 @@ public ImportUriSearchTestFixture(DataStore dataStore, Format format, TestFhirSe public string FixtureTag { get; set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { FixtureTag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs new file mode 100644 index 0000000000..c1cd350123 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.Tests.Common; +using Xunit; + +namespace Microsoft.Health.Fhir.Tests.E2E.Rest +{ + /// + /// Serialises the test classes that add, reindex, or delete search parameters, which share the + /// server's search parameter registry and cannot run alongside each other. + /// + /// + /// This class deliberately carries no traits. xunit v3 puts a collection's traits on the tests of + /// every class in the collection, so a category declared here would be inherited by every member + /// - including members that only joined to be serialised. CI legs that exclude a category would + /// then skip those members as well, and an exclusion filter reports success for what it did not + /// select, so nothing in the leg's output would say the tests were dropped. Categories therefore + /// belong on the individual test classes, never here. + /// + [CollectionDefinition(Categories.IndexAndReindex, DisableParallelization = true)] + public class IndexAndReindexCollection + { + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs index 49c26076b2..afd36799cb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -22,7 +23,7 @@ public MemberMatchTestFixture(DataStore dataStore, Format format, TestFhirServer _tag = new Coding(string.Empty, Guid.NewGuid().ToString()); } - protected override async System.Threading.Tasks.Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Create various resources. Patient[] patients = await TestFhirClient.CreateResourcesAsync( diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Metric/MetricTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Metric/MetricTests.cs index e01524fe7e..12a90effd1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Metric/MetricTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Metric/MetricTests.cs @@ -38,15 +38,15 @@ public MetricTests(MetricTestFixture fixture) _metricHandler = _fixture?.MetricHandler; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { // Send an empty request to guarantee that there is a bearer token set and the call isn't recorded in the metric handler. await _client.HttpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, string.Empty)); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } [Fact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 2de2cabdae..eca946d53b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -26,17 +26,15 @@ using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; -using Microsoft.VisualStudio.TestPlatform.Utilities; using Newtonsoft.Json; using Xunit; -using Xunit.Abstractions; using static Hl7.Fhir.Model.Bundle; using static Microsoft.EntityFrameworkCore.DbLoggerCategory; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Reindex { - [CollectionDefinition(Categories.IndexAndReindex, DisableParallelization = true)] + [Collection(Categories.IndexAndReindex)] [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.IndexAndReindex)] [Trait(Traits.Category, Categories.ReindexOperation)] @@ -54,7 +52,7 @@ public ReindexTests(HttpIntegrationTestFixture fixture, ITestOutputHelper output _output = output; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { // delete leftover resources from previous test runs // this guarantees that we start from clean state even on previous hard failures when we cannot rely on finally @@ -76,9 +74,9 @@ public async Task InitializeAsync() _output.WriteLine($"ReindexTests.InitializeAsync: Completed. Elapsed={(int)sw.Elapsed.TotalMilliseconds} msec."); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } [Fact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs index ebb31fc848..98f88ff91d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs @@ -1,9 +1,10 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Tests.Common; @@ -44,7 +45,7 @@ public CanonicalSearchTestFixture(DataStore dataStore, Format format, TestFhirSe public string ObservationProfileV2 { get; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Observation resource1 = Samples.GetDefaultObservation().ToPoco(); resource1.Meta = new Meta(); @@ -75,7 +76,7 @@ protected override async Task OnInitializedAsync() GivenObservation3 = (await TestFhirClient.CreateAsync(resource3)).Resource; } - protected override async Task OnDisposedAsync() + protected override async ValueTask OnDisposedAsync() { if (GivenObservation1 != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs index 2405a8fadf..ac6fcfc41c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -457,7 +457,7 @@ public ClassFixture(DataStore dataStore, Format format, TestFhirServerFactory te public Device DeviceSnomedSubject { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CompositeSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CompositeSearchTestFixture.cs index 1b3a4147b3..2a1969aa2a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CompositeSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CompositeSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -46,7 +46,7 @@ public CompositeSearchTestFixture(DataStore dataStore, Format format, TestFhirSe public IReadOnlyDictionary DocumentReferences { get; private set; } - protected async override Task OnInitializedAsync() + protected async override ValueTask OnInitializedAsync() { Observations = await CreateResultDictionary(ObservationTestFileNames); DocumentReferences = await CreateResultDictionary(DocumentReferenceTestFiles); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs index 94024475a7..f149749322 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs @@ -20,12 +20,10 @@ using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search { - [CollectionDefinition(Categories.IndexAndReindex, DisableParallelization = true)] [Collection(Categories.IndexAndReindex)] [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Search)] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs index a7654d4f68..185b52f922 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -23,7 +24,7 @@ public DateSearchTestFixture(DataStore dataStore, Format format, TestFhirServerF public IReadOnlyList Observations { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Creates a unique code for searches Coding = new Coding diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTestFixture.cs index 723a1b33ba..a61e9572fb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -101,7 +101,7 @@ public IncludeSearchTestFixture(DataStore dataStore, Format format, TestFhirServ public Location LocationPartOfSelf { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs index 17e3704da3..e9debb7f4c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Models; @@ -120,7 +121,7 @@ public IncludesOperationTestFixture( public string Tag => _tag; - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await base.OnInitializedAsync(); var bundle = TagResources((Bundle)Samples.GetJsonSample(ResourceFileName).ToPoco()); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs index d60294555f..858cfdaca1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Core.Models; @@ -30,7 +31,7 @@ public NotReferencedSearchTestFixture(DataStore dataStore, Format format, TestFh public Patient[] Patients { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Creates a unique code for searches Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs index f993576a1e..8c10b4e8c7 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -23,7 +24,7 @@ public NumberSearchTestFixture(DataStore dataStore, Format format, TestFhirServe public string Tag { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs index af3747cdc5..a90340595a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -23,7 +24,7 @@ public QuantitySearchTestFixture(DataStore dataStore, Format format, TestFhirSer public string Tag { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Prepare the resources used for number search tests. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs index a305522e67..d4d72d36cc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -23,7 +24,7 @@ public ReferenceSearchTestFixture(DataStore dataStore, Format format, TestFhirSe public string Tag { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Prepare the resources used for string search tests. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/SearchTestsBase.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/SearchTestsBase.cs index 09da1878f6..117a6c64f4 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/SearchTestsBase.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/SearchTestsBase.cs @@ -19,7 +19,6 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.E2E.Common; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; using static Hl7.Fhir.Model.OperationOutcome; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs index c58c0142c3..aa73d2c48f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -25,7 +26,7 @@ public StringSearchTestFixture(DataStore dataStore, Format format, TestFhirServe public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Patients = await TestFhirClient.CreateResourcesAsync( p => SetPatientInfo(p, "Seattle", "Smith", given: "Bea"), diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs index 78aa0a540f..15995f3e6d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -6,6 +6,7 @@ using System; using System.Linq; using System.Net; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; using Microsoft.Health.Core.Extensions; @@ -16,7 +17,6 @@ using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -39,7 +39,7 @@ public TokenOverflowTests(HttpIntegrationTestFixture fixture, ITestOutputHelper // not used by the resources in the test must be returned. The goal is then to test if database select logic correctly returns no data (rows). private delegate string GetOneParameter(T resource, bool valid = true); - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await Task.CompletedTask; } @@ -575,6 +575,6 @@ await ExecuteAndValidateBundle( } } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs index b5702f3cf0..cb43ae1b21 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -23,7 +24,7 @@ public TokenSearchTestFixture(DataStore dataStore, Format format, TestFhirServer public IReadOnlyList Observations { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs index 98e2797631..509d4686e9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs @@ -1,10 +1,11 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -23,7 +24,7 @@ public UriSearchTestFixture(DataStore dataStore, Format format, TestFhirServerFa public string FixtureTag { get; set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Prepare the resources used for URI search tests. FixtureTag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServerFactory.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServerFactory.cs index a50302e23f..5e589dd6fc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServerFactory.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServerFactory.cs @@ -162,12 +162,7 @@ private static string GetEnvironmentUrl(DataStore dataStore) } } - Task IAsyncLifetime.InitializeAsync() => Task.CompletedTask; - - async Task IAsyncLifetime.DisposeAsync() - { - await DisposeAsync(); - } + ValueTask IAsyncLifetime.InitializeAsync() => ValueTask.CompletedTask; public async ValueTask DisposeAsync() { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs index 3db48f33b3..31ef312adc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs @@ -1,9 +1,10 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -18,7 +19,7 @@ public ValidateTestFixture(DataStore dataStore, Format format, TestFhirServerFac { } - protected override async System.Threading.Tasks.Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Use PUT (UpdateAsync) with stable IDs to upsert profile resources. // This prevents duplicate StructureDefinitions from accumulating in persistent test environments. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/xunit.runner.json b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/xunit.runner.json index 253b9806cd..553c9ea743 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/xunit.runner.json +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/xunit.runner.json @@ -1,4 +1,5 @@ { + "$schema": "https://xunit.net/schema/v3/configuration", "parallelizeAssembly": false, "parallelizeTestCollections": false } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs index 92e92b1d6f..d60876dbe5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs @@ -1,9 +1,10 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System.Reflection; +using Microsoft.Health.Extensions.Xunit; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using Xunit; @@ -27,5 +28,17 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureTheCustomTestFrameworkIsDeclared() + { + AssemblyValidationsTester.EnsureTestFrameworkIsDeclared(_currentAssembly, typeof(CustomXunitTestFramework)); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs index a46dd50d2b..a7c05656db 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -24,7 +24,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Features.ChangeFeed { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs index 43cbf06f67..c046c523c8 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs @@ -47,12 +47,12 @@ public SqlServerFhirResourceChangeCaptureFixture() public SchemaInformation SchemaInformation => _sqlFixture.SchemaInformation; - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _storageFixture.InitializeAsync(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_storageFixture != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CosmosDbFhirExportTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CosmosDbFhirExportTestFixture.cs index 0bef6d498b..e7d9732992 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CosmosDbFhirExportTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CosmosDbFhirExportTestFixture.cs @@ -23,7 +23,7 @@ public CosmosDbFhirExportTestFixture() { } - public override async Task InitializeAsync() + public override async ValueTask InitializeAsync() { await base.InitializeAsync(); @@ -31,7 +31,7 @@ public override async Task InitializeAsync() await PrepareData(3333); } - public override async Task DisposeAsync() + public override async ValueTask DisposeAsync() { await CleanupTestResourceDocuments(); await base.DisposeAsync(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CreateExportRequestHandlerTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CreateExportRequestHandlerTests.cs index 1819e12827..20d14aef92 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CreateExportRequestHandlerTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CreateExportRequestHandlerTests.cs @@ -303,14 +303,14 @@ public static IEnumerable ValidateTypeFilters } } - public Task InitializeAsync() + public ValueTask InitializeAsync() { - return _fhirStorageTestHelper.DeleteAllExportJobRecordsAsync(_cancellationToken); + return new ValueTask(_fhirStorageTestHelper.DeleteAllExportJobRecordsAsync(_cancellationToken)); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } [Theory] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs index 0015288465..c0f1c69108 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -25,7 +25,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Operations.Export { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreExportTests.cs index a311bcabad..217fcac33b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreExportTests.cs @@ -44,15 +44,15 @@ public FhirOperationDataStoreExportTests(FhirStorageTestsFixture fixture) _testQueueClient = (TestQueueClient)fixture.Service.GetService(typeof(TestQueueClient)); } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _testHelper.DeleteAllExportJobRecordsAsync(); _testQueueClient?.ClearJobs(); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } /// diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreReindexTests.cs index 342ec7e75a..fc2b71e098 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreReindexTests.cs @@ -43,16 +43,16 @@ public FhirOperationDataStoreReindexTests(FhirStorageTestsFixture fixture) _operationDataStore = fixture.TestSqlServerOperationDataStore ?? fixture.OperationDataStore; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { GetTestQueueClient().ClearJobs(); await AssertNoReindexJobsExist(); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } private async Task AssertNoReindexJobsExist() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs index 0c6b9e8e7c..f882ea3363 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs @@ -51,7 +51,6 @@ using Newtonsoft.Json; using NSubstitute; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.Integration.Features.Operations.Reindex @@ -104,7 +103,7 @@ public ReindexJobTests(FhirStorageTestsFixture fixture, ITestOutputHelper output _output = output; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { _operationsConfig.Value.Returns(new OperationsConfiguration()); _coreFeatureConfig.Value.Returns(new CoreFeatureConfiguration { SearchParameterCacheRefreshIntervalSeconds = 1 }); @@ -182,7 +181,7 @@ public async Task InitializeAsync() StartCacheUpdateTask(_backgroundCts.Token); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { // Clean up resources before finishing test class await DeleteTestResources(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Search/SqlServerSearchServiceIntegrationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Search/SqlServerSearchServiceIntegrationTests.cs index c2934c7735..8fc417f610 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Search/SqlServerSearchServiceIntegrationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Search/SqlServerSearchServiceIntegrationTests.cs @@ -45,12 +45,12 @@ public SqlServerSearchServiceIntegrationTests(FhirStorageTestsFixture fixture) _dataStore = fixture.DataStore; } - public Task InitializeAsync() => Task.CompletedTask; + public ValueTask InitializeAsync() => ValueTask.CompletedTask; - public Task DisposeAsync() + public ValueTask DisposeAsync() { // Clean up is handled by fixture disposal - return Task.CompletedTask; + return ValueTask.CompletedTask; } [Fact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs index 4fb9de35cd..12766750a9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs @@ -49,7 +49,7 @@ public SmartSearchSharedFixture(DataStore dataStore) public FhirStorageTestsFixture Fixture => _fixture; - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { if (ModelInfoProvider.Instance.Version != FhirSpecification.R4 && ModelInfoProvider.Instance.Version != FhirSpecification.R4B) @@ -95,7 +95,7 @@ public async Task InitializeAsync() await UpsertResource(Samples.GetJsonSample("Location-example-hq")); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_fixture != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems index 368f7a10da..ba5f17cfd6 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems @@ -62,4 +62,11 @@ + + + xunit.runner.json + PreserveNewest + Never + + \ No newline at end of file diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 2fe961befd..e962a5cd46 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -106,7 +106,7 @@ public CosmosDbFhirStorageTestsFixture() public Container Container => _container; - public virtual async Task InitializeAsync() + public virtual async ValueTask InitializeAsync() { var fhirStoredProcs = typeof(DataPlaneCollectionSetup).Assembly .GetTypes() @@ -350,7 +350,7 @@ public virtual async Task InitializeAsync() _fhirStorageTestHelper = new CosmosDbFhirStorageTestHelper(_container, queueClient); } - public virtual async Task DisposeAsync() + public virtual async ValueTask DisposeAsync() { if (_container != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/ExpiredResourceCleanupWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/ExpiredResourceCleanupWatchdogTests.cs index 77cb0355fb..9dcb901980 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/ExpiredResourceCleanupWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/ExpiredResourceCleanupWatchdogTests.cs @@ -21,7 +21,6 @@ using Microsoft.Health.Test.Utilities; using Newtonsoft.Json; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs index f35083cbf6..74aaafdb4b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using System.Linq; using System.Net.Http; using System.Threading; +using System.Threading.Tasks; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; @@ -93,8 +94,10 @@ internal FhirStorageTestsFixture(IServiceProvider fixture) { _fixture = fixture; - // This step has to be done in the constructor because it uses an AsyncLocal and the tests run with the same - // execution context as the fixture constructor, but not the same as InitializeAsync(). + // ResourceIdProvider holds its factory in an ordinary field, so where it is built does + // not affect what the tests see. The comment that used to be here claimed it had to be + // built in the constructor to keep an AsyncLocal alive; that flow no longer exists under + // xunit.v3 (see FixtureArgumentSetClassRunner) and this type never depended on it. _resourceIdProvider = new ResourceIdProvider(); _dataResourceFilter = new DataResourceFilter(MissingDataFilterCriteria.Default); @@ -204,15 +207,17 @@ public void Dispose() (_fixture as IDisposable)?.Dispose(); } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { if (_fixture is IAsyncLifetime asyncLifetime) { await asyncLifetime.InitializeAsync(); } - // Initialize FhirRequestContext to ensure pending status updates are captured - // This needs to be here (like ResourceIdProvider) because it uses AsyncLocal + // Initialize FhirRequestContext to ensure pending status updates are captured. + // FhirRequestContextAccessor stores the context in an AsyncLocal, so this assignment is + // only visible to callers that share this execution context; tests that need a request + // context of their own set it themselves. FhirRequestContextAccessor.RequestContext = new DefaultFhirRequestContext { BaseUri = new Uri("http://localhost/"), @@ -403,7 +408,7 @@ public async Task InitializeAsync() Mediator = new Mediator(services); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_fixture is IAsyncLifetime asyncLifetime) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryPlanReuseCheckerTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryPlanReuseCheckerTests.cs index e823969e27..e54f86d675 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryPlanReuseCheckerTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryPlanReuseCheckerTests.cs @@ -21,7 +21,6 @@ using Microsoft.Health.Test.Utilities; using Microsoft.SqlServer.Management.Smo; using Xunit; -using Xunit.Abstractions; using SearchParamType = Microsoft.Health.Fhir.ValueSets.SearchParamType; using SortOrder = Microsoft.Health.Fhir.Core.Features.Search.SortOrder; using Task = System.Threading.Tasks.Task; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueueClientTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueueClientTests.cs index 23ed758a5b..18362a4117 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueueClientTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueueClientTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -15,7 +15,6 @@ using Microsoft.Health.JobManagement; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using JobConflictException = Microsoft.Health.JobManagement.JobConflictException; using JobStatus = Microsoft.Health.JobManagement.JobStatus; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlComplexQueryTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlComplexQueryTests.cs index f1c3a3ee36..3cad87594d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlComplexQueryTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlComplexQueryTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -25,7 +25,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute.Core; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; using static Antlr4.Runtime.Atn.SemanticContext; using Task = System.Threading.Tasks.Task; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs index 34368ad05b..5a51a6214e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -18,7 +18,6 @@ using Microsoft.Health.SqlServer; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs index 1afcdeabf2..a85b5f7935 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -21,7 +21,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsTests.cs index 7823199d25..d41d2f7234 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -20,7 +20,6 @@ using Microsoft.VisualStudio.TestPlatform.Utilities; using NSubstitute.Core; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs index 5f7e486557..6309d4e8b8 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Numerics; using System.Threading; +using System.Threading.Tasks; using Azure.Identity; using Medino; using Microsoft.AspNetCore.Components.Forms; @@ -143,7 +144,7 @@ internal static string GetDatabaseName(string test = null) return $"{ModelInfoProvider.Version}{(test == null ? string.Empty : $"_{test}")}_{DateTimeOffset.UtcNow.ToString("s").Replace("-", string.Empty).Replace(":", string.Empty)}_{Guid.NewGuid().ToString().Replace("-", string.Empty)}"; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { _mediator = Substitute.For(); @@ -337,7 +338,7 @@ public async Task InitializeAsync() await _searchParameterStatusManager.EnsureInitializedAsync(CancellationToken.None); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await _testHelper.DeleteDatabase(_databaseName, CancellationToken.None); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerImporterIntegrationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerImporterIntegrationTests.cs index 0ad744658c..f55fba545a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerImporterIntegrationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerImporterIntegrationTests.cs @@ -21,7 +21,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs index d2b0807da0..e0ab029b0c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs @@ -41,7 +41,6 @@ using Microsoft.SqlServer.TransactSql.ScriptDom; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterInitializationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterInitializationTests.cs index 7a091a303d..55ca1b9de6 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterInitializationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterInitializationTests.cs @@ -16,7 +16,6 @@ using Microsoft.Health.SqlServer.Features.Client; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSetMergeTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSetMergeTests.cs index f059002c18..162105f45c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSetMergeTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSetMergeTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -20,7 +20,6 @@ using Microsoft.Health.SqlServer.Features.Client; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs index 0a89be46e8..7e734336c2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs @@ -37,7 +37,6 @@ using Microsoft.SqlServer.Dac.Model; using NSubstitute; using Xunit; -using Xunit.Abstractions; #pragma warning disable SA1116 // Split parameters should start on line after declaration diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json new file mode 100644 index 0000000000..bb9a4a99ec --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://xunit.net/schema/v3/configuration", + "parallelizeAssembly": true, + "parallelizeTestCollections": true +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Smart/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Smart/AssemblyValidationsTests.cs index 0e124cf958..9aae72811b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Smart/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Smart/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Smart/SmartProxy/SmartProxyTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Smart/SmartProxy/SmartProxyTestFixture.cs index c4a73f550b..6816578a63 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Smart/SmartProxy/SmartProxyTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Smart/SmartProxy/SmartProxyTestFixture.cs @@ -38,7 +38,7 @@ public SmartProxyTestFixture(TestFhirServerFactory testFhirServerFactory) public TestFhirClient TestFhirClient { get; private set; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { string environmentUrl = Environment.GetEnvironmentVariable($"TestEnvironmentUrl{Constants.TestEnvironmentVariableVersionSuffix}"); @@ -69,10 +69,10 @@ public async Task InitializeAsync() } } - public Task DisposeAsync() + public ValueTask DisposeAsync() { Dispose(); - return Task.CompletedTask; + return ValueTask.CompletedTask; } public void Dispose() diff --git a/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj index 308eb4b04f..04d6adcfaa 100644 --- a/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj @@ -25,13 +25,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj b/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj index e47fb73cf2..379fb6a92e 100644 --- a/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj +++ b/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj @@ -10,12 +10,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/xunit.runner.json b/xunit.runner.json index 0ff247f3c4..d113d1f854 100644 --- a/xunit.runner.json +++ b/xunit.runner.json @@ -1,4 +1,4 @@ { - "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "$schema": "https://xunit.net/schema/v3/configuration", "parallelizeAssembly": true -} \ No newline at end of file +}