Migrate the test suite from xUnit v2 to xUnit v3 (2/3)

CustomAnalysisRules.Test.ruleset

@@ -129,4 +129,11 @@
     <Rule Id="CA2200" Action="None" />
     <Rule Id="CA2234" Action="None" />
   </Rules>
+  <Rules AnalyzerId="xUnit.Analyzers" RuleNamespace="xUnit.Analyzers">
+    <!-- xUnit1051 (pass TestContext.Current.CancellationToken) has ~7,400 call sites across the
+         test suite. Suppressed to keep the xUnit v2 -> v3 migration reviewable; adopting it is
+         tracked as follow-up work. xUnit1031 is deliberately NOT suppressed: the handful of
+         blocking-task call sites use targeted #pragma disables instead. -->
+    <Rule Id="xUnit1051" Action="None" />
+  </Rules>
 </RuleSet>

Directory.Build.props

@@ -29,14 +29,16 @@
   <Choose>
     <When Condition="$(MSBuildProjectName.Contains('UnitTest'))">
       <ItemGroup>
-        <Content Include="$(MSBuildThisFileDirectory)/xunit.runner.json">
+        <None Include="$(MSBuildThisFileDirectory)/xunit.runner.json">
           <Link>xunit.runner.json</Link>
           <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-        </Content>
+          <CopyToPublishDirectory>Never</CopyToPublishDirectory>
+        </None>
       </ItemGroup>
     </When>
   </Choose>
 
+  <!-- Test project configuration for xUnit 3 and Microsoft Testing Platform -->
   <Choose>
     <When Condition="$(MSBuildProjectName.Contains('Test')) AND '$(IsTestProject)' != 'false'">
       <PropertyGroup>
@@ -45,13 +47,15 @@
         <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
         <TestingPlatformCaptureOutput>false</TestingPlatformCaptureOutput>
         <OutputType>Exe</OutputType>
+        <UseMicrosoftTestingPlatformRunner>true</UseMicrosoftTestingPlatformRunner>
+        <NoWarn>$(NoWarn);xUnit1051</NoWarn>
       </PropertyGroup>
       <ItemGroup>
-        <PackageReference Include="YTest.MTP.XUnit2" />
         <PackageReference Include="Microsoft.Testing.Extensions.Retry" />
         <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" />
         <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
-        <PackageReference Remove="xunit.runner.visualstudio" />
+        <PackageReference Include="Microsoft.Testing.Extensions.Telemetry" />
+        <PackageReference Include="Microsoft.Testing.Platform.MSBuild" />
       </ItemGroup>
     </When>
     <When Condition="$(MSBuildProjectName.Contains('Test')) AND '$(IsTestProject)' == 'false'">

Directory.Packages.props

@@ -46,8 +46,10 @@
     <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.4.0" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version="1.4.0" />
     <PackageVersion Include="Azure.Storage.Blobs" Version="12.21.2" />
-    <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.1.0" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.3.1" />
     <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.0.2" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.Telemetry" Version="2.0.2" />
+    <PackageVersion Include="Microsoft.Testing.Platform.MSBuild" Version="2.0.2" />
     <PackageVersion Include="DotNet.ReproducibleBuilds" Version="1.2.25" />
     <PackageVersion Include="Ensure.That" Version="10.1.0" />
     <PackageVersion Include="FluentValidation" Version="11.11.0" />
@@ -145,13 +147,13 @@
     <PackageVersion Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
     <PackageVersion Include="System.Private.ServiceModel" Version="4.10.3" />
-    <PackageVersion Include="xunit.runner.visualstudio" Version="3.0.2" />
-    <PackageVersion Include="YTest.MTP.XUnit2" Version="1.0.3" />
+    <PackageVersion Include="xunit.v3" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.assert" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.common" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.extensibility.core" Version="3.2.2" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
     <PackageVersion Include="Microsoft.Testing.Extensions.Retry" Version="2.0.2" />
-    <PackageVersion Include="xunit.extensibility.core" Version="2.9.3" />
-    <PackageVersion Include="Xunit.SkippableFact" Version="1.5.23" />
-    <PackageVersion Include="xunit" Version="2.9.3" />
-    <PackageVersion Include="xunit.assert" Version="2.9.3" />
     <PackageVersion Include="System.Drawing.Common" Version="8.0.8" />
     <PackageVersion Include="Moq" Version="4.20.69" />
     <PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />

THIRDPARTYNOTICES.md

@@ -969,58 +969,3 @@ This file is based on or incorporates material from the projects listed below (T
   > WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   > See the License for the specific language governing permissions and
   > limitations under the License.
-
-## YTest.MTP.XUnit2 1.0.3
-* Component Source:   https://github.com/Youssef1313/YTest.MTP.XUnit2
-* Component Copyright and License:
-  > The MIT License (MIT)
-  > 
-  > Copyright (c) Youssef Victor Gerges Fahmy
-  > 
-  > All rights reserved.
-  > 
-  > Permission is hereby granted, free of charge, to any person obtaining a copy
-  > of this software and associated documentation files (the "Software"), to deal
-  > in the Software without restriction, including without limitation the rights
-  > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-  > copies of the Software, and to permit persons to whom the Software is
-  > furnished to do so, subject to the following conditions:
-  > 
-  > The above copyright notice and this permission notice shall be included in all
-  > copies or substantial portions of the Software.
-  > 
-  > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-  > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-  > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-  > SOFTWARE.
-
-## Xunit.SkippableFact 1.4.13
-* Component Source:   https://github.com/AArnott/Xunit.SkippableFact
-* Component Copyright and License:  
- > This software is licensed under the Microsoft Public License
- > http://opensource.org/licenses/ms-pl
- > 
- > This license governs use of the accompanying software. If you use the software, you
- > accept this license. If you do not accept the license, do not use the software.
- > 
- > 1. Definitions
- > The terms "reproduce," "reproduction," "derivative works," and "distribution" have the
- > same meaning here as under U.S. copyright law.
- > A "contribution" is the original software, or any additions or changes to the software.
- > A "contributor" is any person that distributes its contribution under this license.
- > "Licensed patents" are a contributor's patent claims that read directly on its contribution.
- > 
- > 2. Grant of Rights
- > (A) Copyright Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free copyright license to reproduce its contribution, prepare derivative works of its contribution, and distribute its contribution or any derivative works that you create.
- > (B) Patent Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free license under its licensed patents to make, have made, use, sell, offer for sale, import, and/or otherwise dispose of its contribution in the software or derivative works of the contribution in the software.
- > 
- > 3. Conditions and Limitations
- > (A) No Trademark License- This license does not grant you rights to use any contributors' name, logo, or trademarks.
- > (B) If you bring a patent claim against any contributor over patents that you claim are infringed by the software, your patent license from such contributor to the software ends automatically.
- > (C) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software.
- > (D) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license.
- > (E) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement.
-

build/jobs/build.yml

@@ -30,36 +30,96 @@ steps:
       workingDirectory: $(System.DefaultWorkingDirectory)
 
 - ${{ if eq(parameters.unitTest, 'true') }}:
+  - task: PowerShell@2
+    displayName: 'Assert the unit test projects were found'
+    inputs:
+      pwsh: true
+      filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1'
+      arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -ManifestPath "$(Build.SourcesDirectory)/build/jobs/scripts/UnitTestProjects.txt"'
   - task: DotNetCoreCLI@2
     displayName: 'dotnet test'
     inputs:
       command: test
       projects: '**/*UnitTests/*.csproj'
-      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx'
+      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --minimum-expected-tests 1'
+      # Disable the task's built-in result publishing so the PublishTestResults@2 task below,
+      # which controls the glob and how failures are treated, is the sole publisher.
+      publishTestResults: false
+    env:
+      platformOptions__resultDirectory: '$(Agent.TempDirectory)/unit-test-results'
+  - task: PowerShell@2
+    displayName: 'Reconcile retried unit test failures'
+    inputs:
+      pwsh: true
+      filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1'
+      arguments: '-ResultsDirectory "$(Agent.TempDirectory)/unit-test-results"'
+    condition: succeeded()
+  - task: PublishTestResults@2
+    displayName: 'Publish unit test results'
+    inputs:
+      testResultsFormat: 'VSTest'
+      testResultsFiles: '$(Agent.TempDirectory)/unit-test-results/**/*.trx'
+      mergeTestResults: true
       testRunTitle: 'Unit Tests'
+      # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per
+      # attempt under Retries/<id>/<n>/, all of which this glob matches. Attempt 1 therefore
+      # keeps a Failed record for a test that later passed, and mergeTestResults does not
+      # resolve the two, so failing the task on published results would cancel out every
+      # retry. The step that runs the tests fails on the runner's exit code, and the step that
+      # follows it reconciles the attempts against each other, because that exit code counts a
+      # test that failed and then skipped as recovered. The attempts are published here for
+      # visibility only.
+      failTaskOnFailedTests: false
+      # A missing TRX means the run reported nothing; without this the publish step
+      # succeeds silently and there is nothing to notice.
+      failTaskOnMissingResultsFile: true
+    condition: succeededOrFailed()
 
 - ${{ if eq(parameters.codeCoverage, 'true') }}:
+  - task: PowerShell@2
+    displayName: 'Assert the unit test projects were found'
+    inputs:
+      pwsh: true
+      filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1'
+      arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -ManifestPath "$(Build.SourcesDirectory)/build/jobs/scripts/UnitTestProjects.txt"'
   - task: DotNetCoreCLI@2
     displayName: 'dotnet test with coverage'
     inputs:
       command: test
       projects: '**/*UnitTests/*.csproj'
-      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx'
-      testRunTitle: 'Unit Tests'
-      # Disable the task's built-in (non-retry-aware) result publishing so the
-      # explicit retry-aware PublishTestResults@2 task below is the sole publisher.
+      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --minimum-expected-tests 1 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"'
+      # Disable the task's built-in result publishing so the PublishTestResults@2 task below,
+      # which controls the glob and how failures are treated, is the sole publisher.
       publishTestResults: false
     env:
       platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage'
+  - task: PowerShell@2
+    displayName: 'Reconcile retried unit test failures'
+    inputs:
+      pwsh: true
+      filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1'
+      arguments: '-ResultsDirectory "$(Agent.TempDirectory)/coverage"'
+    condition: succeeded()
   - task: PublishTestResults@2
-    displayName: 'Publish unit test results'
-    condition: succeededOrFailed()
+    displayName: 'Publish unit test with coverage results'
     inputs:
       testResultsFormat: 'VSTest'
       testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx'
       mergeTestResults: true
       testRunTitle: 'Unit Tests'
-      failTaskOnFailedTests: true
+      # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per
+      # attempt under Retries/<id>/<n>/, all of which this glob matches. Attempt 1 therefore
+      # keeps a Failed record for a test that later passed, and mergeTestResults does not
+      # resolve the two, so failing the task on published results would cancel out every
+      # retry. The step that runs the tests fails on the runner's exit code, and the step that
+      # follows it reconciles the attempts against each other, because that exit code counts a
+      # test that failed and then skipped as recovered. The attempts are published here for
+      # visibility only.
+      failTaskOnFailedTests: false
+      # A missing TRX means the run reported nothing; without this the publish step
+      # succeeds silently and there is nothing to notice.
+      failTaskOnMissingResultsFile: true
+    condition: succeededOrFailed()
   - task: reportgenerator@5
     displayName: 'aggregate unit test coverage'
     condition: succeededOrFailed()