From 3aa3239a7ba9895795dfd44ed582cf69d2ceb602 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:26:53 -0700 Subject: [PATCH 001/105] Update Directory.Packages.props for xUnit v3 - Add xunit.v3, xunit.v3.mtp-v2, xunit.v3.assert, xunit.v3.common, xunit.v3.extensibility.core (3.2.2) - Add Microsoft.Testing.Extensions.Telemetry, Microsoft.Testing.Platform.MSBuild (2.0.2) - Bump Microsoft.Testing.Extensions.CodeCoverage 18.1.0 -> 18.3.1 - Bump xunit.runner.visualstudio 3.0.2 -> 3.1.5 - Remove YTest.MTP.XUnit2, xunit, xunit.assert, xunit.extensibility.core, Xunit.SkippableFact Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Packages.props | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index bfe0e8bc25..e1201fc290 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -39,8 +39,10 @@ - + + + @@ -132,13 +134,13 @@ - - + + + + + + - - - - From 0d73fd68aec4fe8d48dd230473153443344ffb74 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:27:30 -0700 Subject: [PATCH 002/105] Update Directory.Build.props for xUnit v3 + MTP - Enable UseMicrosoftTestingPlatformRunner - Switch xunit.runner.json from Content to None + CopyToPublishDirectory=Never - Replace YTest.MTP.XUnit2 reference with MTP extensions (Telemetry, MSBuild) - Drop xunit.runner.visualstudio remove directive (now used as normal ref) - Suppress xUnit1031, xUnit1051, SA1208, SA1209, SA1518 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Build.props | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 8e1b2f2f4b..f7ce2bbaec 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -29,14 +29,16 @@ - + xunit.runner.json PreserveNewest - + Never + + @@ -45,13 +47,15 @@ true false Exe + true + $(NoWarn);xUnit1031;xUnit1051;SA1208;SA1209;SA1518 - - + + From 25e30d3451cab0c3638cde02a0d20d928688905f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:27:58 -0700 Subject: [PATCH 003/105] Update ruleset, notices, and xunit.runner.json for xUnit v3 - CustomAnalysisRules.Test.ruleset: suppress xUnit1031, xUnit1051 and StyleCop SA1208/SA1209 - THIRDPARTYNOTICES.md: remove YTest.MTP.XUnit2 and Xunit.SkippableFact entries - xunit.runner.json: update schema URL to v3 configuration schema Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CustomAnalysisRules.Test.ruleset | 8 +++++ THIRDPARTYNOTICES.md | 55 -------------------------------- xunit.runner.json | 4 +-- 3 files changed, 10 insertions(+), 57 deletions(-) diff --git a/CustomAnalysisRules.Test.ruleset b/CustomAnalysisRules.Test.ruleset index 83956bf183..90f8f7625b 100644 --- a/CustomAnalysisRules.Test.ruleset +++ b/CustomAnalysisRules.Test.ruleset @@ -129,4 +129,12 @@ + + + + + + + + diff --git a/THIRDPARTYNOTICES.md b/THIRDPARTYNOTICES.md index f514d9241b..028de23946 100644 --- a/THIRDPARTYNOTICES.md +++ b/THIRDPARTYNOTICES.md @@ -969,58 +969,3 @@ This file is based on or incorporates material from the projects listed below (T > WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > See the License for the specific language governing permissions and > limitations under the License. - -## YTest.MTP.XUnit2 1.0.3 -* Component Source: https://github.com/Youssef1313/YTest.MTP.XUnit2 -* Component Copyright and License: - > The MIT License (MIT) - > - > Copyright (c) Youssef Victor Gerges Fahmy - > - > All rights reserved. - > - > Permission is hereby granted, free of charge, to any person obtaining a copy - > of this software and associated documentation files (the "Software"), to deal - > in the Software without restriction, including without limitation the rights - > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - > copies of the Software, and to permit persons to whom the Software is - > furnished to do so, subject to the following conditions: - > - > The above copyright notice and this permission notice shall be included in all - > copies or substantial portions of the Software. - > - > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - > SOFTWARE. - -## Xunit.SkippableFact 1.4.13 -* Component Source: https://github.com/AArnott/Xunit.SkippableFact -* Component Copyright and License: - > This software is licensed under the Microsoft Public License - > http://opensource.org/licenses/ms-pl - > - > This license governs use of the accompanying software. If you use the software, you - > accept this license. If you do not accept the license, do not use the software. - > - > 1. Definitions - > The terms "reproduce," "reproduction," "derivative works," and "distribution" have the - > same meaning here as under U.S. copyright law. - > A "contribution" is the original software, or any additions or changes to the software. - > A "contributor" is any person that distributes its contribution under this license. - > "Licensed patents" are a contributor's patent claims that read directly on its contribution. - > - > 2. Grant of Rights - > (A) Copyright Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free copyright license to reproduce its contribution, prepare derivative works of its contribution, and distribute its contribution or any derivative works that you create. - > (B) Patent Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free license under its licensed patents to make, have made, use, sell, offer for sale, import, and/or otherwise dispose of its contribution in the software or derivative works of the contribution in the software. - > - > 3. Conditions and Limitations - > (A) No Trademark License- This license does not grant you rights to use any contributors' name, logo, or trademarks. - > (B) If you bring a patent claim against any contributor over patents that you claim are infringed by the software, your patent license from such contributor to the software ends automatically. - > (C) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software. - > (D) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license. - > (E) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement. - diff --git a/xunit.runner.json b/xunit.runner.json index 0ff247f3c4..d113d1f854 100644 --- a/xunit.runner.json +++ b/xunit.runner.json @@ -1,4 +1,4 @@ { - "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "$schema": "https://xunit.net/schema/v3/configuration", "parallelizeAssembly": true -} \ No newline at end of file +} From b856c92de89825fbae2c6cd7133d493c5cc43ca6 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:28:30 -0700 Subject: [PATCH 004/105] Microsoft.Health.Extensions.Xunit: swap xunit -> xunit.v3.* Replace xunit package with xunit.v3.mtp-v2, xunit.v3.common, and xunit.v3.extensibility.core. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Health.Extensions.Xunit.csproj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj b/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj index f493041c0b..8769cf00d2 100644 --- a/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj +++ b/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj @@ -16,7 +16,9 @@ - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive From 61da2c3f9922d9e7937077961c577f36fd03ea00 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:29:50 -0700 Subject: [PATCH 005/105] Rewrite Custom* xunit test framework types for xUnit v3 - CustomXunitTestFramework now inherits the v3 XunitTestFramework (configFileName ctor), overrides CreateDiscoverer/CreateExecutor taking Assembly. - CustomXunitTestFrameworkDiscoverer uses v3 XunitTestAssembly + IXunitTestClass + ValueTask-based FindTestsForType/FindTestsForMethod with Func> callback; integrates new FixtureArgumentSet* types. - CustomXunitTestFrameworkExecutor is restructured around RunTestCases + custom XunitTestAssemblyRunner/CollectionRunner/ClassRunner/TestRunner types that propagate FixtureArgumentSet variants, and uses reflection to seed the v3 XunitTestAssembly AssemblyFixtureTypes lazy. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFramework.cs | 32 +- .../CustomXunitTestFrameworkDiscoverer.cs | 53 +-- .../CustomXunitTestFrameworkExecutor.cs | 373 ++++++++++-------- 3 files changed, 258 insertions(+), 200 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs index 4c60b395a0..c742fc013f 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs @@ -4,46 +4,32 @@ // ------------------------------------------------------------------------------------------------- using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { /// /// An XunitTestFramework implementation that allows parameterizing class fixtures with combinations of argument values. /// To use, decorate the test assembly with - /// [assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] - /// Also allows an assembly to declare one or more , which are created before any tests + /// [assembly: TestFramework(typeof(CustomXunitTestFramework))] + /// Also allows an assembly to declare one or more , which are created before any tests /// are executed and disposed at the end of the test run. /// public sealed class CustomXunitTestFramework : XunitTestFramework { - /// - /// This type's assembly name. - /// - public const string AssemblyName = nameof(Microsoft) + "." + - nameof(Microsoft.Health) + "." + - nameof(Microsoft.Health.Extensions) + "." + - nameof(Microsoft.Health.Extensions.Xunit); - - /// - /// The full name of this type. Intended to be used as an attribute argument. - /// - public const string TypeName = AssemblyName + "." + nameof(CustomXunitTestFramework); - - public CustomXunitTestFramework(IMessageSink messageSink) - : base(messageSink) + public CustomXunitTestFramework() + : base(configFileName: null) { } - protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) + protected override ITestFrameworkDiscoverer CreateDiscoverer(Assembly assembly) { - return new CustomXunitTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); + return new CustomXunitTestFrameworkDiscoverer(assembly); } - protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + protected override ITestFrameworkExecutor CreateExecutor(Assembly assembly) { - return new CustomXunitTestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink); + return new CustomXunitTestFrameworkExecutor(assembly); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 3d5dadde68..091e5fef50 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -6,51 +6,55 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; +using System.Threading.Tasks; using EnsureThat; -using Xunit.Abstractions; +using Xunit; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { /// /// An implementation of that supports discovering tests with parameterized fixtures. /// - internal sealed class CustomXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer, ITestFrameworkDiscoverer + internal sealed class CustomXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer { - public CustomXunitTestFrameworkDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, IMessageSink diagnosticMessageSink, IXunitTestCollectionFactory collectionFactory = null) - : base(assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory) + public CustomXunitTestFrameworkDiscoverer(Assembly assembly, IXunitTestCollectionFactory collectionFactory = null) + : base(new XunitTestAssembly(assembly, configFileName: null, assembly.GetName().Version, UniqueIDGenerator.ForAssembly(assembly.Location, null)), collectionFactory) { } - protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) + protected override async ValueTask FindTestsForType(IXunitTestClass testClass, ITestFrameworkDiscoveryOptions discoveryOptions, Func> callback) { EnsureArg.IsNotNull(testClass, nameof(testClass)); - EnsureArg.IsNotNull(messageBus, nameof(messageBus)); + EnsureArg.IsNotNull(callback, nameof(callback)); EnsureArg.IsNotNull(discoveryOptions, nameof(discoveryOptions)); - var attributeInfo = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute)).SingleOrDefault(); + var attribute = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; - if (attributeInfo == null) + if (attribute == null) { - return base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions); + return await base.FindTestsForType(testClass, discoveryOptions, callback); } // get the class-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) - SingleFlag[][] classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attributeInfo); + SingleFlag[][] classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attribute); // convert these to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) SingleFlag[][] classLevelClosedParameterSets = CartesianProduct(classLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); - foreach (var method in testClass.Class.GetMethods(true)) + foreach (var method in testClass.Methods) { - IAttributeInfo fixtureParameterAttributeInfo = method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute)).SingleOrDefault(); + var fixtureParameterAttribute = method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; + var effectiveAttribute = fixtureParameterAttribute ?? attribute; SingleFlag[][] closedSets = classLevelClosedParameterSets; - if (fixtureParameterAttributeInfo != null) + if (fixtureParameterAttribute != null) { // get the method-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) - SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttributeInfo); + SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); bool hasOverride = false; for (int i = 0; i < methodLevelOpenParameterSets.Length; i++) @@ -75,10 +79,17 @@ protected override bool FindTestsForType(ITestClass testClass, bool includeSourc foreach (SingleFlag[] closedVariant in closedSets) { - var closedVariantTestClass = new TestClassWithFixtureArguments(testClass.TestCollection, testClass.Class, closedVariant); - var closedVariantTestMethod = new TestMethod(closedVariantTestClass, method); + var testClassName = effectiveAttribute.CollectionBehavior == FixtureArgumentSetCollectionBehavior.PerClass + ? testClass.Class.FullName + : null; + var closedVariantTestCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection.TestAssembly, closedVariant, testClassName); + var closedVariantTestClass = new FixtureArgumentSetTestClass(testClass.Class, closedVariantTestCollection, closedVariant, UniqueIDGenerator.ForTestClass(closedVariantTestCollection.UniqueID, testClass.Class.FullName)); + closedVariantTestClass.ApplyFixtureArguments(closedVariant); + var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); + + closedVariantTestMethod.UpdateArgumentsFromMethod(); - if (!FindTestsForMethod(closedVariantTestMethod, includeSourceInformation, messageBus, discoveryOptions)) + if (!await FindTestsForMethod(closedVariantTestMethod, discoveryOptions, callback)) { return false; } @@ -88,7 +99,7 @@ protected override bool FindTestsForType(ITestClass testClass, bool includeSourc return true; } - private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(IAttributeInfo attributeInfo) + private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(FixtureArgumentSetsAttribute attribute) { bool IsPowerOfTwo(long x) { @@ -117,9 +128,7 @@ IEnumerable GetSingleValuedFlags(Enum e) } } - return attributeInfo - .GetConstructorArguments() - .Cast() + return attribute.GetArgumentSets() .Select(e => GetSingleValuedFlags(e).ToArray()) .ToArray(); } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index a00c512da7..060ef2bca9 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -5,16 +5,13 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Reflection; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using EnsureThat; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { @@ -23,228 +20,294 @@ namespace Microsoft.Health.Extensions.Xunit /// internal sealed class CustomXunitTestFrameworkExecutor : XunitTestFrameworkExecutor { - public CustomXunitTestFrameworkExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink) - : base(assemblyName, sourceInformationProvider, diagnosticMessageSink) + public CustomXunitTestFrameworkExecutor(Assembly assembly) + : base(new FixtureArgumentSetTestAssembly(assembly)) { - AssemblyInfo = new CustomAssemblyInfo(AssemblyInfo); } - protected override void RunTestCases(IEnumerable testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + public override async ValueTask RunTestCases(IReadOnlyCollection testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions, CancellationToken cancellationToken) { - using (var assemblyRunner = new AssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions)) - { - assemblyRunner.RunAsync().GetAwaiter().GetResult(); - } + var runner = new FixtureArgumentSetAssemblyRunner(); + await runner.Run((IXunitTestAssembly)TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); } - private sealed class AssemblyRunner : XunitTestAssemblyRunner + private sealed class FixtureArgumentSetTestAssembly : XunitTestAssembly { - private readonly Dictionary _assemblyFixtureMappings = new Dictionary(); - private ExecutionContext _context; + private readonly Assembly _assembly; + private IReadOnlyCollection _assemblyFixtureTypes; + private static readonly FieldInfo AssemblyFixtureTypesField = typeof(XunitTestAssembly).GetField("assemblyFixtureTypes", BindingFlags.Instance | BindingFlags.NonPublic); - public AssemblyRunner(ITestAssembly testAssembly, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) - : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) + public FixtureArgumentSetTestAssembly(Assembly assembly) + : base(assembly, configFileName: null, assembly.GetName().Version, UniqueIDGenerator.ForAssembly(assembly.Location, null)) { + _assembly = assembly; } - protected override async Task AfterTestAssemblyStartingAsync() +#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes + public FixtureArgumentSetTestAssembly() { - // Let everything initialize - await base.AfterTestAssemblyStartingAsync(); - - // Find the AssemblyFixtureAttributes placed on the test assembly - Aggregator.Run(() => - { - var fixtureAttributes = ((IReflectionAssemblyInfo)TestAssembly.Assembly).Assembly - .GetCustomAttributes(typeof(AssemblyFixtureAttribute), false) - .Cast(); - - foreach (var fixtureAttr in fixtureAttributes) - { - _assemblyFixtureMappings[fixtureAttr.FixtureType] = Activator.CreateInstance(fixtureAttr.FixtureType); - } - }); - - _context = ExecutionContext.Capture(); } +#pragma warning restore CS0618 - protected override async Task BeforeTestAssemblyFinishedAsync() + public new IReadOnlyCollection AssemblyFixtureTypes { - foreach (var fixture in _assemblyFixtureMappings.Values) + get { - switch (fixture) + if (_assemblyFixtureTypes == null) { - case IAsyncLifetime d: - await d.DisposeAsync(); - break; - case IAsyncDisposable d: - await d.DisposeAsync(); - break; - case IDisposable d: - d.Dispose(); - break; +#pragma warning disable CS0618 // AssemblyFixtureAttribute is obsolete; usage is required for assembly fixture discovery. + _assemblyFixtureTypes = _assembly + .GetCustomAttributes(typeof(global::Xunit.AssemblyFixtureAttribute), inherit: false) + .Cast() + .Select(attribute => attribute.AssemblyFixtureType) + .ToArray(); +#pragma warning restore CS0618 + + AssemblyFixtureTypesField?.SetValue(this, new Lazy>(() => _assemblyFixtureTypes)); } - } - await base.BeforeTestAssemblyFinishedAsync(); + return _assemblyFixtureTypes; + } } + } - protected override Task RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) + private sealed class FixtureArgumentSetAssemblyRunner : XunitTestAssemblyRunner + { + protected override async ValueTask RunTestCollection(XunitTestAssemblyRunnerContext context, IXunitTestCollection testCollection, IReadOnlyCollection testCases) { - Task result = null; - ExecutionContext.Run(_context, state => result = new CollectionRunner(_assemblyFixtureMappings, testCollection, testCases, DiagnosticMessageSink, messageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), cancellationTokenSource).RunAsync(), state: null); - return result; + var testCaseOrderer = context.AssemblyTestCaseOrderer ?? DefaultTestCaseOrderer.Instance ?? StableFallbackTestCaseOrderer.Instance; + var runner = new FixtureArgumentSetCollectionRunner(); + var summary = await runner.Run(testCollection, testCases, context.ExplicitOption, context.MessageBus, testCaseOrderer, context.Aggregator, context.CancellationTokenSource, context.AssemblyFixtureMappings); + return summary; } + } - public override void Dispose() + private sealed class FixtureArgumentSetCollectionRunner : XunitTestCollectionRunner + { + protected override async ValueTask RunTestClass(XunitTestCollectionRunnerContext context, IXunitTestClass testClass, IReadOnlyCollection testCases) { - _context?.Dispose(); - base.Dispose(); + var testCaseOrderer = context.TestCaseOrderer ?? DefaultTestCaseOrderer.Instance ?? StableFallbackTestCaseOrderer.Instance; + var classRunner = new FixtureArgumentSetClassRunner(); + var summary = await classRunner.Run(testClass, testCases, context.ExplicitOption, context.MessageBus, testCaseOrderer, context.Aggregator, context.CancellationTokenSource, context.CollectionFixtureMappings); + return summary; } } - private sealed class CollectionRunner : XunitTestCollectionRunner + private sealed class FixtureArgumentSetClassRunner : XunitTestClassRunner { - private readonly Dictionary _assemblyFixtureMappings; + private static readonly FieldInfo FixtureCacheField = typeof(FixtureMappingManager) + .GetField("fixtureCache", BindingFlags.Instance | BindingFlags.NonPublic); + + private static readonly FieldInfo ParentMappingManagerField = typeof(FixtureMappingManager) + .GetField("parentMappingManager", BindingFlags.Instance | BindingFlags.NonPublic); - public CollectionRunner(Dictionary assemblyFixtureMappings, ITestCollection testCollection, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ITestCaseOrderer testCaseOrderer, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) - : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource) + protected override ValueTask OnTestClassStarting(XunitTestClassRunnerContext context) { - _assemblyFixtureMappings = assemblyFixtureMappings; + InjectFixtureArguments(context); + return base.OnTestClassStarting(context); } - protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable testCases) + protected override ValueTask GetConstructorArgument(XunitTestClassRunnerContext context, ConstructorInfo constructor, int index, ParameterInfo parameter) { - EnsureArg.IsNotNull(testClass, nameof(testClass)); - EnsureArg.IsNotNull(@class, nameof(@class)); - EnsureArg.IsNotNull(testCases, nameof(testCases)); - - Dictionary combinedMappings = null; - - if (testClass.Class is TestClassWithFixtureArgumentsTypeInfo classWithFixtureArguments) + if (context?.TestClass is FixtureArgumentSetTestClass fixtureTestClass) { - combinedMappings = new Dictionary(CollectionFixtureMappings); - - foreach (var variant in classWithFixtureArguments.FixtureArguments) + var fixtureArguments = fixtureTestClass.GetFixtureArguments(); + if (fixtureArguments.Count > 0) { - combinedMappings.Add(variant.EnumValue.GetType(), variant.EnumValue); + foreach (var argument in fixtureArguments) + { + var enumValue = argument.EnumValue; + if (enumValue != null && parameter.ParameterType == enumValue.GetType()) + { + return new ValueTask(enumValue); + } + } } } - if (_assemblyFixtureMappings.Count > 0 && combinedMappings == null) + return base.GetConstructorArgument(context, constructor, index, parameter); + } + + protected override ValueTask CreateTestClassConstructorArguments(XunitTestClassRunnerContext context) + { + return base.CreateTestClassConstructorArguments(context); + } + + private static void InjectFixtureArguments(XunitTestClassRunnerContext context) + { + if (context?.TestClass == null) { - combinedMappings = new Dictionary(CollectionFixtureMappings); + return; } - foreach (var assemblyFixtureMapping in _assemblyFixtureMappings) + if (FixtureCacheField == null) { - combinedMappings.Add(assemblyFixtureMapping.Key, assemblyFixtureMapping.Value); + return; } - return new ExecutionContextFlowingClassRunner( - testClass, - @class, - testCases, - DiagnosticMessageSink, - MessageBus, - TestCaseOrderer, - new ExceptionAggregator(Aggregator), - CancellationTokenSource, - combinedMappings ?? CollectionFixtureMappings) - .RunAsync(); - } - } + var cacheOwner = context.ClassFixtureMappings; + if (ParentMappingManagerField?.GetValue(cacheOwner) is FixtureMappingManager parentMappingManager) + { + cacheOwner = parentMappingManager; + } - /// - /// An that runs tests in the same as when all fixture constructors ran. - /// This means that s set during a fixture constructor can be read during test method execution. - /// - private sealed class ExecutionContextFlowingClassRunner : XunitTestClassRunner - { - private ExecutionContext _context; + var cache = FixtureCacheField.GetValue(cacheOwner) as IDictionary; + if (cache == null) + { + return; + } - public ExecutionContextFlowingClassRunner(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ITestCaseOrderer testCaseOrderer, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, IDictionary collectionFixtureMappings) - : base(testClass, @class, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource, collectionFixtureMappings) - { - } + var fixtureArguments = new List(); + if (context.TestClass is FixtureArgumentSetTestClass fixtureTestClass) + { + fixtureArguments.AddRange(fixtureTestClass.GetFixtureArguments().Select(argument => argument.EnumValue)); + } - protected override void CreateClassFixture(Type fixtureType) - { - base.CreateClassFixture(fixtureType); - _context = ExecutionContext.Capture(); - } + var fixtureParameterTypes = new HashSet(); + var classFixtureTypes = context.TestClass.ClassFixtureTypes; + if (classFixtureTypes != null) + { + foreach (var fixtureType in classFixtureTypes) + { + var constructor = fixtureType.GetConstructors() + .SingleOrDefault(ctor => !ctor.IsStatic && ctor.IsPublic); + if (constructor == null) + { + continue; + } + + foreach (var parameter in constructor.GetParameters()) + { + if (parameter.ParameterType.IsEnum) + { + fixtureParameterTypes.Add(parameter.ParameterType); + } + } + } + } - protected override Task RunTestMethodsAsync() - { - if (_context == null) + var resolvedArguments = new Dictionary(); + + foreach (var enumValue in fixtureArguments) { - // no class fixtures, so context needed + var argumentType = enumValue.GetType(); - return base.RunTestMethodsAsync(); + foreach (var parameterType in fixtureParameterTypes) + { + if (string.Equals(parameterType.FullName, argumentType.FullName, StringComparison.Ordinal) + && !resolvedArguments.ContainsKey(parameterType)) + { + resolvedArguments[parameterType] = parameterType == argumentType + ? enumValue + : Enum.ToObject(parameterType, Convert.ToInt64(enumValue)); + } + } } - Task result = null; - ExecutionContext.Run(_context, state => result = base.RunTestMethodsAsync(), null); - return result; - } - } + if (resolvedArguments.Count == 0 && fixtureParameterTypes.Count > 0) + { + var traits = GetTraits(context.TestCases); + foreach (var parameterType in fixtureParameterTypes) + { + var traitKey = parameterType.Name; + if (!traits.TryGetValue(traitKey, out var values) || values.Count == 0) + { + continue; + } + + if (values.Count > 1) + { + throw new TestPipelineException($"Fixture argument '{traitKey}' had multiple values: {string.Join(", ", values)}"); + } + + var value = values.First(); + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + if (!Enum.TryParse(parameterType, value, ignoreCase: true, out var parsedValue)) + { + throw new TestPipelineException($"Fixture argument '{traitKey}' value '{value}' could not be parsed as {parameterType.FullName}."); + } + + resolvedArguments[parameterType] = parsedValue; + } + } - /// - /// facts have special optimized serialization. Their deserialization needs to be able to instantiate - /// the synthetic classes that we created of the form Namespace.Class(Arg1, Arg2). For these, we want to create a - /// with Arg1 and Arg2 as the fixture arguments - /// - private sealed class CustomAssemblyInfo : LongLivedMarshalByRefObject, IAssemblyInfo - { - private readonly IAssemblyInfo _assemblyInfoImplementation; - private readonly Regex _argumentsRegex = new Regex(@"\((\s*(?[^, )]+)\s*,?)*\)"); + foreach (var resolvedArgument in resolvedArguments) + { + if (!cache.ContainsKey(resolvedArgument.Key)) + { + cache[resolvedArgument.Key] = resolvedArgument.Value; + } + } + } - public CustomAssemblyInfo(IAssemblyInfo assemblyInfoImplementation) + private static Dictionary> GetTraits(IReadOnlyCollection testCases) { - _assemblyInfoImplementation = assemblyInfoImplementation; - } + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); - public string AssemblyPath => _assemblyInfoImplementation.AssemblyPath; + foreach (var testCase in testCases) + { + if (testCase is XunitTestCase xunitTestCase) + { + MergeTraits(traits, xunitTestCase.Traits.ToDictionary( + kvp => kvp.Key, + kvp => (IReadOnlyCollection)kvp.Value.ToArray(), + StringComparer.OrdinalIgnoreCase)); + continue; + } - public string Name => _assemblyInfoImplementation.Name; + if (testCase is ITestCaseMetadata metadata) + { + MergeTraits(traits, metadata.Traits); + } + } - public IEnumerable GetCustomAttributes(string assemblyQualifiedAttributeTypeName) - { - return _assemblyInfoImplementation.GetCustomAttributes(assemblyQualifiedAttributeTypeName); + return traits.ToDictionary( + kvp => kvp.Key, + kvp => (IReadOnlyCollection)kvp.Value.ToArray(), + StringComparer.OrdinalIgnoreCase); } - public ITypeInfo GetType(string typeName) + private static void MergeTraits(Dictionary> target, IReadOnlyDictionary> source) { - EnsureArg.IsNotNull(typeName, nameof(typeName)); - - // parse out the (Arg1, Arg2) - var match = _argumentsRegex.Match(typeName); - if (!match.Success) + foreach (var kvp in source) { - return _assemblyInfoImplementation.GetType(typeName); + if (!target.TryGetValue(kvp.Key, out var values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + target[kvp.Key] = values; + } + + values.UnionWith(kvp.Value); } + } + } - // retrieve the real type - var typeInfo = _assemblyInfoImplementation.GetType(typeName.Substring(0, match.Index)); - Debug.Assert(typeInfo != null, $"Could not find type {typeName} in assembly"); + private sealed class StableFallbackTestCaseOrderer : ITestCaseOrderer + { + internal static ITestCaseOrderer Instance { get; } = new StableFallbackTestCaseOrderer(); - // now get the the arguments. We don't know what type they are, so we look at the FixtureArgumentSetsAttribute on the class and look at its arguments - IAttributeInfo attributeInfo = typeInfo.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute)).Single(); + public IReadOnlyCollection OrderTestCases(IReadOnlyCollection testCases) + where TTestCase : ITestCase + { + ArgumentNullException.ThrowIfNull(testCases); - SingleFlag[] arguments = attributeInfo - .GetConstructorArguments() - .Cast() - .Zip( - match.Groups["VALUE"].Captures, - (e, c) => new SingleFlag((Enum)Enum.Parse(e.GetType(), c.Value))) + return testCases + .OrderBy(testCase => GetSortKey(testCase)) .ToArray(); - - return new TestClassWithFixtureArgumentsTypeInfo(typeInfo, arguments); } - public IEnumerable GetTypes(bool includePrivateTypes) + private static string GetSortKey(TTestCase testCase) { - return _assemblyInfoImplementation.GetTypes(includePrivateTypes); + if (testCase is ITestCaseMetadata metadata) + { + return metadata.TestCaseDisplayName ?? string.Empty; + } + + return testCase?.ToString() ?? string.Empty; } } } From b37f0e067de82dc793104540543ba47f6db8518c Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:30:10 -0700 Subject: [PATCH 006/105] Replace TestClassWithFixtureArguments with FixtureArgumentSet* for v3 xUnit v3 needs its own IXunitTestClass/IXunitTestCollection/IXunitTestMethod implementations that carry the fixture-argument-set variants. - Add FixtureArgumentSetTestClass, FixtureArgumentSetTestCollection, FixtureArgumentSetTestMethod. - Add FixtureArgumentSetsAttribute (carries the Flags enum combinations + CollectionBehavior). - Remove obsolete TestClassWithFixtureArguments and TestClassWithFixtureArgumentsTypeInfo (v2 ITypeInfo-based). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../FixtureArgumentSetTestClass.cs | 72 +++++++ .../FixtureArgumentSetTestCollection.cs | 73 +++++++ .../FixtureArgumentSetTestMethod.cs | 181 ++++++++++++++++++ .../FixtureArgumentSetsAttribute.cs | 31 +++ .../TestClassWithFixtureArguments.cs | 91 --------- .../TestClassWithFixtureArgumentsTypeInfo.cs | 76 -------- 6 files changed, 357 insertions(+), 167 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs delete mode 100644 src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArguments.cs delete mode 100644 src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs new file mode 100644 index 0000000000..4fc13b2293 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs @@ -0,0 +1,72 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using EnsureThat; +using Xunit.Sdk; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit +{ + internal sealed class FixtureArgumentSetTestClass : XunitTestClass + { + private static readonly FieldInfo UniqueIdField = typeof(XunitTestClass).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic); + private Type _testClassType; + private IReadOnlyList _fixtureArguments; + + public FixtureArgumentSetTestClass(Type testClassType, IXunitTestCollection testCollection, IReadOnlyList fixtureArguments, string uniqueId) + : base(testClassType, testCollection, uniqueId) + { + EnsureArg.IsNotNull(testClassType, nameof(testClassType)); + EnsureArg.IsNotNull(testCollection, nameof(testCollection)); + EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); + + _testClassType = testClassType; + _fixtureArguments = fixtureArguments; + UpdateUniqueId(); + } + +#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes + public FixtureArgumentSetTestClass() + { + _fixtureArguments = Array.Empty(); + } +#pragma warning restore CS0618 + +#pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists + public new Type Class => _testClassType ?? base.Class; +#pragma warning restore SA1100 + + internal IReadOnlyList GetFixtureArguments() + { + return _fixtureArguments; + } + + internal void ApplyFixtureArguments(IReadOnlyList fixtureArguments) + { + if (fixtureArguments == null) + { + return; + } + + _fixtureArguments = fixtureArguments; + UpdateUniqueId(); + } + + private void UpdateUniqueId() + { + if (_fixtureArguments.Count == 0) + { + return; + } + + var className = $"{TestClassName}({string.Join(", ", _fixtureArguments.Select(v => v.EnumValue))})"; + UniqueIdField?.SetValue(this, UniqueIDGenerator.ForTestClass(TestCollection.UniqueID, className)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs new file mode 100644 index 0000000000..83a1f2e9dd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs @@ -0,0 +1,73 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using EnsureThat; +using Xunit.Sdk; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit +{ + internal sealed class FixtureArgumentSetTestCollection : XunitTestCollection + { + private static readonly FieldInfo TestCollectionDisplayNameField = typeof(XunitTestCollection).GetField("testCollectionDisplayName", BindingFlags.Instance | BindingFlags.NonPublic); + private static readonly FieldInfo UniqueIdField = typeof(XunitTestCollection).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic); + + private IReadOnlyList _fixtureArguments; + private string _testClassName; + + public FixtureArgumentSetTestCollection(IXunitTestAssembly testAssembly, IReadOnlyList fixtureArguments, string testClassName = null) + : base(testAssembly, collectionDefinition: null, disableParallelization: false, displayName: string.Empty, uniqueID: string.Empty) + { + EnsureArg.IsNotNull(testAssembly, nameof(testAssembly)); + EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); + + _fixtureArguments = fixtureArguments; + _testClassName = testClassName; + + UpdateDisplayAndUniqueId(testAssembly.UniqueID); + } + +#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes + public FixtureArgumentSetTestCollection() + { + _fixtureArguments = Array.Empty(); + } +#pragma warning restore CS0618 + + internal void ApplyFixtureArguments(IReadOnlyList fixtureArguments) + { + if (fixtureArguments == null) + { + return; + } + + _fixtureArguments = fixtureArguments; + UpdateDisplayAndUniqueId(TestAssembly.UniqueID); + } + + private void UpdateDisplayAndUniqueId(string assemblyUniqueId) + { + if (_fixtureArguments.Count == 0) + { + return; + } + + // Collection display name is either the variant label (shared-per-variant) or + // test class + variant (per-class), depending on the caller's choice. +#pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists + var argsLabel = string.Join(", ", _fixtureArguments.Select(v => v.EnumValue)); + var displayName = string.IsNullOrEmpty(_testClassName) + ? $"{base.TestCollectionDisplayName}({argsLabel})" + : $"{_testClassName}({argsLabel})"; + TestCollectionDisplayNameField?.SetValue(this, displayName); + UniqueIdField?.SetValue(this, UniqueIDGenerator.ForTestCollection(assemblyUniqueId, displayName, base.TestCollectionClassName)); +#pragma warning restore SA1100 + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs new file mode 100644 index 0000000000..80134a09ab --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -0,0 +1,181 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using EnsureThat; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit +{ + internal sealed class FixtureArgumentSetTestMethod : XunitTestMethod + { + private static readonly FieldInfo TestMethodArgumentsField = typeof(XunitTestMethod).GetField("testMethodArguments", BindingFlags.Instance | BindingFlags.NonPublic); + private static readonly FieldInfo TraitsField = typeof(XunitTestMethod).GetField("traits", BindingFlags.Instance | BindingFlags.NonPublic); + private static readonly FieldInfo UniqueIdField = typeof(XunitTestMethod).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic); + private static readonly FieldInfo MethodField = typeof(XunitTestMethod).GetField("method", BindingFlags.Instance | BindingFlags.NonPublic); + + private readonly FixtureArgumentSetTestClass _testClass; + private readonly MethodInfo _methodInfo; + private readonly IReadOnlyList _fixtureArguments; + private readonly string _uniqueId; + + public FixtureArgumentSetTestMethod(FixtureArgumentSetTestClass testClass, MethodInfo methodInfo, IReadOnlyList fixtureArguments, string uniqueId) + : base(testClass, methodInfo, testMethodArguments: Array.Empty(), uniqueId) + { + EnsureArg.IsNotNull(testClass, nameof(testClass)); + EnsureArg.IsNotNull(methodInfo, nameof(methodInfo)); + EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); + + _testClass = testClass; + _methodInfo = methodInfo; + _fixtureArguments = fixtureArguments; + _uniqueId = uniqueId; + + UpdateMethodArguments(Array.Empty()); + UniqueIdField?.SetValue(this, _uniqueId); + MethodField?.SetValue(this, _methodInfo); + } + +#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes + public FixtureArgumentSetTestMethod() + { + } +#pragma warning restore CS0618 + +#pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists + public new string GetDisplayName(string baseDisplayName, string displayName, object[] testMethodArguments, Type[] genericTypes) + { + var combinedArguments = CombineFixtureAndMethodArguments(testMethodArguments ?? Array.Empty()); + return base.GetDisplayName(baseDisplayName, displayName, combinedArguments, genericTypes); + } + + public new object[] ResolveMethodArguments(object[] arguments) + { + var combinedArguments = CombineFixtureAndMethodArguments(arguments ?? Array.Empty()); + UpdateMethodArguments(combinedArguments); + return base.ResolveMethodArguments(combinedArguments); + } +#pragma warning restore SA1100 + + private object[] CombineFixtureAndMethodArguments(object[] methodArguments) + { + if (_fixtureArguments.Count == 0) + { + return methodArguments; + } + + var result = new object[_fixtureArguments.Count + methodArguments.Length]; + for (int i = 0; i < _fixtureArguments.Count; i++) + { + result[i] = _fixtureArguments[i].EnumValue; + } + + if (methodArguments.Length > 0) + { + Array.Copy(methodArguments, 0, result, _fixtureArguments.Count, methodArguments.Length); + } + + return result; + } + + private void UpdateMethodArguments(object[] methodArguments) + { + if (_fixtureArguments.Count == 0) + { + return; + } + + TestMethodArgumentsField?.SetValue(this, methodArguments); + } + + internal void UpdateArgumentsFromMethod() + { + if (_fixtureArguments.Count == 0) + { + return; + } + + var combinedArguments = CombineFixtureAndMethodArguments(Array.Empty()); + TestMethodArgumentsField?.SetValue(this, combinedArguments); + + if (TraitsField != null) + { + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); +#pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists + foreach (var kvp in base.Traits) + { + traits[kvp.Key] = new HashSet(kvp.Value, StringComparer.OrdinalIgnoreCase); + } +#pragma warning restore SA1100 + + for (int i = 0; i < _fixtureArguments.Count; i++) + { + var enumValue = _fixtureArguments[i].EnumValue; + var enumValueText = enumValue.ToString(); + + string key = $"{enumValue.GetType().Name}"; + if (!traits.TryGetValue(key, out var values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + traits[key] = values; + } + + values.Add(enumValueText); + + if (TryGetFixtureTraitName(enumValue, out var traitName)) + { + AddTrait(traits, traitName, enumValueText); + } + } + + var typedTraits = traits.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyCollection)kvp.Value, StringComparer.OrdinalIgnoreCase); + TraitsField.SetValue(this, new Lazy>>(() => typedTraits)); + } + } + + private static void AddTrait(Dictionary> traits, string key, string value) + { + if (!traits.TryGetValue(key, out var values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + traits[key] = values; + } + + values.Add(value); + } + + private static bool TryGetFixtureTraitName(Enum enumValue, out string traitName) + { + traitName = null; + if (enumValue == null) + { + return false; + } + + var enumType = enumValue.GetType(); + if (!string.Equals(enumType.Namespace, "Microsoft.Health.Fhir.Tests.Common.FixtureParameters", StringComparison.Ordinal)) + { + return false; + } + + if (string.Equals(enumType.Name, "DataStore", StringComparison.Ordinal)) + { + traitName = "DataStore"; + return true; + } + + if (string.Equals(enumType.Name, "Format", StringComparison.Ordinal)) + { + traitName = "Format"; + return true; + } + + return false; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs index 0483db4bc6..9133fa16b4 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs @@ -4,10 +4,27 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using EnsureThat; namespace Microsoft.Health.Extensions.Xunit { + /// + /// Controls how test collections are grouped for fixture argument variants. + /// + public enum FixtureArgumentSetCollectionBehavior + { + /// + /// All test classes with the same fixture argument variant share a collection. + /// + SharedPerVariant, + + /// + /// Each test class gets its own collection for a given fixture argument variant. + /// + PerClass, + } + /// /// Derive from this attribute to declare combinations of argument values that a class fixture's constructor should be called with. /// Arguments are required to be flags enums. @@ -15,9 +32,23 @@ namespace Microsoft.Health.Extensions.Xunit [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, Inherited = false)] public abstract class FixtureArgumentSetsAttribute : Attribute { + private readonly Enum[] _argumentSets; + protected FixtureArgumentSetsAttribute(params Enum[] argumentSets) { EnsureArg.IsNotNull(argumentSets, nameof(argumentSets)); + _argumentSets = argumentSets; + CollectionBehavior = FixtureArgumentSetCollectionBehavior.SharedPerVariant; + } + + /// + /// Gets the collection grouping behavior for fixture argument variants. + /// + public FixtureArgumentSetCollectionBehavior CollectionBehavior { get; protected set; } + + internal IReadOnlyList GetArgumentSets() + { + return _argumentSets; } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArguments.cs b/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArguments.cs deleted file mode 100644 index b7c609b0ae..0000000000 --- a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArguments.cs +++ /dev/null @@ -1,91 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using EnsureThat; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Microsoft.Health.Extensions.Xunit -{ - /// - /// A where the class' is a . - /// This test class uses a fixture with a single set of constructor arguments. - /// - internal class TestClassWithFixtureArguments : LongLivedMarshalByRefObject, ITestClass - { - private ITypeInfo _underlyingClass; - - [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] - public TestClassWithFixtureArguments() - { - } - - public TestClassWithFixtureArguments(ITestCollection testCollection, ITypeInfo underlyingClass, SingleFlag[] fixtureArguments) - { - EnsureArg.IsNotNull(testCollection, nameof(testCollection)); - EnsureArg.IsNotNull(underlyingClass, nameof(underlyingClass)); - EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - - _underlyingClass = underlyingClass; - Class = new TestClassWithFixtureArgumentsTypeInfo(underlyingClass, fixtureArguments); - FixtureArguments = fixtureArguments; - TestCollection = testCollection; - } - - /// - public ITypeInfo Class { get; set; } - - public IReadOnlyList FixtureArguments { get; set; } - - /// - public ITestCollection TestCollection { get; set; } - - /// - public void Serialize(IXunitSerializationInfo info) - { - info.AddValue("TestCollection", TestCollection); - info.AddValue("ClassAssemblyName", _underlyingClass.Assembly.Name); - info.AddValue("ClassTypeName", _underlyingClass.Name); - info.AddValue("FixtureArguments", FixtureArguments.Select(a => a.EnumValue).ToArray()); - } - - /// - public void Deserialize(IXunitSerializationInfo info) - { - TestCollection = info.GetValue("TestCollection"); - - var assemblyName = info.GetValue("ClassAssemblyName"); - var typeName = info.GetValue("ClassTypeName"); - - _underlyingClass = Reflector.Wrap(GetType(assemblyName, typeName)); - - Enum[] rawFixtureArguments = info.GetValue("FixtureArguments"); - - FixtureArguments = Array.ConvertAll(rawFixtureArguments, e => new SingleFlag(e)); - - Class = new TestClassWithFixtureArgumentsTypeInfo(_underlyingClass, FixtureArguments); - } - - /// - /// Converts an assembly name + type name into a object. - /// - /// The assembly name. - /// The type name. - /// The instance of the , if available; null, otherwise. - public static Type GetType(string assemblyName, string typeName) - { - // Make sure we only use the short form - var an = new AssemblyName(assemblyName); - var assembly = Assembly.Load(new AssemblyName { Name = an.Name, Version = an.Version }); - - return assembly == null ? null : assembly.GetType(typeName); - } - } -} diff --git a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs b/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs deleted file mode 100644 index 58c5052f3a..0000000000 --- a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs +++ /dev/null @@ -1,76 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using EnsureThat; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Microsoft.Health.Extensions.Xunit -{ - /// - /// A special for a test class that will use a fixture instantiated with with a single set of constructor arguments. - /// The property is customized to be of the form Namespace.Class(Arg1, Arg2) - /// - internal sealed class TestClassWithFixtureArgumentsTypeInfo : LongLivedMarshalByRefObject, IReflectionTypeInfo - { - private readonly ITypeInfo _typeInfoImplementation; - - public TestClassWithFixtureArgumentsTypeInfo(ITypeInfo typeInfoImplementation, IReadOnlyList fixtureArguments) - { - EnsureArg.IsNotNull(typeInfoImplementation, nameof(typeInfoImplementation)); - EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - - _typeInfoImplementation = typeInfoImplementation; - FixtureArguments = fixtureArguments; - Name = $"{typeInfoImplementation.Name}({string.Join(", ", fixtureArguments.Select(v => $"{v.EnumValue}"))})"; - } - - public IReadOnlyList FixtureArguments { get; } - - public IAssemblyInfo Assembly => _typeInfoImplementation.Assembly; - - public ITypeInfo BaseType => _typeInfoImplementation.BaseType; - - public IEnumerable Interfaces => _typeInfoImplementation.Interfaces; - - public bool IsAbstract => _typeInfoImplementation.IsAbstract; - - public bool IsGenericParameter => _typeInfoImplementation.IsGenericParameter; - - public bool IsGenericType => _typeInfoImplementation.IsGenericType; - - public bool IsSealed => _typeInfoImplementation.IsSealed; - - public bool IsValueType => _typeInfoImplementation.IsValueType; - - public string Name { get; } - - public Type Type => ((ReflectionTypeInfo)_typeInfoImplementation).Type; - - public IEnumerable GetCustomAttributes(string assemblyQualifiedAttributeTypeName) - { - return _typeInfoImplementation.GetCustomAttributes(assemblyQualifiedAttributeTypeName); - } - - public IEnumerable GetGenericArguments() - { - return _typeInfoImplementation.GetGenericArguments(); - } - - public IMethodInfo GetMethod(string methodName, bool includePrivateMethod) - { - return _typeInfoImplementation.GetMethod(methodName, includePrivateMethod); - } - - public IEnumerable GetMethods(bool includePrivateMethods) - { - return _typeInfoImplementation.GetMethods(includePrivateMethods); - } - } -} From a4c99e91353ac3356c66187560d9462878d262da Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:30:30 -0700 Subject: [PATCH 007/105] Port retry attributes and add local Skip/SkippableFact/SkippableTheory - AssemblyFixtureAttribute, RetryFactAttribute, RetryFactDiscoverer, RetryTestCase, RetryTheoryAttribute, RetryTheoryDiscoverer updated for xUnit v3 API (Assembly-based ctors, ValueTask discovery, IXunitTestCase/IXunitTestCollection abstractions). - Add local Skip, SkippableFactAttribute, SkippableTheoryAttribute classes replacing the now-removed Xunit.SkippableFact package. Skip.If/IfNot throw SkipException.ForSkip(reason) which xUnit v3 recognizes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AssemblyFixtureAttribute.cs | 16 +-- .../RetryFactAttribute.cs | 15 ++- .../RetryFactDiscoverer.cs | 62 +++++----- .../RetryTestCase.cs | 91 ++++++++------ .../RetryTheoryAttribute.cs | 15 ++- .../RetryTheoryDiscoverer.cs | 111 ++++++++---------- src/Microsoft.Health.Extensions.Xunit/Skip.cs | 41 +++++++ .../SkippableFactAttribute.cs | 27 +++++ .../SkippableTheoryAttribute.cs | 27 +++++ 9 files changed, 256 insertions(+), 149 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit/Skip.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs diff --git a/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs index 1893dfeb76..9737306b33 100644 --- a/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs @@ -4,24 +4,12 @@ // ------------------------------------------------------------------------------------------------- using System; -using EnsureThat; namespace Microsoft.Health.Extensions.Xunit { - /// - /// Placed on an assembly to indicate that the given type should be instantiated - /// before any tests are executed and disposed (if it implements IDisposable) - /// at the end of the test run. - /// - [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)] + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true, Inherited = false)] + [Obsolete("Use Xunit.AssemblyFixtureAttribute instead.")] public sealed class AssemblyFixtureAttribute : Attribute { - public AssemblyFixtureAttribute(Type fixtureType) - { - EnsureArg.IsNotNull(fixtureType, nameof(fixtureType)); - FixtureType = fixtureType; - } - - public Type FixtureType { get; } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs index 60f6e32b2d..1d9a7974ae 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs @@ -4,8 +4,9 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Runtime.CompilerServices; using Xunit; -using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { @@ -13,10 +14,20 @@ namespace Microsoft.Health.Extensions.Xunit /// Attribute that marks a test method to be retried a specified number of times if it fails. /// Useful for handling transient failures in integration and end-to-end tests. /// - [XunitTestCaseDiscoverer("Microsoft.Health.Extensions.Xunit.RetryFactDiscoverer", "Microsoft.Health.Extensions.Xunit")] + [XunitTestCaseDiscoverer(typeof(RetryFactDiscoverer))] [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class RetryFactAttribute : FactAttribute { + public RetryFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + { + SourceFilePath = sourceFilePath; + SourceLineNumber = sourceLineNumber; + } + + internal new string SourceFilePath { get; } + + internal new int SourceLineNumber { get; } + /// /// Gets or sets the maximum number of retry attempts (default is 3). /// diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs index f45ab9690f..9d30b5ca09 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs @@ -4,51 +4,49 @@ // ------------------------------------------------------------------------------------------------- using System.Collections.Generic; -using Xunit.Abstractions; +using System.Linq; +using System.Threading.Tasks; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { /// /// Test case discoverer for . /// - public class RetryFactDiscoverer : IXunitTestCaseDiscoverer + public sealed class RetryFactDiscoverer : IXunitTestCaseDiscoverer { - private readonly IMessageSink _diagnosticMessageSink; - - public RetryFactDiscoverer(IMessageSink diagnosticMessageSink) - { - _diagnosticMessageSink = diagnosticMessageSink; - } - - public IEnumerable Discover( + public ValueTask> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo factAttribute) + IXunitTestMethod testMethod, + IFactAttribute factAttribute) { - var maxRetries = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.MaxRetries)); - var delayMs = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.DelayBetweenRetriesMs)); - var retryOnAssertionFailure = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.RetryOnAssertionFailure)); - - // Use default values if not specified - if (maxRetries == 0) - { - maxRetries = 3; - } + var attribute = (RetryFactAttribute)factAttribute; - if (delayMs == 0) - { - delayMs = 5000; - } + var maxRetries = attribute.MaxRetries; + var delayMs = attribute.DelayBetweenRetriesMs; + var retryOnAssertionFailure = attribute.RetryOnAssertionFailure; - yield return new RetryTestCase( - _diagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), + var testCase = new RetryTestCase( testMethod, - maxRetries, - delayMs, - retryOnAssertionFailure); + testMethod.GetDisplayName(testMethod.MethodName, label: null, testMethodArguments: null, methodGenericTypes: null), + UniqueIDGenerator.ForTestCase(testMethod.UniqueID, null, null), + @explicit: attribute.Explicit, + skipExceptions: attribute.SkipExceptions, + skipReason: attribute.Skip, + skipType: attribute.SkipType, + skipUnless: attribute.SkipUnless, + skipWhen: attribute.SkipWhen, + traits: testMethod.Traits.ToDictionary(kvp => kvp.Key, kvp => new HashSet(kvp.Value)), + testMethodArguments: null, + sourceFile: attribute.SourceFilePath, + sourceLine: attribute.SourceLineNumber, + timeout: attribute.Timeout, + maxRetries: maxRetries, + delayMs: delayMs, + retryOnAssertionFailure: retryOnAssertionFailure); + + return new ValueTask>(new[] { testCase }); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 1afc54c07f..3975e5624d 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -4,19 +4,22 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; -using Xunit.Abstractions; +using Xunit; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { /// /// Test case that implements retry logic. /// - public class RetryTestCase : XunitTestCase + public sealed class RetryTestCase : XunitTestCase, ISelfExecutingXunitTestCase { private int _maxRetries; private int _delayMs; @@ -29,30 +32,39 @@ public RetryTestCase() } public RetryTestCase( - IMessageSink diagnosticMessageSink, - TestMethodDisplay defaultMethodDisplay, - TestMethodDisplayOptions defaultMethodDisplayOptions, - ITestMethod testMethod, + IXunitTestMethod testMethod, + string displayName, + string uniqueId, + bool @explicit, + Type[] skipExceptions, + string skipReason, + Type skipType, + string skipUnless, + string skipWhen, + Dictionary> traits, + object[] testMethodArguments, + string sourceFile, + int? sourceLine, + int? timeout, int maxRetries, int delayMs, - bool retryOnAssertionFailure = false, - object[] testMethodArguments = null) - : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) + bool retryOnAssertionFailure) + : base(testMethod, displayName, uniqueId, @explicit, skipExceptions, skipReason, skipType, skipUnless, skipWhen, traits, testMethodArguments, sourceFile, sourceLine, timeout) { _maxRetries = maxRetries; _delayMs = delayMs; _retryOnAssertionFailure = retryOnAssertionFailure; } - public override async Task RunAsync( - IMessageSink diagnosticMessageSink, + public async ValueTask Run( + ExplicitOption explicitOption, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { // Use System.Diagnostics.Trace for ADO visibility - Trace.WriteLine($"##vso[task.logdetail]RetryFact starting test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' with MaxRetries={_maxRetries}, DelayMs={_delayMs}, RetryOnAssertionFailure={_retryOnAssertionFailure}"); + Trace.WriteLine($"##vso[task.logdetail]RetryFact starting test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' with MaxRetries={_maxRetries}, DelayMs={_delayMs}, RetryOnAssertionFailure={_retryOnAssertionFailure}"); var runSummary = new RunSummary { Total = 1 }; Exception lastException = null; @@ -61,7 +73,7 @@ public override async Task RunAsync( { var isLastAttempt = attempt == _maxRetries; - Trace.WriteLine($"##vso[task.logdetail]RetryFact attempt {attempt}/{_maxRetries} for test '{TestMethod.Method.Name}'"); + Trace.WriteLine($"##vso[task.logdetail]RetryFact attempt {attempt}/{_maxRetries} for test '{TestMethod.MethodName}'"); // Create a fresh aggregator for each attempt var attemptAggregator = new ExceptionAggregator(); @@ -83,21 +95,22 @@ public override async Task RunAsync( try { - var summary = await base.RunAsync( - diagnosticMessageSink, + var summary = await XunitRunnerHelper.RunXunitTestCase( + this, busToUse, - constructorArguments, + cancellationTokenSource, attemptAggregator, - cancellationTokenSource); + explicitOption, + constructorArguments); runSummary.Time = summary.Time; if (summary.Failed == 0) { // Test passed - success message already went through to Test Explorer - Trace.WriteLine($"##vso[task.logdetail]RetryFact test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' passed on attempt {attempt}/{_maxRetries}"); - diagnosticMessageSink.OnMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' passed on attempt {attempt}/{_maxRetries}")); + Trace.WriteLine($"##vso[task.logdetail]RetryFact test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' passed on attempt {attempt}/{_maxRetries}"); + messageBus.QueueMessage( + new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' passed on attempt {attempt}/{_maxRetries}")); runSummary.Failed = 0; return runSummary; @@ -140,9 +153,9 @@ public override async Task RunAsync( if (!shouldRetry) { - Trace.WriteLine($"##vso[task.logissue type=warning]Test '{TestMethod.Method.Name}' failed with non-retriable exception. Skipping retries."); - diagnosticMessageSink.OnMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.Method.Name}' failed with non-retriable exception. Skipping retries.")); + Trace.WriteLine($"##vso[task.logissue type=warning]Test '{TestMethod.MethodName}' failed with non-retriable exception. Skipping retries."); + messageBus.QueueMessage( + new DiagnosticMessage($"[RetryFact] Test '{TestMethod.MethodName}' failed with non-retriable exception. Skipping retries.")); if (lastException != null) { @@ -154,18 +167,18 @@ public override async Task RunAsync( } // Not the last attempt - the failure was intercepted, so retry - Trace.WriteLine($"##vso[task.logissue type=warning]Test '{TestMethod.Method.Name}' failed on attempt {attempt}/{_maxRetries}, will retry after {_delayMs}ms"); - diagnosticMessageSink.OnMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' failed on attempt {attempt}/{_maxRetries}. Retrying after {_delayMs}ms delay. Error: {lastException?.Message ?? "No exception message"}")); + Trace.WriteLine($"##vso[task.logissue type=warning]Test '{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}, will retry after {_delayMs}ms"); + messageBus.QueueMessage( + new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}. Retrying after {_delayMs}ms delay. Error: {lastException?.Message ?? "No exception message"}")); await Task.Delay(_delayMs); } else { // Last attempt - failure message already went through to Test Explorer - Trace.WriteLine($"##vso[task.logissue type=error]Test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' failed after {_maxRetries} attempts. Final error: {lastException?.Message ?? "No exception captured"}"); - diagnosticMessageSink.OnMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}' failed after {_maxRetries} attempts. Last exception: {lastException?.Message ?? "No exception message"}")); + Trace.WriteLine($"##vso[task.logissue type=error]Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed after {_maxRetries} attempts. Final error: {lastException?.Message ?? "No exception captured"}"); + messageBus.QueueMessage( + new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed after {_maxRetries} attempts. Last exception: {lastException?.Message ?? "No exception message"}")); if (lastException != null) { @@ -199,20 +212,20 @@ public override async Task RunAsync( return runSummary; } - public override void Serialize(IXunitSerializationInfo data) + protected override void Serialize(IXunitSerializationInfo info) { - base.Serialize(data); - data.AddValue(nameof(_maxRetries), _maxRetries); - data.AddValue(nameof(_delayMs), _delayMs); - data.AddValue(nameof(_retryOnAssertionFailure), _retryOnAssertionFailure); + base.Serialize(info); + info.AddValue(nameof(_maxRetries), _maxRetries); + info.AddValue(nameof(_delayMs), _delayMs); + info.AddValue(nameof(_retryOnAssertionFailure), _retryOnAssertionFailure); } - public override void Deserialize(IXunitSerializationInfo data) + protected override void Deserialize(IXunitSerializationInfo info) { - base.Deserialize(data); - _maxRetries = data.GetValue(nameof(_maxRetries)); - _delayMs = data.GetValue(nameof(_delayMs)); - _retryOnAssertionFailure = data.GetValue(nameof(_retryOnAssertionFailure)); + base.Deserialize(info); + _maxRetries = info.GetValue(nameof(_maxRetries)); + _delayMs = info.GetValue(nameof(_delayMs)); + _retryOnAssertionFailure = info.GetValue(nameof(_retryOnAssertionFailure)); } /// diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs index 88a65b6b54..48ca973d04 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs @@ -4,8 +4,9 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Runtime.CompilerServices; using Xunit; -using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { @@ -13,10 +14,20 @@ namespace Microsoft.Health.Extensions.Xunit /// Attribute that marks a theory method to be retried a specified number of times if it fails. /// Useful for handling transient failures in integration and end-to-end tests with parameterized data. /// - [XunitTestCaseDiscoverer("Microsoft.Health.Extensions.Xunit.RetryTheoryDiscoverer", "Microsoft.Health.Extensions.Xunit")] + [XunitTestCaseDiscoverer(typeof(RetryTheoryDiscoverer))] [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class RetryTheoryAttribute : TheoryAttribute { + public RetryTheoryAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + { + SourceFilePath = sourceFilePath; + SourceLineNumber = sourceLineNumber; + } + + internal new string SourceFilePath { get; } + + internal new int SourceLineNumber { get; } + /// /// Gets or sets the maximum number of retry attempts (default is 3). /// diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index 9539e3c1db..ee5445fe27 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -5,8 +5,10 @@ using System.Collections.Generic; using System.Linq; -using Xunit.Abstractions; +using System.Threading.Tasks; +using Xunit; using Xunit.Sdk; +using Xunit.v3; namespace Microsoft.Health.Extensions.Xunit { @@ -15,77 +17,66 @@ namespace Microsoft.Health.Extensions.Xunit /// For Theory tests, we need to let xUnit discover the data-driven test cases first, /// then wrap each one with retry logic. /// - public class RetryTheoryDiscoverer : TheoryDiscoverer + public sealed class RetryTheoryDiscoverer : TheoryDiscoverer { - public RetryTheoryDiscoverer(IMessageSink diagnosticMessageSink) - : base(diagnosticMessageSink) - { - } - - protected override IEnumerable CreateTestCasesForDataRow( + protected override async ValueTask> CreateTestCasesForDataRow( ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo theoryAttribute, - object[] dataRow) + IXunitTestMethod testMethod, + ITheoryAttribute theoryAttribute, + ITheoryDataRow dataRow, + object[] testMethodArguments) { - var maxRetries = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.MaxRetries)); - var delayMs = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); - var retryOnAssertionFailure = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.RetryOnAssertionFailure)); - - // Use default values if not specified - if (maxRetries == 0) - { - maxRetries = 3; - } + var attribute = (RetryTheoryAttribute)theoryAttribute; - if (delayMs == 0) - { - delayMs = 5000; - } + var maxRetries = attribute.MaxRetries; + var delayMs = attribute.DelayBetweenRetriesMs; + var retryOnAssertionFailure = attribute.RetryOnAssertionFailure; - // Create a RetryTestCase for each data row - yield return new RetryTestCase( - DiagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), - testMethod, - maxRetries, - delayMs, - retryOnAssertionFailure, - dataRow); + var baseCases = await base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments); + return baseCases + .Select(testCase => WrapTestCase(testMethod, testCase, attribute, maxRetries, delayMs, retryOnAssertionFailure)) + .ToArray(); } - protected override IEnumerable CreateTestCasesForTheory( + protected override async ValueTask> CreateTestCasesForTheory( ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo theoryAttribute) + IXunitTestMethod testMethod, + ITheoryAttribute theoryAttribute) { - var maxRetries = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.MaxRetries)); - var delayMs = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); - var retryOnAssertionFailure = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.RetryOnAssertionFailure)); + var attribute = (RetryTheoryAttribute)theoryAttribute; - // Use default values if not specified - if (maxRetries == 0) - { - maxRetries = 3; - } + var maxRetries = attribute.MaxRetries; + var delayMs = attribute.DelayBetweenRetriesMs; + var retryOnAssertionFailure = attribute.RetryOnAssertionFailure; - if (delayMs == 0) - { - delayMs = 5000; - } + var baseCases = await base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute); + return baseCases + .Select(testCase => WrapTestCase(testMethod, testCase, attribute, maxRetries, delayMs, retryOnAssertionFailure)) + .ToArray(); + } + + private static RetryTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTestCase testCase, RetryTheoryAttribute attribute, int maxRetries, int delayMs, bool retryOnAssertionFailure) + { + var xunitTestCase = (XunitTestCase)testCase; - // For theories without data (will be skipped), wrap in retry - return base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute) - .Select(testCase => new RetryTestCase( - DiagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), - testMethod, - maxRetries, - delayMs, - retryOnAssertionFailure, - testCase.TestMethodArguments)); + return new RetryTestCase( + testMethod, + xunitTestCase.TestCaseDisplayName, + xunitTestCase.UniqueID, + xunitTestCase.Explicit, + xunitTestCase.SkipExceptions, + xunitTestCase.SkipReason, + xunitTestCase.SkipType, + xunitTestCase.SkipUnless, + xunitTestCase.SkipWhen, + new Dictionary>(xunitTestCase.Traits.ToDictionary(kvp => kvp.Key, kvp => new HashSet(kvp.Value))), + xunitTestCase.TestMethodArguments, + xunitTestCase.SourceFilePath, + xunitTestCase.SourceLineNumber, + xunitTestCase.Timeout == 0 ? null : xunitTestCase.Timeout, + maxRetries, + delayMs, + retryOnAssertionFailure); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/Skip.cs b/src/Microsoft.Health.Extensions.Xunit/Skip.cs new file mode 100644 index 0000000000..5e2ce9997b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/Skip.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit.Sdk; + +namespace Xunit +{ + /// + /// Provides legacy SkippableFact-compatible helpers for conditional skips. + /// + public static class Skip + { + /// + /// Skips the current test when the condition is true. + /// + /// The condition that triggers the skip. + /// The reason for the skip. + public static void If(bool condition, string reason = null) + { + if (condition) + { + throw SkipException.ForSkip(reason ?? "Skipped"); + } + } + + /// + /// Skips the current test when the condition is false. + /// + /// The condition that prevents the skip. + /// The reason for the skip. + public static void IfNot(bool condition, string reason = null) + { + if (!condition) + { + throw SkipException.ForSkip(reason ?? "Skipped"); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs new file mode 100644 index 0000000000..0cef141402 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Runtime.CompilerServices; + +namespace Xunit +{ + /// + /// Compatibility attribute for legacy SkippableFact usage (temporary shim to reduce PR review churn). + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public sealed class SkippableFactAttribute : FactAttribute + { + public SkippableFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + { + SourceFilePath = sourceFilePath; + SourceLineNumber = sourceLineNumber; + } + + internal new string SourceFilePath { get; } + + internal new int SourceLineNumber { get; } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs new file mode 100644 index 0000000000..bff37554fc --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Runtime.CompilerServices; + +namespace Xunit +{ + /// + /// Compatibility attribute for legacy SkippableTheory usage (temporary shim to reduce PR review churn). + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public sealed class SkippableTheoryAttribute : TheoryAttribute + { + public SkippableTheoryAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + { + SourceFilePath = sourceFilePath; + SourceLineNumber = sourceLineNumber; + } + + internal new string SourceFilePath { get; } + + internal new int SourceLineNumber { get; } + } +} From 492e799c3d0ab98f1b0bdae57710cc70a9691bcf Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:38:08 -0700 Subject: [PATCH 008/105] Update unit test csprojs: xunit -> xunit.v3.mtp-v2 - Swap 'xunit' PackageReference to 'xunit.v3.mtp-v2' across all 19 *UnitTests*.csproj and Microsoft.Health.Fhir.Tests.Common.csproj. - Remove 'Xunit.SkippableFact' PackageReference from Azure.UnitTests and the four *.Core.UnitTests projects (replaced by local Skip/SkippableFact types in Microsoft.Health.Extensions.Xunit). - Add ProjectReference to Microsoft.Health.Extensions.Xunit in Azure.UnitTests (needed for the local SkippableFact attribute). - Swap 'xunit.assert'/'xunit.extensibility.core' to their 'xunit.v3.*' equivalents in SchemaManager.UnitTests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Health.Fhir.R4.Api.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.Azure.UnitTests.csproj | 4 ++-- .../Microsoft.Health.Fhir.Core.UnitTests.csproj | 2 +- ...osoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.R4.Core.UnitTests.csproj | 3 +-- .../Microsoft.Health.Fhir.R4.Web.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj | 3 +-- .../Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.R5.Api.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.R5.Core.UnitTests.csproj | 3 +-- .../Microsoft.Health.Fhir.R5.Web.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj | 4 ++-- .../Microsoft.Health.Fhir.SqlServer.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj | 3 +-- .../Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj | 2 +- .../Microsoft.Health.Fhir.Tests.Common.csproj | 2 +- .../Microsoft.Health.TaskManagement.UnitTests.csproj | 2 +- 20 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj b/src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj index 756a8af417..30312d6d04 100644 --- a/src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj b/src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj index 321082960e..6e0e19a7e2 100644 --- a/src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj @@ -4,12 +4,12 @@ - + - + diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj index 5b9bc2a7af..51fc2861bb 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj @@ -14,7 +14,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj index 57eb3e5ddb..cd8ab85abd 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj @@ -2,7 +2,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj index 4363686f48..51736adb61 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj @@ -3,7 +3,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj index 7ca466d163..f0ecda85f4 100644 --- a/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj @@ -18,9 +18,8 @@ - + - diff --git a/src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj index 0c6a754cd0..9ad126019d 100644 --- a/src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj index a6c1e0949b..e831b9613c 100644 --- a/src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj index bde41ef4b6..fbd12b04b6 100644 --- a/src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj @@ -12,9 +12,8 @@ - + - diff --git a/src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj b/src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj index 50b6b8b4df..5f29c3dfd9 100644 --- a/src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj b/src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj index 534575bd3e..e64b2e51fb 100644 --- a/src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj index 912d83e5f5..4fe1f6f95e 100644 --- a/src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj @@ -11,9 +11,8 @@ - + - diff --git a/src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj b/src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj index bd10b85ab1..829b474b36 100644 --- a/src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj b/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj index 7674c947ea..8f0cc39410 100644 --- a/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj @@ -10,8 +10,8 @@ - - + + diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj index ed90df8d67..501d312522 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj @@ -1,7 +1,7 @@  - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj b/src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj index 6985b50153..5a30e00f43 100644 --- a/src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj @@ -9,7 +9,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj b/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj index 2821ce0af8..1f286e2b0c 100644 --- a/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj @@ -15,12 +15,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj b/src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj index 28437cd92a..0aeaa9dc69 100644 --- a/src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj index 5441486a6a..bf59337cb1 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj +++ b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj @@ -622,7 +622,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj b/src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj index 4aa9d4b638..6ce557dc80 100644 --- a/src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj +++ b/src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 5f7098538300569ae2e15b1962a6007f64efa59f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:39:35 -0700 Subject: [PATCH 009/105] Update integration test csprojs: xunit -> xunit.v3.mtp-v2 Swap 'xunit' PackageReference to 'xunit.v3.mtp-v2' and drop 'Xunit.SkippableFact' from the four *.Tests.Integration.csproj files (R4, R4B, R5, Stu3). The xunit.runner.json copy metadata is handled in the shared .projitems (coming in a later commit). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Health.Fhir.R4.Tests.Integration.csproj | 3 +-- .../Microsoft.Health.Fhir.R4B.Tests.Integration.csproj | 3 +-- .../Microsoft.Health.Fhir.R5.Tests.Integration.csproj | 3 +-- .../Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.Health.Fhir.R4.Tests.Integration/Microsoft.Health.Fhir.R4.Tests.Integration.csproj b/test/Microsoft.Health.Fhir.R4.Tests.Integration/Microsoft.Health.Fhir.R4.Tests.Integration.csproj index 0296252158..5ad4086fbd 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.Integration/Microsoft.Health.Fhir.R4.Tests.Integration.csproj +++ b/test/Microsoft.Health.Fhir.R4.Tests.Integration/Microsoft.Health.Fhir.R4.Tests.Integration.csproj @@ -10,12 +10,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/test/Microsoft.Health.Fhir.R4B.Tests.Integration/Microsoft.Health.Fhir.R4B.Tests.Integration.csproj b/test/Microsoft.Health.Fhir.R4B.Tests.Integration/Microsoft.Health.Fhir.R4B.Tests.Integration.csproj index c1c40ac931..cb29974b67 100644 --- a/test/Microsoft.Health.Fhir.R4B.Tests.Integration/Microsoft.Health.Fhir.R4B.Tests.Integration.csproj +++ b/test/Microsoft.Health.Fhir.R4B.Tests.Integration/Microsoft.Health.Fhir.R4B.Tests.Integration.csproj @@ -10,12 +10,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/test/Microsoft.Health.Fhir.R5.Tests.Integration/Microsoft.Health.Fhir.R5.Tests.Integration.csproj b/test/Microsoft.Health.Fhir.R5.Tests.Integration/Microsoft.Health.Fhir.R5.Tests.Integration.csproj index cd5d5b9b3c..7d33ec9902 100644 --- a/test/Microsoft.Health.Fhir.R5.Tests.Integration/Microsoft.Health.Fhir.R5.Tests.Integration.csproj +++ b/test/Microsoft.Health.Fhir.R5.Tests.Integration/Microsoft.Health.Fhir.R5.Tests.Integration.csproj @@ -10,12 +10,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj b/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj index e47fb73cf2..379fb6a92e 100644 --- a/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj +++ b/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/Microsoft.Health.Fhir.Stu3.Tests.Integration.csproj @@ -10,12 +10,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - From 8f03bdf5ac745c8f1b8557f69fca1783ab82b5cb Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:40:38 -0700 Subject: [PATCH 010/105] Update E2E test csprojs: xunit -> xunit.v3.mtp-v2 Swap 'xunit' PackageReference to 'xunit.v3.mtp-v2' and drop 'Xunit.SkippableFact' and 'System.IO.FileSystem.AccessControl' across all four *.Tests.E2E.csproj files (R4, R4B, R5, Stu3). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Health.Fhir.R4.Tests.E2E.csproj | 4 +--- .../Microsoft.Health.Fhir.R4B.Tests.E2E.csproj | 4 +--- .../Microsoft.Health.Fhir.R5.Tests.E2E.csproj | 4 +--- .../Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj index b636f46578..b63ebabc30 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj @@ -27,13 +27,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/test/Microsoft.Health.Fhir.R4B.Tests.E2E/Microsoft.Health.Fhir.R4B.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.R4B.Tests.E2E/Microsoft.Health.Fhir.R4B.Tests.E2E.csproj index 4b85049f96..f07875a543 100644 --- a/test/Microsoft.Health.Fhir.R4B.Tests.E2E/Microsoft.Health.Fhir.R4B.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.R4B.Tests.E2E/Microsoft.Health.Fhir.R4B.Tests.E2E.csproj @@ -20,13 +20,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/test/Microsoft.Health.Fhir.R5.Tests.E2E/Microsoft.Health.Fhir.R5.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.R5.Tests.E2E/Microsoft.Health.Fhir.R5.Tests.E2E.csproj index 37b6c8e25d..124773d96e 100644 --- a/test/Microsoft.Health.Fhir.R5.Tests.E2E/Microsoft.Health.Fhir.R5.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.R5.Tests.E2E/Microsoft.Health.Fhir.R5.Tests.E2E.csproj @@ -18,13 +18,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj index d255f5e5f0..31aff86417 100644 --- a/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj @@ -23,13 +23,11 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - From f5923df2b6742c9d2bb66d1526d6df30c61bf25a Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:41:14 -0700 Subject: [PATCH 011/105] Update AssemblyInfo attributes for xUnit v3 - TestFramework now uses typeof() syntax (v3 no longer takes typeName/assemblyName string pair). - AssemblyFixture moves to Xunit namespace (fully qualified to disambiguate from Microsoft.Health.Extensions.Xunit). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AssemblyInfo.cs | 4 ++-- .../AssemblyInfo.cs | 4 ++-- .../AssemblyInfo.cs | 4 ++-- .../AssemblyInfo.cs | 4 ++-- .../AssemblyInfo.cs | 4 ++-- .../AssemblyInfo.cs | 4 ++-- test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs | 6 +++--- .../AssemblyInfo.cs | 4 ++-- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/test/Microsoft.Health.Fhir.R4.Tests.Integration/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.R4.Tests.Integration/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.Integration/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.R4.Tests.Integration/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/test/Microsoft.Health.Fhir.R4B.Tests.Integration/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.R4B.Tests.Integration/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/test/Microsoft.Health.Fhir.R4B.Tests.Integration/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.R4B.Tests.Integration/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/test/Microsoft.Health.Fhir.R5.Tests.Integration/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.R5.Tests.Integration/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/test/Microsoft.Health.Fhir.R5.Tests.Integration/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.R5.Tests.Integration/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs index 5591b40d4b..3af3a48b34 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyInfo.cs @@ -8,6 +8,6 @@ using Microsoft.Health.Fhir.Tests.E2E.Rest; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] -[assembly: AssemblyFixture(typeof(TestFhirServerFactory))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(TestFhirServerFactory))] diff --git a/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/AssemblyInfo.cs b/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/AssemblyInfo.cs index ab4c81a0aa..6ae966b291 100644 --- a/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/AssemblyInfo.cs +++ b/test/Microsoft.Health.Fhir.Stu3.Tests.Integration/AssemblyInfo.cs @@ -7,5 +7,5 @@ using Microsoft.Health.Fhir.Tests.Common; using Xunit; -[assembly: TestFramework(typeName: CustomXunitTestFramework.TypeName, assemblyName: CustomXunitTestFramework.AssemblyName)] -[assembly: AssemblyFixture(typeof(SetModelInfoProviderAssemblyFixture))] +[assembly: TestFramework(typeof(CustomXunitTestFramework))] +[assembly: Xunit.AssemblyFixtureAttribute(typeof(SetModelInfoProviderAssemblyFixture))] From d26ae5de6e9bb35aeee1e473dc2b6e8d8f76f55a Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:50:34 -0700 Subject: [PATCH 012/105] Convert test fixtures to ValueTask-based IAsyncLifetime for xUnit v3 xUnit v3 changes IAsyncLifetime.InitializeAsync/DisposeAsync from Task to ValueTask. Update all test fixtures and test classes that implement IAsyncLifetime, OnInitializedAsync, or OnDisposedAsync hooks accordingly. Also remove stale 'using Xunit.Abstractions;' where ITestOutputHelper moved to the Xunit namespace. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Features/Throttling/ThrottlingMiddlewareTests.cs | 6 +++--- .../SearchParameterDefinitionManagerTests.cs | 4 ++-- .../SearchParameterSupportResolverTests.cs | 4 ++-- .../Features/Search/Expressions/IncludeRewriterTests.cs | 4 ++-- .../Rest/CreateAllFhirResourcesTestsFixture.cs | 3 ++- .../Rest/CompartmentTestFixture.cs | 5 +++-- .../Rest/Conformance/DocRefOperationTestFixture.cs | 5 +++-- .../Rest/Conformance/ExpandOperationTestFixture.cs | 5 +++-- .../Rest/EverythingOperationTestFixture.cs | 4 ++-- .../Rest/Export/ExportDataTestFixture.cs | 2 +- .../Rest/HistoryTests.cs | 4 ++-- .../Rest/HttpIntegrationTestFixture{TStartup}.cs | 8 ++++---- .../Rest/Import/ImportBasicSearchTestFixture.cs | 5 +++-- .../Rest/Import/ImportCompositeSearchTestFixture.cs | 5 +++-- .../Rest/Import/ImportDateSearchTestFixture.cs | 5 +++-- .../Rest/Import/ImportHistorySoftDeleteTestFixture.cs | 2 +- .../Rest/Import/ImportNumberSearchTestFixture.cs | 5 +++-- .../Rest/Import/ImportQuantitySearchTestFixture.cs | 5 +++-- .../Rest/Import/ImportReferenceSearchTestFixture.cs | 5 +++-- .../Rest/Import/ImportStringSearchTestFixture.cs | 5 +++-- .../Rest/Import/ImportTokenSearchTestFixture.cs | 5 +++-- .../Rest/Import/ImportUriSearchTestFixture.cs | 5 +++-- .../Rest/MemberMatchTestFixture.cs | 5 +++-- .../Rest/Metric/MetricTests.cs | 6 +++--- .../Rest/Search/CanonicalSearchTestFixture.cs | 7 ++++--- .../Rest/Search/ChainingSearchTests.cs | 4 ++-- .../Rest/Search/CompositeSearchTestFixture.cs | 4 ++-- .../Rest/Search/DateSearchTestFixture.cs | 5 +++-- .../Rest/Search/IncludeSearchTestFixture.cs | 4 ++-- .../Rest/Search/IncludesOperationTestFixture.cs | 5 +++-- .../Rest/Search/NotReferencedSearchTestFixture.cs | 3 ++- .../Rest/Search/NumberSearchTestFixture.cs | 5 +++-- .../Rest/Search/QuantitySearchTestFixture.cs | 5 +++-- .../Rest/Search/ReferenceSearchTestFixture.cs | 5 +++-- .../Rest/Search/StringSearchTestFixture.cs | 5 +++-- .../Rest/Search/TokenOverflowTests.cs | 9 +++++---- .../Rest/Search/TokenSearchTestFixture.cs | 5 +++-- .../Rest/Search/UriSearchTestFixture.cs | 5 +++-- .../Rest/TestFhirServerFactory.cs | 7 +------ .../Rest/ValidateTestFixture.cs | 5 +++-- .../SqlServerFhirResourceChangeCaptureFixture.cs | 4 ++-- .../Operations/Export/CosmosDbFhirExportTestFixture.cs | 4 ++-- .../Operations/Export/CreateExportRequestHandlerTests.cs | 8 ++++---- .../Operations/FhirOperationDataStoreExportTests.cs | 6 +++--- .../Operations/FhirOperationDataStoreReindexTests.cs | 6 +++--- .../Features/Operations/Reindex/ReindexJobTests.cs | 4 ++-- .../Search/SqlServerSearchServiceIntegrationTests.cs | 6 +++--- .../Features/Smart/SmartSearchTests.cs | 6 +++--- .../Persistence/CosmosDbFhirStorageTestsFixture.cs | 4 ++-- .../Persistence/FhirStorageTestsFixture.cs | 7 ++++--- .../Persistence/SqlServerFhirStorageTestsFixture.cs | 4 ++-- .../SmartProxy/SmartProxyTestFixture.cs | 6 +++--- 52 files changed, 141 insertions(+), 119 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Throttling/ThrottlingMiddlewareTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Throttling/ThrottlingMiddlewareTests.cs index 34bcd7ada0..7fcccddd3d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Throttling/ThrottlingMiddlewareTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Throttling/ThrottlingMiddlewareTests.cs @@ -38,7 +38,7 @@ namespace Microsoft.Health.Fhir.Shared.Api.UnitTests.Features.Throttling [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Throttling)] [Trait(Traits.Category, Categories.Web)] - public class ThrottlingMiddlewareTests : IAsyncLifetime + public class ThrottlingMiddlewareTests : IAsyncLifetime, IAsyncDisposable { private HttpContext _httpContext = new DefaultHttpContext(); private Lazy _middleware; @@ -316,8 +316,8 @@ public async Task GivenARequest_ThatResultsInRequestRateExceeded_ReturnsValidFhi return output; } - public Task InitializeAsync() => Task.CompletedTask; + public ValueTask InitializeAsync() => ValueTask.CompletedTask; - async Task IAsyncLifetime.DisposeAsync() => await _middleware.Value.DisposeAsync(); + public ValueTask DisposeAsync() => _middleware.Value.DisposeAsync(); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs index dd6bd9a702..17923a18d2 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs @@ -160,12 +160,12 @@ public SearchParameterDefinitionManagerTests() NullLogger.Instance); } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _searchParameterDefinitionManager.EnsureInitializedAsync(CancellationToken.None); } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; [Fact] public async Task GivenSupportedParams_WhenGettingSupported_ThenSupportedParamsReturned() diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs index 969ab8d328..ffb73da7fe 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs @@ -20,13 +20,13 @@ public class SearchParameterSupportResolverTests : IAsyncLifetime { private SearchParameterSupportResolver _resolver; - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { _resolver = new SearchParameterSupportResolver( await SearchParameterFixtureData.GetFhirTypedElementToSearchValueConverterManagerAsync()); } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; [Fact] public void GivenASupportedSearchParameter_WhenResolvingSupport_ThenTrueIsReturned() diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs index a5aacb6c31..3823b64d52 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs @@ -39,12 +39,12 @@ public IncludeRewriterTests(IncludeRewriterFixture fixture) _searchParameterDefinitionManager = fixture.SearchParameterDefinitionManager; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _fixture.Start(); } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; // Basic Queries with 0-2 include search parameters with all the pair combinations diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/CreateAllFhirResourcesTestsFixture.cs b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/CreateAllFhirResourcesTestsFixture.cs index 40b53d06e3..f484c6e960 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/CreateAllFhirResourcesTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/CreateAllFhirResourcesTestsFixture.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; @@ -25,7 +26,7 @@ public CreateAllFhirResourcesTestsFixture( public List ResourcesToCleanup { get; } = new List(); - protected override async Task OnDisposedAsync() + protected override async ValueTask OnDisposedAsync() { await base.OnDisposedAsync(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs index 3ab1816e4f..b5ad18ad2f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -6,6 +6,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest @@ -27,7 +28,7 @@ public CompartmentTestFixture(DataStore dataStore, Format format, TestFhirServer public Condition Condition { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Create various resources. Device = await TestFhirClient.CreateAsync(Samples.GetJsonSample("Device-d1")); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs index 50ac5ea977..5771511618 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -11,6 +11,7 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Xunit; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Conformance @@ -63,7 +64,7 @@ public IReadOnlyList GetDocumentReferences(string patientId) return new List(); } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await base.OnInitializedAsync(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs index b47bb8e40a..8cbcd1e60f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -11,6 +11,7 @@ using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Xunit; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Conformance @@ -36,7 +37,7 @@ public ExpandOperationTestFixture( public IReadOnlyList ValueSets => _valueSets; - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await base.OnInitializedAsync(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/EverythingOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/EverythingOperationTestFixture.cs index 6f71f8ce28..9c88b9e09b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/EverythingOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/EverythingOperationTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -78,7 +78,7 @@ internal async Task UpdatePatient(Patient patientToUpdate) await TestFhirClient.UpdateAsync(patientToUpdate); } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Test case #1 // Create resources that directly referenced by the Patient resource diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index 22e0ee0699..735786f297 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -77,7 +77,7 @@ internal string ExportTestFilterQueryParameters(params string[] uniqueResourceTy return $"_type={string.Join(',', uniqueResourceTypes)}&_typeFilter={typeFilterPart}&_since={TestDataInsertionTime:O}"; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await SaveTestResourcesToServer(); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs index caa67fbb10..842e725c60 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs @@ -54,12 +54,12 @@ public HistoryTests(HttpIntegrationTestFixture fixture) protected HttpIntegrationTestFixture Fixture { get; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { _createdResource = await _client.CreateByUpdateAsync(Samples.GetDefaultObservation().ToPoco()); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_createdResource?.Resource != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HttpIntegrationTestFixture{TStartup}.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HttpIntegrationTestFixture{TStartup}.cs index c836d38d28..d275f9d6d8 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HttpIntegrationTestFixture{TStartup}.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HttpIntegrationTestFixture{TStartup}.cs @@ -57,7 +57,7 @@ public HttpIntegrationTestFixture(DataStore dataStore, Format format, TestFhirSe protected internal TestFhirServer TestFhirServer { get; private set; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { Exception lastException = null; @@ -102,7 +102,7 @@ public async Task InitializeAsync() lastException); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await OnDisposedAsync(); } @@ -112,8 +112,8 @@ public string GenerateFullUrl(string relativeUrl) return $"{TestFhirServer.BaseAddress}{relativeUrl}"; } - protected virtual Task OnInitializedAsync() => Task.CompletedTask; + protected virtual ValueTask OnInitializedAsync() => ValueTask.CompletedTask; - protected virtual Task OnDisposedAsync() => Task.CompletedTask; + protected virtual ValueTask OnDisposedAsync() => ValueTask.CompletedTask; } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs index 9ed90a041f..b922f4fb0f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -46,7 +47,7 @@ public ImportBasicSearchTestFixture(DataStore dataStore, Format format, TestFhir public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs index 48059bdbc8..d8947fb014 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -44,7 +45,7 @@ public ImportCompositeSearchTestFixture(DataStore dataStore, Format format, Test public IReadOnlyDictionary DocumentReferences { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Observations = CreateResultDictionary(ObservationTestFileNames); DocumentReferences = CreateResultDictionary(DocumentReferenceTestFiles); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs index 0283a338de..6a95c4ff96 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -22,7 +23,7 @@ public ImportDateSearchTestFixture(DataStore dataStore, Format format, TestFhirS public IReadOnlyList Observations { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Observations = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportHistorySoftDeleteTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportHistorySoftDeleteTestFixture.cs index 833699ba6e..ec4260c770 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportHistorySoftDeleteTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportHistorySoftDeleteTestFixture.cs @@ -26,7 +26,7 @@ public ImportHistorySoftDeleteTestFixture(DataStore dataStore, Format format, Te public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { TestResources.Add("NewImplicitVersionId", GetNewImplicitVersionIdResources()); TestResources.Add("ImportOverExistingVersionId", GetImportOverExistingVersionIdResources()); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs index 2a6ec0f631..165fdc9cef 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -22,7 +23,7 @@ public ImportNumberSearchTestFixture(DataStore dataStore, Format format, TestFhi public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { RiskAssessments = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs index 4c9f3ac154..7dba29f12d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -22,7 +23,7 @@ public ImportQuantitySearchTestFixture(DataStore dataStore, Format format, TestF public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Observations = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs index a69e3c26a4..898afcc08e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -22,7 +23,7 @@ public ImportReferenceSearchTestFixture(DataStore dataStore, Format format, Test public IReadOnlyList Patients { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Patients = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs index 880f988519..e676de800e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -24,7 +25,7 @@ public ImportStringSearchTestFixture(DataStore dataStore, Format format, TestFhi public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Patients = await ImportTestHelper.ImportToServerAsync( TestFhirClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs index 7560ebf05a..9db6185d6c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -23,7 +24,7 @@ public ImportTokenSearchTestFixture(DataStore dataStore, Format format, TestFhir public IReadOnlyList Observations { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await TestFhirClient.CreateResourcesAsync(p => { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs index 20b63003ea..f766781da1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import @@ -22,7 +23,7 @@ public ImportUriSearchTestFixture(DataStore dataStore, Format format, TestFhirSe public string FixtureTag { get; set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { FixtureTag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs index 49c26076b2..0f71f01a31 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -9,6 +9,7 @@ using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Fhir.Web; +using System.Threading.Tasks; namespace Microsoft.Health.Fhir.Tests.E2E.Rest { @@ -22,7 +23,7 @@ public MemberMatchTestFixture(DataStore dataStore, Format format, TestFhirServer _tag = new Coding(string.Empty, Guid.NewGuid().ToString()); } - protected override async System.Threading.Tasks.Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Create various resources. Patient[] patients = await TestFhirClient.CreateResourcesAsync( diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Metric/MetricTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Metric/MetricTests.cs index e01524fe7e..12a90effd1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Metric/MetricTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Metric/MetricTests.cs @@ -38,15 +38,15 @@ public MetricTests(MetricTestFixture fixture) _metricHandler = _fixture?.MetricHandler; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { // Send an empty request to guarantee that there is a bearer token set and the call isn't recorded in the metric handler. await _client.HttpClient.SendAsync(new HttpRequestMessage(HttpMethod.Get, string.Empty)); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } [Fact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs index ebb31fc848..b9af410287 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -44,7 +45,7 @@ public CanonicalSearchTestFixture(DataStore dataStore, Format format, TestFhirSe public string ObservationProfileV2 { get; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Observation resource1 = Samples.GetDefaultObservation().ToPoco(); resource1.Meta = new Meta(); @@ -75,7 +76,7 @@ protected override async Task OnInitializedAsync() GivenObservation3 = (await TestFhirClient.CreateAsync(resource3)).Resource; } - protected override async Task OnDisposedAsync() + protected override async ValueTask OnDisposedAsync() { if (GivenObservation1 != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs index 5a9e37a36f..1d9007f7ef 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -397,7 +397,7 @@ public ClassFixture(DataStore dataStore, Format format, TestFhirServerFactory te public Device DeviceSnomedSubject { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CompositeSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CompositeSearchTestFixture.cs index 1b3a4147b3..2a1969aa2a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CompositeSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CompositeSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -46,7 +46,7 @@ public CompositeSearchTestFixture(DataStore dataStore, Format format, TestFhirSe public IReadOnlyDictionary DocumentReferences { get; private set; } - protected async override Task OnInitializedAsync() + protected async override ValueTask OnInitializedAsync() { Observations = await CreateResultDictionary(ObservationTestFileNames); DocumentReferences = await CreateResultDictionary(DocumentReferenceTestFiles); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs index a7654d4f68..d0222c3996 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -23,7 +24,7 @@ public DateSearchTestFixture(DataStore dataStore, Format format, TestFhirServerF public IReadOnlyList Observations { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Creates a unique code for searches Coding = new Coding diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTestFixture.cs index 723a1b33ba..a61e9572fb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -101,7 +101,7 @@ public IncludeSearchTestFixture(DataStore dataStore, Format format, TestFhirServ public Location LocationPartOfSelf { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs index 17e3704da3..5f94c13adf 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -13,6 +13,7 @@ using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.SqlServer.Management.Smo; using NSubstitute.Core; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -120,7 +121,7 @@ public IncludesOperationTestFixture( public string Tag => _tag; - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { await base.OnInitializedAsync(); var bundle = TagResources((Bundle)Samples.GetJsonSample(ResourceFileName).ToPoco()); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs index d60294555f..684aec0448 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs @@ -9,6 +9,7 @@ using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -30,7 +31,7 @@ public NotReferencedSearchTestFixture(DataStore dataStore, Format format, TestFh public Patient[] Patients { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Creates a unique code for searches Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs index f993576a1e..a06b58dc28 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -23,7 +24,7 @@ public NumberSearchTestFixture(DataStore dataStore, Format format, TestFhirServe public string Tag { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs index af3747cdc5..b8b910f6e9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -23,7 +24,7 @@ public QuantitySearchTestFixture(DataStore dataStore, Format format, TestFhirSer public string Tag { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Prepare the resources used for number search tests. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs index a305522e67..ffb7bf2ef7 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -23,7 +24,7 @@ public ReferenceSearchTestFixture(DataStore dataStore, Format format, TestFhirSe public string Tag { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Prepare the resources used for string search tests. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs index c58c0142c3..fdb0f41951 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -25,7 +26,7 @@ public StringSearchTestFixture(DataStore dataStore, Format format, TestFhirServe public string FixtureTag { get; } = Guid.NewGuid().ToString(); - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Patients = await TestFhirClient.CreateResourcesAsync( p => SetPatientInfo(p, "Seattle", "Smith", given: "Bea"), diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs index 78aa0a540f..2b4b41c983 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -16,7 +16,7 @@ using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -39,7 +39,7 @@ public TokenOverflowTests(HttpIntegrationTestFixture fixture, ITestOutputHelper // not used by the resources in the test must be returned. The goal is then to test if database select logic correctly returns no data (rows). private delegate string GetOneParameter(T resource, bool valid = true); - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await Task.CompletedTask; } @@ -575,6 +575,7 @@ await ExecuteAndValidateBundle( } } - public Task DisposeAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; } } + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs index b5702f3cf0..df3d9cd99f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -23,7 +24,7 @@ public TokenSearchTestFixture(DataStore dataStore, Format format, TestFhirServer public IReadOnlyList Observations { get; private set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { Tag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs index 98e2797631..66d851380c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search @@ -23,7 +24,7 @@ public UriSearchTestFixture(DataStore dataStore, Format format, TestFhirServerFa public string FixtureTag { get; set; } - protected override async Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Prepare the resources used for URI search tests. FixtureTag = Guid.NewGuid().ToString(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServerFactory.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServerFactory.cs index a50302e23f..5e589dd6fc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServerFactory.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServerFactory.cs @@ -162,12 +162,7 @@ private static string GetEnvironmentUrl(DataStore dataStore) } } - Task IAsyncLifetime.InitializeAsync() => Task.CompletedTask; - - async Task IAsyncLifetime.DisposeAsync() - { - await DisposeAsync(); - } + ValueTask IAsyncLifetime.InitializeAsync() => ValueTask.CompletedTask; public async ValueTask DisposeAsync() { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs index 06b4d9c6de..f280557df1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -8,6 +8,7 @@ using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Fhir.Web; +using System.Threading.Tasks; namespace Microsoft.Health.Fhir.Tests.E2E.Rest { @@ -18,7 +19,7 @@ public ValidateTestFixture(DataStore dataStore, Format format, TestFhirServerFac { } - protected override async System.Threading.Tasks.Task OnInitializedAsync() + protected override async ValueTask OnInitializedAsync() { // Delete all profile related resources before starting the test suite. var sd = new List() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs index 750f583b12..0936ec16b5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs @@ -47,12 +47,12 @@ public SqlServerFhirResourceChangeCaptureFixture() public SchemaInformation SchemaInformation => _sqlFixture.SchemaInformation; - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _storageFixture.InitializeAsync(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_storageFixture != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CosmosDbFhirExportTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CosmosDbFhirExportTestFixture.cs index 0bef6d498b..e7d9732992 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CosmosDbFhirExportTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CosmosDbFhirExportTestFixture.cs @@ -23,7 +23,7 @@ public CosmosDbFhirExportTestFixture() { } - public override async Task InitializeAsync() + public override async ValueTask InitializeAsync() { await base.InitializeAsync(); @@ -31,7 +31,7 @@ public override async Task InitializeAsync() await PrepareData(3333); } - public override async Task DisposeAsync() + public override async ValueTask DisposeAsync() { await CleanupTestResourceDocuments(); await base.DisposeAsync(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CreateExportRequestHandlerTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CreateExportRequestHandlerTests.cs index dd6ac7a3c9..496de643ba 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CreateExportRequestHandlerTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/CreateExportRequestHandlerTests.cs @@ -297,14 +297,14 @@ public static IEnumerable ValidateTypeFilters } } - public Task InitializeAsync() + public ValueTask InitializeAsync() { - return _fhirStorageTestHelper.DeleteAllExportJobRecordsAsync(_cancellationToken); + return new ValueTask(_fhirStorageTestHelper.DeleteAllExportJobRecordsAsync(_cancellationToken)); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } [Theory] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreExportTests.cs index c881568706..bb0604f0eb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreExportTests.cs @@ -42,15 +42,15 @@ public FhirOperationDataStoreExportTests(FhirStorageTestsFixture fixture) _testHelper = fixture.TestHelper; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _testHelper.DeleteAllExportJobRecordsAsync(); GetTestQueueClient().ClearJobs(); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } /// diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreReindexTests.cs index f998c014e8..17b77fb212 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/FhirOperationDataStoreReindexTests.cs @@ -45,7 +45,7 @@ public FhirOperationDataStoreReindexTests(FhirStorageTestsFixture fixture) _testHelper = fixture.TestHelper; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await _testHelper.DeleteAllReindexJobRecordsAsync(); await CancelActiveReindexJobIfExists(); @@ -55,9 +55,9 @@ public async Task InitializeAsync() await AssertNoReindexJobsExist(); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } private async Task AssertNoReindexJobsExist() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs index 998b489181..c5bd7c6516 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs @@ -104,7 +104,7 @@ public ReindexJobTests(FhirStorageTestsFixture fixture, ITestOutputHelper output _output = output; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { _operationsConfig.Value.Returns(new OperationsConfiguration()); _coreFeatureConfig.Value.Returns(new CoreFeatureConfiguration { SearchParameterCacheRefreshIntervalSeconds = 1 }); @@ -177,7 +177,7 @@ public async Task InitializeAsync() StartCacheUpdateTask(_backgroundCts.Token); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { // Clean up resources before finishing test class await DeleteTestResources(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Search/SqlServerSearchServiceIntegrationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Search/SqlServerSearchServiceIntegrationTests.cs index 81c1d4a1cd..7966e3f33f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Search/SqlServerSearchServiceIntegrationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Search/SqlServerSearchServiceIntegrationTests.cs @@ -45,12 +45,12 @@ public SqlServerSearchServiceIntegrationTests(FhirStorageTestsFixture fixture) _dataStore = fixture.DataStore; } - public Task InitializeAsync() => Task.CompletedTask; + public ValueTask InitializeAsync() => ValueTask.CompletedTask; - public Task DisposeAsync() + public ValueTask DisposeAsync() { // Clean up is handled by fixture disposal - return Task.CompletedTask; + return ValueTask.CompletedTask; } [Fact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs index fe7e4eff8d..1663c6e1f0 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs @@ -74,7 +74,7 @@ public SmartSearchTests(FhirStorageTestsFixture fixture) _testHelper = _fixture.TestHelper; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { if (ModelInfoProvider.Instance.Version == FhirSpecification.R4 || ModelInfoProvider.Instance.Version == FhirSpecification.R4B) @@ -154,9 +154,9 @@ public async Task InitializeAsync() } } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } [SkippableFact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 6f5a43f1c7..405945380f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -102,7 +102,7 @@ public CosmosDbFhirStorageTestsFixture() public Container Container => _container; - public virtual async Task InitializeAsync() + public virtual async ValueTask InitializeAsync() { var fhirStoredProcs = typeof(DataPlaneCollectionSetup).Assembly .GetTypes() @@ -308,7 +308,7 @@ public virtual async Task InitializeAsync() _fhirStorageTestHelper = new CosmosDbFhirStorageTestHelper(_container, queueClient); } - public virtual async Task DisposeAsync() + public virtual async ValueTask DisposeAsync() { if (_container != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs index 27db98b04b..9c67105e34 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Net.Http; using System.Threading; +using System.Threading.Tasks; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; @@ -154,7 +155,7 @@ public void Dispose() (_fixture as IDisposable)?.Dispose(); } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { if (_fixture is IAsyncLifetime asyncLifetime) { @@ -245,7 +246,7 @@ public async Task InitializeAsync() Mediator = new Mediator(services); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_fixture is IAsyncLifetime asyncLifetime) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs index 79e784f999..7918767ce5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs @@ -132,7 +132,7 @@ internal static string GetDatabaseName(string test = null) return $"{ModelInfoProvider.Version}{(test == null ? string.Empty : $"_{test}")}_{DateTimeOffset.UtcNow.ToString("s").Replace("-", string.Empty).Replace(":", string.Empty)}_{Guid.NewGuid().ToString().Replace("-", string.Empty)}"; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { var scriptProvider = new ScriptProvider(); var baseScriptProvider = new BaseScriptProvider(); @@ -322,7 +322,7 @@ public async Task InitializeAsync() await _searchParameterDefinitionManager.EnsureInitializedAsync(CancellationToken.None); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await _testHelper.DeleteDatabase(_databaseName, CancellationToken.None); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Smart/SmartProxy/SmartProxyTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Smart/SmartProxy/SmartProxyTestFixture.cs index c4a73f550b..6816578a63 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Smart/SmartProxy/SmartProxyTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Smart/SmartProxy/SmartProxyTestFixture.cs @@ -38,7 +38,7 @@ public SmartProxyTestFixture(TestFhirServerFactory testFhirServerFactory) public TestFhirClient TestFhirClient { get; private set; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { string environmentUrl = Environment.GetEnvironmentVariable($"TestEnvironmentUrl{Constants.TestEnvironmentVariableVersionSuffix}"); @@ -69,10 +69,10 @@ public async Task InitializeAsync() } } - public Task DisposeAsync() + public ValueTask DisposeAsync() { Dispose(); - return Task.CompletedTask; + return ValueTask.CompletedTask; } public void Dispose() From ae17d467a6eb8c4253e090380e6156b1cf6a04b1 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:51:36 -0700 Subject: [PATCH 013/105] Register Shared.Tests.Integration xunit.runner.json and update E2E projitems for MTP xUnit v3 MTP runner requires xunit.runner.json to be copied to the test output directory. Add the Shared.Tests.Integration xunit.runner.json and register it (plus the E2E one) in the shared .projitems with Link + CopyToPublishDirectory=Never so each downstream test project picks it up. Add v3 $schema URL. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Health.Fhir.Shared.Tests.E2E.projitems | 2 ++ .../xunit.runner.json | 1 + ...icrosoft.Health.Fhir.Shared.Tests.Integration.projitems | 7 +++++++ .../xunit.runner.json | 4 ++++ 4 files changed, 14 insertions(+) create mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems index c5bbdb2aad..c891cfa50d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems @@ -152,7 +152,9 @@ + xunit.runner.json PreserveNewest + Never \ No newline at end of file diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/xunit.runner.json b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/xunit.runner.json index 253b9806cd..553c9ea743 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/xunit.runner.json +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/xunit.runner.json @@ -1,4 +1,5 @@ { + "$schema": "https://xunit.net/schema/v3/configuration", "parallelizeAssembly": false, "parallelizeTestCollections": false } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems index 538ef138b5..9b5ae7bc08 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems @@ -58,4 +58,11 @@ + + + xunit.runner.json + PreserveNewest + Never + + \ No newline at end of file diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json new file mode 100644 index 0000000000..d113d1f854 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://xunit.net/schema/v3/configuration", + "parallelizeAssembly": true +} From adf3e04cb4220f9efa59549e0192267c2365e426 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:52:10 -0700 Subject: [PATCH 014/105] Disable parallelization for CustomQueries tests under xUnit v3 xUnit v3 parallelizes more aggressively by default. CustomQueriesUnitTests mutates shared static state (CustomQueriesHash) so add a CollectionDefinition with DisableParallelization=true and attach the test class to it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Search/CustomQueriesTestsCollection.cs | 17 +++++++++++++++++ .../Features/Search/CustomQueriesUnitTests.cs | 1 + 2 files changed, 18 insertions(+) create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesTestsCollection.cs diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesTestsCollection.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesTestsCollection.cs new file mode 100644 index 0000000000..369f10ccc7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesTestsCollection.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search +{ + /// + /// Disables parallelization for CustomQueries tests due to shared static state. + /// + [CollectionDefinition("CustomQueriesTests", DisableParallelization = true)] + public sealed class CustomQueriesTestsCollection + { + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs index 5e6066b0d8..41a2590fc3 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs @@ -16,6 +16,7 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Search)] + [Collection("CustomQueriesTests")] public class CustomQueriesUnitTests { [Fact] From cb2da1d5bca7f8877be051fede63a089d49257d8 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:52:42 -0700 Subject: [PATCH 015/105] Remove stale Xunit.Abstractions using in SqlServerImporterIntegrationTests ITestOutputHelper moved to the Xunit namespace in xUnit v3. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Persistence/SqlServerImporterIntegrationTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerImporterIntegrationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerImporterIntegrationTests.cs index 0ad744658c..f55fba545a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerImporterIntegrationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerImporterIntegrationTests.cs @@ -21,7 +21,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { From 417318176eb08f9df533d8408c939a84d6ec6e84 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:53:18 -0700 Subject: [PATCH 016/105] Publish TRX results from MTP runner in CI When using Microsoft Testing Platform (xUnit v3) the DotNetCoreCLI@2 'test' command no longer emits TRX automatically. Pass --report-trx, disable built-in test-result publishing, and add an explicit PublishTestResults@2 step. Keep failTaskOnFailedTests=true for the non-coverage path and keep it false for the coverage path until testfx#7167 is fixed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/jobs/build.yml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/build/jobs/build.yml b/build/jobs/build.yml index 649013478c..af75d062c2 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -35,8 +35,19 @@ steps: inputs: command: test projects: '**/*UnitTests/*.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3' + arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx' + publishTestResults: false + env: + platformOptions__resultDirectory: '$(Agent.TempDirectory)/unit-test-results' + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '$(Agent.TempDirectory)/unit-test-results/**/*.trx' + mergeTestResults: true testRunTitle: 'Unit Tests' + failTaskOnFailedTests: true + condition: succeededOrFailed() - ${{ if eq(parameters.codeCoverage, 'true') }}: - task: DotNetCoreCLI@2 @@ -44,10 +55,20 @@ steps: inputs: command: test projects: '**/*UnitTests/*.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"' - testRunTitle: 'Unit Tests' + arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"' + publishTestResults: false env: platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' + - task: PublishTestResults@2 + displayName: 'Publish unit test with coverage results' + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' + mergeTestResults: true + testRunTitle: 'Unit Tests' + # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed + failTaskOnFailedTests: false + condition: succeededOrFailed() - task: reportgenerator@5 displayName: 'aggregate unit test coverage' condition: succeededOrFailed() From 6ecc54ef5592ea6722f30a9c43f2f02fbcbd7966 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:54:53 -0700 Subject: [PATCH 017/105] Convert CI test filters to xUnit v3 MTP filter-query syntax The MTP runner does not accept the legacy VSTest '--filter FullyQualifiedName~X' expressions. Switch to '--filter-query' with trait expressions (DataStore + Category) for E2E and export jobs, and use '--filter-not-class'/'--filter-not-trait' for integration-test exclusions. Also pass 'publishTestResults: false' on the integration-test DotNetCoreCLI@2 tasks so the separate PublishTestResults@2 step owns TRX publishing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/jobs/e2e-tests.yml | 11 ++++++++--- build/jobs/run-cosmos-tests.yml | 4 ++-- build/jobs/run-export-tests.yml | 8 ++++---- build/jobs/run-sql-tests.yml | 4 ++-- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index e8fe2da731..7199f8f9e9 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -37,10 +37,15 @@ steps: throw "Could not find $testName.dll under $testRoot" } - $filter = "FullyQualifiedName~${{ parameters.appServiceType }}&${{ parameters.categoryFilter }}" - $args = @('--filter', $filter, '--retry-failed-tests', '3', '--report-trx') + # xUnit v3 MTP filter-query: match DataStore trait for this app service type and any extra trait filters + $categoryFilter = '${{ parameters.categoryFilter }}' + $parts = $categoryFilter -split '&' + $allParts = @("DataStore=${{ parameters.appServiceType }}") + $parts + $traitFilter = "[" + (($allParts | ForEach-Object { "($_)" }) -join '&') + "]" + $filterQuery = "/$traitFilter" + $args = @('--filter-query', $filterQuery, '--retry-failed-tests', '3', '--report-trx') - Write-Host "Running dotnet $($dll.FullName)" + Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" & dotnet $dll.FullName @args if ($LASTEXITCODE -ne 0) { diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index bd891217c3..b5fabb4007 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -67,8 +67,8 @@ jobs: inputs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' + publishTestResults: false env: 'CosmosDb__Host': $(CosmosDb--Host) 'FhirServer__ResourceManager__DataStoreResourceId': '$(DataStoreResourceId)' diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index b6bb2f468a..43ad7949c6 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -100,9 +100,9 @@ jobs: throw "Could not find $testName.dll under $testRoot" } - $args = @('--filter', 'FullyQualifiedName~CosmosDb&Category=ExportLongRunning', '--retry-failed-tests', '3', '--report-trx') + $args = @('--filter-query', '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]', '--retry-failed-tests', '3', '--report-trx') - Write-Host "Running dotnet $($dll.FullName)" + Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=CosmosDb)&(Category=ExportLongRunning)]" & dotnet $dll.FullName @args if ($LASTEXITCODE -ne 0) { @@ -240,9 +240,9 @@ jobs: throw "Could not find $testName.dll under $testRoot" } - $args = @('--filter', 'FullyQualifiedName~SqlServer&Category=ExportLongRunning', '--retry-failed-tests', '3', '--report-trx') + $args = @('--filter-query', '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]', '--retry-failed-tests', '3', '--report-trx') - Write-Host "Running dotnet $($dll.FullName)" + Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=SqlServer)&(Category=ExportLongRunning)]" & dotnet $dll.FullName @args if ($LASTEXITCODE -ne 0) { diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 89226f92ac..2af5be5d17 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -56,8 +56,8 @@ jobs: inputs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - testRunTitle: '${{ parameters.version }} SQL Integration Tests' + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' + publishTestResults: false env: 'SqlServer:ConnectionString': 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);' platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' From 1f2d5102354f211a937668476d6dc787197766b5 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 22:55:28 -0700 Subject: [PATCH 018/105] Add pre-test 'Verify xUnit config' CI step for integration jobs MTP requires xunit.runner.json to be copied into each test project's output directory. Add a PowerShell task that asserts the file is present for every integration-test project prior to test execution, and prints its contents to the build log for diagnosis. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/jobs/run-cosmos-tests.yml | 23 +++++++++++++++++++++++ build/jobs/run-sql-tests.yml | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index b5fabb4007..ecd04eb976 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -62,6 +62,29 @@ jobs: projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' + - task: PowerShell@2 + displayName: 'Verify xUnit config' + inputs: + targetType: inline + pwsh: true + script: | + $ErrorActionPreference = 'Stop' + $projects = Get-ChildItem -Path "$(Pipeline.Workspace)/source/test" -Recurse -Filter "*${{ parameters.version }}.Tests.Integration.csproj" + if (-not $projects) { + throw "No integration test projects found under $(Pipeline.Workspace)/source/test." + } + + foreach ($project in $projects) { + $outputDir = Join-Path $project.Directory.FullName "bin\$(buildConfiguration)\$(defaultBuildFramework)" + $configPath = Join-Path $outputDir "xunit.runner.json" + if (-not (Test-Path $configPath)) { + throw "xunit.runner.json not found for $($project.FullName) at $configPath" + } + + Write-Host "xUnit config for $($project.BaseName): $configPath" + Get-Content -Path $configPath | ForEach-Object { Write-Host $_ } + } + - task: DotNetCoreCLI@2 displayName: 'Run Cosmos Integration Tests with coverage' inputs: diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 2af5be5d17..9806850177 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -51,6 +51,29 @@ jobs: projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' + - task: PowerShell@2 + displayName: 'Verify xUnit config' + inputs: + targetType: inline + pwsh: true + script: | + $ErrorActionPreference = 'Stop' + $projects = Get-ChildItem -Path "$(Pipeline.Workspace)/source/test" -Recurse -Filter "*${{ parameters.version }}.Tests.Integration.csproj" + if (-not $projects) { + throw "No integration test projects found under $(Pipeline.Workspace)/source/test." + } + + foreach ($project in $projects) { + $outputDir = Join-Path $project.Directory.FullName "bin\$(buildConfiguration)\$(defaultBuildFramework)" + $configPath = Join-Path $outputDir "xunit.runner.json" + if (-not (Test-Path $configPath)) { + throw "xunit.runner.json not found for $($project.FullName) at $configPath" + } + + Write-Host "xUnit config for $($project.BaseName): $configPath" + Get-Content -Path $configPath | ForEach-Object { Write-Host $_ } + } + - task: DotNetCoreCLI@2 displayName: 'Run SQL Integration Tests with coverage' inputs: From a509b20ffa9ce1bf671e73c3b71dcdd956580eeb Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 16 Apr 2026 23:06:25 -0700 Subject: [PATCH 019/105] Fix remaining xUnit v3 compile errors in shared tests - Remove 'using Xunit.Abstractions;' (namespace removed in v3; ITestOutputHelper lives in Xunit) - Add 'using System.Threading.Tasks;' where IAsyncLifetime now returns ValueTask - Add 'using Xunit;' to files that lost the only Xunit reference via the Abstractions removal Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Extensions/XUnitLogger.cs | 2 +- .../SearchParameters/SearchConverterForAllSearchTypes.cs | 3 +-- .../Search/SearchParameters/SearchParameterComparerTests.cs | 3 +-- .../SearchParameters/SearchParameterDefinitionManagerTests.cs | 1 + .../Rest/BulkDeleteTests.cs | 1 - .../Rest/Export/ExportDataTestFixture.cs | 1 + .../Rest/Export/ExportDataTests.cs | 1 - .../Rest/Export/ExportDataValidationTests.cs | 1 - .../Rest/Export/ExportTestHelper.cs | 2 +- .../Rest/Search/CustomSearchParamTests.cs | 1 - .../Rest/Search/SearchTestsBase.cs | 1 - .../SqlServerFhirResourceChangeCaptureEnabledTests.cs | 3 +-- .../Features/Operations/Export/SqlServerExportTests.cs | 3 +-- .../Features/Operations/Reindex/ReindexJobTests.cs | 3 +-- .../Persistence/ExpiredResourceCleanupWatchdogTests.cs | 1 - .../Persistence/QueueClientTests.cs | 3 +-- .../Persistence/SqlComplexQueryTests.cs | 3 +-- .../Persistence/SqlRetryServiceTests.cs | 3 +-- .../Persistence/SqlServerCreateStatsForSmartTests.cs | 3 +-- .../Persistence/SqlServerCreateStatsTests.cs | 3 +-- .../Persistence/SqlServerFhirStorageTestsFixture.cs | 1 + .../Persistence/SqlServerSearchParameterInitializationTests.cs | 1 - .../Persistence/SqlServerSetMergeTests.cs | 3 +-- .../Persistence/SqlServerWatchdogTests.cs | 3 +-- 24 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XUnitLogger.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XUnitLogger.cs index 0bf0debf5e..19a8d60f94 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XUnitLogger.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XUnitLogger.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Health.Extensions.DependencyInjection; using NSubstitute; -using Xunit.Abstractions; +using Xunit; namespace Microsoft.Health.Fhir.Core.UnitTests.Extensions; diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs index 986b649ac3..aa85c7f840 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -20,7 +20,6 @@ using Newtonsoft.Json; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterComparerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterComparerTests.cs index 6bb613b4f1..8061b5dbd3 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterComparerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterComparerTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -20,7 +20,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Shared.Core.UnitTests.Features.Search.SearchParameters { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs index 17923a18d2..cc5f636093 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs index 983a2d0d81..44abddf919 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs @@ -26,7 +26,6 @@ using Newtonsoft.Json; using Polly; using Xunit; -using Xunit.Abstractions; using static Hl7.Fhir.Model.Encounter; using Task = System.Threading.Tasks.Task; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index 735786f297..7662bb1b02 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; +using System.Threading.Tasks; using Antlr4.Runtime.Atn; using Hl7.Fhir.Model; using MediatR; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index 8edabe954c..34a935b04e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -14,7 +14,6 @@ using Microsoft.Health.Fhir.Tests.E2E.Common; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs index 4a9d51dba8..fa60995d87 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs @@ -15,7 +15,6 @@ using Microsoft.Health.Fhir.Tests.E2E.Common; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using FhirGroup = Hl7.Fhir.Model.Group; using Task = System.Threading.Tasks.Task; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs index db1464980f..ee87fb7afb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs @@ -17,7 +17,7 @@ using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; using Microsoft.Health.Fhir.Tests.E2E.Common; using Newtonsoft.Json; -using Xunit.Abstractions; +using Xunit; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs index 6d6df267a7..de1affb6cb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs @@ -18,7 +18,6 @@ using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/SearchTestsBase.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/SearchTestsBase.cs index 09da1878f6..117a6c64f4 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/SearchTestsBase.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/SearchTestsBase.cs @@ -19,7 +19,6 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.E2E.Common; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; using static Hl7.Fhir.Model.OperationOutcome; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs index a46dd50d2b..a7c05656db 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -24,7 +24,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Features.ChangeFeed { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs index 0015288465..c0f1c69108 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -25,7 +25,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Operations.Export { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs index c5bd7c6516..e7e539913e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -49,7 +49,6 @@ using Newtonsoft.Json; using NSubstitute; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.Integration.Features.Operations.Reindex diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/ExpiredResourceCleanupWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/ExpiredResourceCleanupWatchdogTests.cs index 77cb0355fb..9dcb901980 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/ExpiredResourceCleanupWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/ExpiredResourceCleanupWatchdogTests.cs @@ -21,7 +21,6 @@ using Microsoft.Health.Test.Utilities; using Newtonsoft.Json; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueueClientTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueueClientTests.cs index 23ed758a5b..18362a4117 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueueClientTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueueClientTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -15,7 +15,6 @@ using Microsoft.Health.JobManagement; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using JobConflictException = Microsoft.Health.JobManagement.JobConflictException; using JobStatus = Microsoft.Health.JobManagement.JobStatus; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlComplexQueryTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlComplexQueryTests.cs index 3f7567ad39..37eeee9d46 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlComplexQueryTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlComplexQueryTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -24,7 +24,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute.Core; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; using static Antlr4.Runtime.Atn.SemanticContext; using Task = System.Threading.Tasks.Task; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs index 34368ad05b..5a51a6214e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -18,7 +18,6 @@ using Microsoft.Health.SqlServer; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs index 6ac26899ea..c1b6897aed 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -21,7 +21,6 @@ using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsTests.cs index 56e0b26f7e..f0474cf25d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -20,7 +20,6 @@ using Microsoft.VisualStudio.TestPlatform.Utilities; using NSubstitute.Core; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs index 7918767ce5..18237a22ec 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Numerics; using System.Threading; +using System.Threading.Tasks; using Azure.Identity; using MediatR; using Microsoft.AspNetCore.Components.Forms; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterInitializationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterInitializationTests.cs index 71a95a42b7..01491000ba 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterInitializationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterInitializationTests.cs @@ -16,7 +16,6 @@ using Microsoft.Health.SqlServer.Features.Client; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSetMergeTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSetMergeTests.cs index f059002c18..162105f45c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSetMergeTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSetMergeTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -20,7 +20,6 @@ using Microsoft.Health.SqlServer.Features.Client; using Microsoft.Health.Test.Utilities; using Xunit; -using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs index 76a32680f7..009f96cd5b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -34,7 +34,6 @@ using Microsoft.SqlServer.Dac.Model; using NSubstitute; using Xunit; -using Xunit.Abstractions; #pragma warning disable SA1116 // Split parameters should start on line after declaration From 7ed048e661aea34f6f26f886a2ba6aa511145035 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 17 Apr 2026 05:58:11 -0700 Subject: [PATCH 020/105] Honor CollectionDefinition.DisableParallelization in custom discoverer The CustomXunitTestFrameworkDiscoverer was creating FixtureArgumentSetTestCollection with disableParallelization hard-coded to false, ignoring the [CollectionDefinition(DisableParallelization = true)] attribute on test classes like ReindexJobTests. This caused long-running integration tests that share a single SQL fixture to run concurrently and hang on resource contention. Now the discoverer reads CollectionDefinitionAttribute.DisableParallelization from the test class and propagates it to the test collection created for each fixture-argument variant. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFrameworkDiscoverer.cs | 7 ++++++- .../FixtureArgumentSetTestCollection.cs | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 091e5fef50..9f3721dd7f 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -38,6 +38,11 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl return await base.FindTestsForType(testClass, discoveryOptions, callback); } + // Honor [CollectionDefinition(DisableParallelization = true)] on the test class so that classes + // explicitly opting into serial execution are not parallelized by our custom test collection. + var collectionDefinitionAttribute = testClass.Class.GetCustomAttributes(typeof(CollectionDefinitionAttribute), inherit: false).OfType().FirstOrDefault(); + bool disableParallelization = collectionDefinitionAttribute != null && collectionDefinitionAttribute.DisableParallelization; + // get the class-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) SingleFlag[][] classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attribute); @@ -82,7 +87,7 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl var testClassName = effectiveAttribute.CollectionBehavior == FixtureArgumentSetCollectionBehavior.PerClass ? testClass.Class.FullName : null; - var closedVariantTestCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection.TestAssembly, closedVariant, testClassName); + var closedVariantTestCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection.TestAssembly, closedVariant, testClassName, disableParallelization); var closedVariantTestClass = new FixtureArgumentSetTestClass(testClass.Class, closedVariantTestCollection, closedVariant, UniqueIDGenerator.ForTestClass(closedVariantTestCollection.UniqueID, testClass.Class.FullName)); closedVariantTestClass.ApplyFixtureArguments(closedVariant); var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs index 83a1f2e9dd..cae6d826fc 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs @@ -21,8 +21,8 @@ internal sealed class FixtureArgumentSetTestCollection : XunitTestCollection private IReadOnlyList _fixtureArguments; private string _testClassName; - public FixtureArgumentSetTestCollection(IXunitTestAssembly testAssembly, IReadOnlyList fixtureArguments, string testClassName = null) - : base(testAssembly, collectionDefinition: null, disableParallelization: false, displayName: string.Empty, uniqueID: string.Empty) + public FixtureArgumentSetTestCollection(IXunitTestAssembly testAssembly, IReadOnlyList fixtureArguments, string testClassName = null, bool disableParallelization = false) + : base(testAssembly, collectionDefinition: null, disableParallelization: disableParallelization, displayName: string.Empty, uniqueID: string.Empty) { EnsureArg.IsNotNull(testAssembly, nameof(testAssembly)); EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); From 195fb3fb046ada68d913d71b244e9233ac141212 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 17 Apr 2026 07:56:37 -0700 Subject: [PATCH 021/105] Disable test collection parallelization for Integration tests xunit v3 defaults parallelizeTestCollections to true. Integration tests share SQL fixtures and hang when collections run concurrently (ReindexJobTests). Match the E2E config by setting parallelizeTestCollections: false. Reverts the speculative DisableParallelization plumbing in the custom discoverer (the custom discoverer creates one collection per method, so DisableParallelization on a single collection had no effect). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFrameworkDiscoverer.cs | 7 +------ .../FixtureArgumentSetTestCollection.cs | 4 ++-- .../xunit.runner.json | 3 ++- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 9f3721dd7f..091e5fef50 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -38,11 +38,6 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl return await base.FindTestsForType(testClass, discoveryOptions, callback); } - // Honor [CollectionDefinition(DisableParallelization = true)] on the test class so that classes - // explicitly opting into serial execution are not parallelized by our custom test collection. - var collectionDefinitionAttribute = testClass.Class.GetCustomAttributes(typeof(CollectionDefinitionAttribute), inherit: false).OfType().FirstOrDefault(); - bool disableParallelization = collectionDefinitionAttribute != null && collectionDefinitionAttribute.DisableParallelization; - // get the class-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) SingleFlag[][] classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attribute); @@ -87,7 +82,7 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl var testClassName = effectiveAttribute.CollectionBehavior == FixtureArgumentSetCollectionBehavior.PerClass ? testClass.Class.FullName : null; - var closedVariantTestCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection.TestAssembly, closedVariant, testClassName, disableParallelization); + var closedVariantTestCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection.TestAssembly, closedVariant, testClassName); var closedVariantTestClass = new FixtureArgumentSetTestClass(testClass.Class, closedVariantTestCollection, closedVariant, UniqueIDGenerator.ForTestClass(closedVariantTestCollection.UniqueID, testClass.Class.FullName)); closedVariantTestClass.ApplyFixtureArguments(closedVariant); var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs index cae6d826fc..83a1f2e9dd 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs @@ -21,8 +21,8 @@ internal sealed class FixtureArgumentSetTestCollection : XunitTestCollection private IReadOnlyList _fixtureArguments; private string _testClassName; - public FixtureArgumentSetTestCollection(IXunitTestAssembly testAssembly, IReadOnlyList fixtureArguments, string testClassName = null, bool disableParallelization = false) - : base(testAssembly, collectionDefinition: null, disableParallelization: disableParallelization, displayName: string.Empty, uniqueID: string.Empty) + public FixtureArgumentSetTestCollection(IXunitTestAssembly testAssembly, IReadOnlyList fixtureArguments, string testClassName = null) + : base(testAssembly, collectionDefinition: null, disableParallelization: false, displayName: string.Empty, uniqueID: string.Empty) { EnsureArg.IsNotNull(testAssembly, nameof(testAssembly)); EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json index d113d1f854..6161434f4d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json @@ -1,4 +1,5 @@ { "$schema": "https://xunit.net/schema/v3/configuration", - "parallelizeAssembly": true + "parallelizeAssembly": true, + "parallelizeTestCollections": false } From 52352fd65320b7e8441b9df62f7805f7a30ae913 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 17 Apr 2026 09:45:14 -0700 Subject: [PATCH 022/105] Fix test parallelism: share collection per (class, variant) to match xunit v2 behavior In xUnit v2, all methods of a test class belong to a single test collection and therefore run serially with a shared IClassFixture lifetime. The v3 rewrite of CustomXunitTestFrameworkDiscoverer accidentally created a new FixtureArgumentSetTestCollection for every (method, variant) combination, which caused xUnit v3's scheduler to run methods of the same class in parallel. This broke tests like ReindexJobTests which rely on per-class serialization and shared fixture state, causing intermittent hangs on the SQL integration test pipeline stages. Cache collection/class instances per (class, variant) inside FindTestsForType so methods of the same test class + variant share the same collection instance, matching v2 semantics. Revert the over-broad parallelizeTestCollections=false workaround from xunit.runner.json; cross-class parallelism is restored. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFrameworkDiscoverer.cs | 25 ++++++++++++++++--- .../xunit.runner.json | 2 +- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 091e5fef50..3f19a91f5e 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -44,6 +44,12 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl // convert these to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) SingleFlag[][] classLevelClosedParameterSets = CartesianProduct(classLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + // Cache collections (and classes) per variant within a single test class so that all methods of the same class + variant + // share the same test collection instance. This matches the xunit v2 behavior where methods of a test class belong to + // a single collection and therefore run serially (and share IClassFixture lifetime). Without this caching, xunit v3 + // schedules each method as its own collection and runs them in parallel, breaking fixture/state sharing assumptions. + var collectionCache = new Dictionary(StringComparer.Ordinal); + foreach (var method in testClass.Methods) { var fixtureParameterAttribute = method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; @@ -82,10 +88,21 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl var testClassName = effectiveAttribute.CollectionBehavior == FixtureArgumentSetCollectionBehavior.PerClass ? testClass.Class.FullName : null; - var closedVariantTestCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection.TestAssembly, closedVariant, testClassName); - var closedVariantTestClass = new FixtureArgumentSetTestClass(testClass.Class, closedVariantTestCollection, closedVariant, UniqueIDGenerator.ForTestClass(closedVariantTestCollection.UniqueID, testClass.Class.FullName)); - closedVariantTestClass.ApplyFixtureArguments(closedVariant); - var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); + + // Key the cache by the collection display identity (variant + optional class scoping). Methods that + // resolve to the same key share one collection instance. + var variantKey = (testClassName ?? string.Empty) + "|" + string.Join(",", closedVariant.Select(v => v.EnumValue)); + + if (!collectionCache.TryGetValue(variantKey, out var cached)) + { + var newCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection.TestAssembly, closedVariant, testClassName); + var newClass = new FixtureArgumentSetTestClass(testClass.Class, newCollection, closedVariant, UniqueIDGenerator.ForTestClass(newCollection.UniqueID, testClass.Class.FullName)); + newClass.ApplyFixtureArguments(closedVariant); + cached = (newCollection, newClass); + collectionCache[variantKey] = cached; + } + + var closedVariantTestMethod = new FixtureArgumentSetTestMethod(cached.Class, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(cached.Class.UniqueID, method.Name)); closedVariantTestMethod.UpdateArgumentsFromMethod(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json index 6161434f4d..bb9a4a99ec 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/xunit.runner.json @@ -1,5 +1,5 @@ { "$schema": "https://xunit.net/schema/v3/configuration", "parallelizeAssembly": true, - "parallelizeTestCollections": false + "parallelizeTestCollections": true } From bddbf25be7a31922d32e608d8b20159134fb205d Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 17 Apr 2026 11:54:25 -0700 Subject: [PATCH 023/105] Scope fixture-argument-set test collections per class to restore parallelism In xunit v3 test collections are the unit of parallelism: methods within a collection run serially, collections run in parallel. The previous discoverer used SharedPerVariant as the default, which collapsed every class with the same fixture argument variant (for example [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)]) into one collection. All SqlServer integration tests across all classes then ran serially, causing R4/R4B SQL test stages to exceed the 60 minute timeout. Always scope the collection identity to the test class so each (class, variant) pair is its own collection. This matches xunit v2 behavior: one IClassFixture per class, methods within a class run serially, and classes run in parallel. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFrameworkDiscoverer.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 3f19a91f5e..f7a84be3d2 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -85,13 +85,17 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl foreach (SingleFlag[] closedVariant in closedSets) { - var testClassName = effectiveAttribute.CollectionBehavior == FixtureArgumentSetCollectionBehavior.PerClass - ? testClass.Class.FullName - : null; - - // Key the cache by the collection display identity (variant + optional class scoping). Methods that + // Always scope the collection to the test class. In xunit v3 test collections are the unit of + // parallelism (methods within a collection run serially, collections run in parallel). If multiple + // classes share a collection (as they would under SharedPerVariant), every test across those classes + // is forced to run serially, which causes massive slowdowns in large integration test runs. + // Scoping per-class preserves xunit v2 behavior: one IClassFixture per class, methods within a + // class run serially, classes run in parallel. + var testClassName = testClass.Class.FullName; + + // Key the cache by the collection display identity (variant + class scoping). Methods that // resolve to the same key share one collection instance. - var variantKey = (testClassName ?? string.Empty) + "|" + string.Join(",", closedVariant.Select(v => v.EnumValue)); + var variantKey = testClassName + "|" + string.Join(",", closedVariant.Select(v => v.EnumValue)); if (!collectionCache.TryGetValue(variantKey, out var cached)) { From d18d66e4e73b745dc944dd45516b630d2c8a5ea1 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 18 Apr 2026 01:36:14 -0700 Subject: [PATCH 024/105] Align unit test publish to MTP retry workaround (testfx#7167) The Linux unit-test 'Publish unit test results' task was the only publish step still using failTaskOnFailedTests: true. MTP's --retry-failed-tests records the original (pre-retry) failure in the trx, so a retried-and-passed test still failed the publish step even though dotnet test succeeded. Match the workaround used by every other PublishTestResults task in the repo (coverage, e2e, sql, cosmos, export). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/jobs/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/jobs/build.yml b/build/jobs/build.yml index af75d062c2..29d2f4fe77 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -46,7 +46,8 @@ steps: testResultsFiles: '$(Agent.TempDirectory)/unit-test-results/**/*.trx' mergeTestResults: true testRunTitle: 'Unit Tests' - failTaskOnFailedTests: true + # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed + failTaskOnFailedTests: false condition: succeededOrFailed() - ${{ if eq(parameters.codeCoverage, 'true') }}: From adebd461871167a37fa037b70a5abc97cf330ccb Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 18 Apr 2026 23:28:34 -0700 Subject: [PATCH 025/105] Fix xUnit3 migration review issues Address correctness and minimality issues from PR review comments for xUnit3 migration. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/jobs/run-cosmos-tests.yml | 2 +- build/jobs/run-sql-tests.yml | 2 +- .../CustomXunitTestFrameworkDiscoverer.cs | 44 +++++++++++---- .../CustomXunitTestFrameworkExecutor.cs | 41 +++++++++++--- .../FixtureArgumentSetTestClass.cs | 2 +- .../FixtureArgumentSetTestCollection.cs | 56 +++++++------------ .../FixtureArgumentSetsAttribute.cs | 22 -------- 7 files changed, 89 insertions(+), 80 deletions(-) diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index ecd04eb976..af202cfed1 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -75,7 +75,7 @@ jobs: } foreach ($project in $projects) { - $outputDir = Join-Path $project.Directory.FullName "bin\$(buildConfiguration)\$(defaultBuildFramework)" + $outputDir = [System.IO.Path]::Combine($project.Directory.FullName, "bin", "$(buildConfiguration)", "$(defaultBuildFramework)") $configPath = Join-Path $outputDir "xunit.runner.json" if (-not (Test-Path $configPath)) { throw "xunit.runner.json not found for $($project.FullName) at $configPath" diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 9806850177..ee2901e12b 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -64,7 +64,7 @@ jobs: } foreach ($project in $projects) { - $outputDir = Join-Path $project.Directory.FullName "bin\$(buildConfiguration)\$(defaultBuildFramework)" + $outputDir = [System.IO.Path]::Combine($project.Directory.FullName, "bin", "$(buildConfiguration)", "$(defaultBuildFramework)") $configPath = Join-Path $outputDir "xunit.runner.json" if (-not (Test-Path $configPath)) { throw "xunit.runner.json not found for $($project.FullName) at $configPath" diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index f7a84be3d2..6ad33ee100 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -32,17 +32,26 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl EnsureArg.IsNotNull(discoveryOptions, nameof(discoveryOptions)); var attribute = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; + var methodAttributes = testClass.Methods.ToDictionary( + method => method, + method => method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute); - if (attribute == null) + if (attribute == null && methodAttributes.Values.All(value => value == null)) { return await base.FindTestsForType(testClass, discoveryOptions, callback); } - // get the class-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) - SingleFlag[][] classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attribute); + SingleFlag[][] classLevelOpenParameterSets = Array.Empty(); + SingleFlag[][] classLevelClosedParameterSets = Array.Empty(); - // convert these to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) - SingleFlag[][] classLevelClosedParameterSets = CartesianProduct(classLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + if (attribute != null) + { + // get the class-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) + classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attribute); + + // convert these to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) + classLevelClosedParameterSets = CartesianProduct(classLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + } // Cache collections (and classes) per variant within a single test class so that all methods of the same class + variant // share the same test collection instance. This matches the xunit v2 behavior where methods of a test class belong to @@ -52,12 +61,28 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl foreach (var method in testClass.Methods) { - var fixtureParameterAttribute = method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; - var effectiveAttribute = fixtureParameterAttribute ?? attribute; + var fixtureParameterAttribute = methodAttributes[method]; + + if (attribute == null && fixtureParameterAttribute == null) + { + var passthroughTestMethod = new XunitTestMethod(testClass, method, Array.Empty(), uniqueID: UniqueIDGenerator.ForTestMethod(testClass.UniqueID, method.Name)); + if (!await FindTestsForMethod(passthroughTestMethod, discoveryOptions, callback)) + { + return false; + } + + continue; + } SingleFlag[][] closedSets = classLevelClosedParameterSets; - if (fixtureParameterAttribute != null) + if (attribute == null) + { + // Method-level parameter sets with no class-level fallback. + SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); + closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + } + else if (fixtureParameterAttribute != null) { // get the method-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); @@ -99,9 +124,8 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl if (!collectionCache.TryGetValue(variantKey, out var cached)) { - var newCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection.TestAssembly, closedVariant, testClassName); + var newCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection, closedVariant, testClassName); var newClass = new FixtureArgumentSetTestClass(testClass.Class, newCollection, closedVariant, UniqueIDGenerator.ForTestClass(newCollection.UniqueID, testClass.Class.FullName)); - newClass.ApplyFixtureArguments(closedVariant); cached = (newCollection, newClass); collectionCache[variantKey] = cached; } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index 060ef2bca9..7a79b107ed 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -28,7 +28,7 @@ public CustomXunitTestFrameworkExecutor(Assembly assembly) public override async ValueTask RunTestCases(IReadOnlyCollection testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions, CancellationToken cancellationToken) { var runner = new FixtureArgumentSetAssemblyRunner(); - await runner.Run((IXunitTestAssembly)TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); + await runner.Run(TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); } private sealed class FixtureArgumentSetTestAssembly : XunitTestAssembly @@ -63,7 +63,12 @@ public FixtureArgumentSetTestAssembly() .ToArray(); #pragma warning restore CS0618 - AssemblyFixtureTypesField?.SetValue(this, new Lazy>(() => _assemblyFixtureTypes)); + if (AssemblyFixtureTypesField == null) + { + throw new InvalidOperationException("Unable to initialize assembly fixtures because XunitTestAssembly.assemblyFixtureTypes was not found."); + } + + AssemblyFixtureTypesField.SetValue(this, new Lazy>(() => _assemblyFixtureTypes)); } return _assemblyFixtureTypes; @@ -101,10 +106,33 @@ private sealed class FixtureArgumentSetClassRunner : XunitTestClassRunner private static readonly FieldInfo ParentMappingManagerField = typeof(FixtureMappingManager) .GetField("parentMappingManager", BindingFlags.Instance | BindingFlags.NonPublic); - protected override ValueTask OnTestClassStarting(XunitTestClassRunnerContext context) + private ExecutionContext _executionContext; + + protected override async ValueTask OnTestClassStarting(XunitTestClassRunnerContext context) { InjectFixtureArguments(context); - return base.OnTestClassStarting(context); + var result = await base.OnTestClassStarting(context); + _executionContext = ExecutionContext.Capture(); + return result; + } + + protected override async ValueTask OnTestClassFinished(XunitTestClassRunnerContext context, RunSummary summary) + { + _executionContext?.Dispose(); + _executionContext = null; + return await base.OnTestClassFinished(context, summary); + } + + protected override ValueTask RunTestMethod(XunitTestClassRunnerContext context, IXunitTestMethod testMethod, IReadOnlyCollection testCases, object[] constructorArguments) + { + if (_executionContext == null) + { + return base.RunTestMethod(context, testMethod, testCases, constructorArguments); + } + + ValueTask summary = default; + ExecutionContext.Run(_executionContext, _ => summary = base.RunTestMethod(context, testMethod, testCases, constructorArguments), null); + return summary; } protected override ValueTask GetConstructorArgument(XunitTestClassRunnerContext context, ConstructorInfo constructor, int index, ParameterInfo parameter) @@ -259,10 +287,7 @@ private static Dictionary> GetTraits(IReadOn continue; } - if (testCase is ITestCaseMetadata metadata) - { - MergeTraits(traits, metadata.Traits); - } + MergeTraits(traits, ((ITestCaseMetadata)testCase).Traits); } return traits.ToDictionary( diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs index 4fc13b2293..698fb56f6d 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs @@ -16,7 +16,7 @@ namespace Microsoft.Health.Extensions.Xunit internal sealed class FixtureArgumentSetTestClass : XunitTestClass { private static readonly FieldInfo UniqueIdField = typeof(XunitTestClass).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic); - private Type _testClassType; + private readonly Type _testClassType; private IReadOnlyList _fixtureArguments; public FixtureArgumentSetTestClass(Type testClassType, IXunitTestCollection testCollection, IReadOnlyList fixtureArguments, string uniqueId) diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs index 83a1f2e9dd..a755aef50c 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Reflection; using EnsureThat; using Xunit.Sdk; using Xunit.v3; @@ -15,59 +14,42 @@ namespace Microsoft.Health.Extensions.Xunit { internal sealed class FixtureArgumentSetTestCollection : XunitTestCollection { - private static readonly FieldInfo TestCollectionDisplayNameField = typeof(XunitTestCollection).GetField("testCollectionDisplayName", BindingFlags.Instance | BindingFlags.NonPublic); - private static readonly FieldInfo UniqueIdField = typeof(XunitTestCollection).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic); - - private IReadOnlyList _fixtureArguments; - private string _testClassName; - - public FixtureArgumentSetTestCollection(IXunitTestAssembly testAssembly, IReadOnlyList fixtureArguments, string testClassName = null) - : base(testAssembly, collectionDefinition: null, disableParallelization: false, displayName: string.Empty, uniqueID: string.Empty) + public FixtureArgumentSetTestCollection(IXunitTestCollection sourceCollection, IReadOnlyList fixtureArguments, string testClassName = null) + : base( + EnsureArg.IsNotNull(sourceCollection, nameof(sourceCollection)).TestAssembly, + sourceCollection.CollectionDefinition, + sourceCollection.DisableParallelization, + BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments, testClassName), + uniqueID: BuildUniqueId(sourceCollection, fixtureArguments, testClassName)) { - EnsureArg.IsNotNull(testAssembly, nameof(testAssembly)); EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - - _fixtureArguments = fixtureArguments; - _testClassName = testClassName; - - UpdateDisplayAndUniqueId(testAssembly.UniqueID); } #pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes public FixtureArgumentSetTestCollection() { - _fixtureArguments = Array.Empty(); } #pragma warning restore CS0618 - internal void ApplyFixtureArguments(IReadOnlyList fixtureArguments) + private static string BuildDisplayName(string baseDisplayName, IReadOnlyList fixtureArguments, string testClassName) { - if (fixtureArguments == null) + EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); + + if (fixtureArguments.Count == 0) { - return; + return string.IsNullOrEmpty(testClassName) ? baseDisplayName : testClassName; } - _fixtureArguments = fixtureArguments; - UpdateDisplayAndUniqueId(TestAssembly.UniqueID); + var argsLabel = string.Join(", ", fixtureArguments.Select(v => v.EnumValue)); + return string.IsNullOrEmpty(testClassName) + ? $"{baseDisplayName}({argsLabel})" + : $"{testClassName}({argsLabel})"; } - private void UpdateDisplayAndUniqueId(string assemblyUniqueId) + private static string BuildUniqueId(IXunitTestCollection sourceCollection, IReadOnlyList fixtureArguments, string testClassName) { - if (_fixtureArguments.Count == 0) - { - return; - } - - // Collection display name is either the variant label (shared-per-variant) or - // test class + variant (per-class), depending on the caller's choice. -#pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists - var argsLabel = string.Join(", ", _fixtureArguments.Select(v => v.EnumValue)); - var displayName = string.IsNullOrEmpty(_testClassName) - ? $"{base.TestCollectionDisplayName}({argsLabel})" - : $"{_testClassName}({argsLabel})"; - TestCollectionDisplayNameField?.SetValue(this, displayName); - UniqueIdField?.SetValue(this, UniqueIDGenerator.ForTestCollection(assemblyUniqueId, displayName, base.TestCollectionClassName)); -#pragma warning restore SA1100 + var displayName = BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments, testClassName); + return UniqueIDGenerator.ForTestCollection(sourceCollection.TestAssembly.UniqueID, displayName, sourceCollection.TestCollectionClassName); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs index 9133fa16b4..42845e289f 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs @@ -9,22 +9,6 @@ namespace Microsoft.Health.Extensions.Xunit { - /// - /// Controls how test collections are grouped for fixture argument variants. - /// - public enum FixtureArgumentSetCollectionBehavior - { - /// - /// All test classes with the same fixture argument variant share a collection. - /// - SharedPerVariant, - - /// - /// Each test class gets its own collection for a given fixture argument variant. - /// - PerClass, - } - /// /// Derive from this attribute to declare combinations of argument values that a class fixture's constructor should be called with. /// Arguments are required to be flags enums. @@ -38,14 +22,8 @@ protected FixtureArgumentSetsAttribute(params Enum[] argumentSets) { EnsureArg.IsNotNull(argumentSets, nameof(argumentSets)); _argumentSets = argumentSets; - CollectionBehavior = FixtureArgumentSetCollectionBehavior.SharedPerVariant; } - /// - /// Gets the collection grouping behavior for fixture argument variants. - /// - public FixtureArgumentSetCollectionBehavior CollectionBehavior { get; protected set; } - internal IReadOnlyList GetArgumentSets() { return _argumentSets; From bf4d0c12ff5968ec7fcf7fa433d2bf6143f939d5 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 28 Apr 2026 15:15:47 -0700 Subject: [PATCH 026/105] Address CodeQL review suggestions Refactor fixture argument handling loops to use explicit LINQ Select and Where calls as requested by the PR review comments.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFrameworkExecutor.cs | 56 ++++++++----------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index 7a79b107ed..a97cdeb3ad 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -142,13 +142,13 @@ protected override ValueTask GetConstructorArgument(XunitTestClassRunner var fixtureArguments = fixtureTestClass.GetFixtureArguments(); if (fixtureArguments.Count > 0) { - foreach (var argument in fixtureArguments) + var enumValue = fixtureArguments + .Select(argument => argument.EnumValue) + .FirstOrDefault(value => value != null && parameter.ParameterType == value.GetType()); + + if (enumValue != null) { - var enumValue = argument.EnumValue; - if (enumValue != null && parameter.ParameterType == enumValue.GetType()) - { - return new ValueTask(enumValue); - } + return new ValueTask(enumValue); } } } @@ -204,44 +204,38 @@ private static void InjectFixtureArguments(XunitTestClassRunnerContext context) continue; } - foreach (var parameter in constructor.GetParameters()) + foreach (var parameterType in constructor.GetParameters() + .Select(parameter => parameter.ParameterType) + .Where(parameterType => parameterType.IsEnum)) { - if (parameter.ParameterType.IsEnum) - { - fixtureParameterTypes.Add(parameter.ParameterType); - } + fixtureParameterTypes.Add(parameterType); } } } var resolvedArguments = new Dictionary(); - foreach (var enumValue in fixtureArguments) + foreach (var argument in fixtureArguments + .Select(enumValue => (EnumValue: enumValue, ArgumentType: enumValue.GetType()))) { - var argumentType = enumValue.GetType(); - - foreach (var parameterType in fixtureParameterTypes) + foreach (var parameterType in fixtureParameterTypes + .Where(parameterType => string.Equals(parameterType.FullName, argument.ArgumentType.FullName, StringComparison.Ordinal) + && !resolvedArguments.ContainsKey(parameterType))) { - if (string.Equals(parameterType.FullName, argumentType.FullName, StringComparison.Ordinal) - && !resolvedArguments.ContainsKey(parameterType)) - { - resolvedArguments[parameterType] = parameterType == argumentType - ? enumValue - : Enum.ToObject(parameterType, Convert.ToInt64(enumValue)); - } + resolvedArguments[parameterType] = parameterType == argument.ArgumentType + ? argument.EnumValue + : Enum.ToObject(parameterType, Convert.ToInt64(argument.EnumValue)); } } if (resolvedArguments.Count == 0 && fixtureParameterTypes.Count > 0) { var traits = GetTraits(context.TestCases); - foreach (var parameterType in fixtureParameterTypes) + foreach (var parameterType in fixtureParameterTypes + .Where(parameterType => traits.TryGetValue(parameterType.Name, out var values) && values.Count > 0)) { var traitKey = parameterType.Name; - if (!traits.TryGetValue(traitKey, out var values) || values.Count == 0) - { - continue; - } + var values = traits[traitKey]; if (values.Count > 1) { @@ -263,12 +257,10 @@ private static void InjectFixtureArguments(XunitTestClassRunnerContext context) } } - foreach (var resolvedArgument in resolvedArguments) + foreach (var resolvedArgument in resolvedArguments + .Where(resolvedArgument => !cache.ContainsKey(resolvedArgument.Key))) { - if (!cache.ContainsKey(resolvedArgument.Key)) - { - cache[resolvedArgument.Key] = resolvedArgument.Value; - } + cache[resolvedArgument.Key] = resolvedArgument.Value; } } From 840621e3f7fb25ee1f2e0bb99dfda41c4c010dfb Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 27 May 2026 13:05:33 -0700 Subject: [PATCH 027/105] Fix xUnit v3 test compile failures Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Rest/BulkDeleteTests.cs | 1 - .../Rest/Reindex/ReindexTests.cs | 6 +++--- .../Persistence/QueryPlanReuseCheckerTests.cs | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs index 5da0503b8e..2b1ae4ae49 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BulkDeleteTests.cs @@ -26,7 +26,6 @@ using Microsoft.Health.Test.Utilities; using Newtonsoft.Json; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; using static Hl7.Fhir.Model.Encounter; using Task = System.Threading.Tasks.Task; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 809c065bcb..4cc6f9987b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -44,7 +44,7 @@ public ReindexTests(HttpIntegrationTestFixture fixture, ITestOutputHelper output _output = output; } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { // delete leftover resources from previous test runs // this guarantees that we start from clean state even on previous hard failures when we cannot rely on finally @@ -62,9 +62,9 @@ public async Task InitializeAsync() _output.WriteLine($"ReindexTests.InitializeAsync: Completed. Elapsed={(int)sw.Elapsed.TotalMilliseconds} msec."); } - public Task DisposeAsync() + public ValueTask DisposeAsync() { - return Task.CompletedTask; + return ValueTask.CompletedTask; } [Fact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryPlanReuseCheckerTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryPlanReuseCheckerTests.cs index ccfafcc782..be2f22c6ad 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryPlanReuseCheckerTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryPlanReuseCheckerTests.cs @@ -21,7 +21,6 @@ using Microsoft.Health.Test.Utilities; using Microsoft.SqlServer.Management.Smo; using Xunit; -using Xunit.Abstractions; using SearchParamType = Microsoft.Health.Fhir.ValueSets.SearchParamType; using SortOrder = Microsoft.Health.Fhir.Core.Features.Search.SortOrder; using Task = System.Threading.Tasks.Task; From 7e5719a944e2335e4a7091fee6a288c4b8d7b6bd Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 27 May 2026 20:23:47 -0700 Subject: [PATCH 028/105] Run E2E tests serially under xUnit v3 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/jobs/e2e-tests.yml | 2 +- build/jobs/run-export-tests.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index 7199f8f9e9..d01c2e23f6 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -43,7 +43,7 @@ steps: $allParts = @("DataStore=${{ parameters.appServiceType }}") + $parts $traitFilter = "[" + (($allParts | ForEach-Object { "($_)" }) -join '&') + "]" $filterQuery = "/$traitFilter" - $args = @('--filter-query', $filterQuery, '--retry-failed-tests', '3', '--report-trx') + $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx') Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" & dotnet $dll.FullName @args diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index 43ad7949c6..097d4e23b7 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -100,7 +100,7 @@ jobs: throw "Could not find $testName.dll under $testRoot" } - $args = @('--filter-query', '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]', '--retry-failed-tests', '3', '--report-trx') + $args = @('--filter-query', '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx') Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=CosmosDb)&(Category=ExportLongRunning)]" & dotnet $dll.FullName @args @@ -240,7 +240,7 @@ jobs: throw "Could not find $testName.dll under $testRoot" } - $args = @('--filter-query', '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]', '--retry-failed-tests', '3', '--report-trx') + $args = @('--filter-query', '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx') Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=SqlServer)&(Category=ExportLongRunning)]" & dotnet $dll.FullName @args From 7008e8f1dfd6217bb16e34bcba19bbb6fa566c45 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 28 May 2026 09:33:04 -0700 Subject: [PATCH 029/105] Address xUnit migration review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFrameworkDiscoverer.cs | 64 +++++++++++-------- .../CustomXunitTestFrameworkExecutor.cs | 36 +++++++---- .../FixtureArgumentSetTestCollection.cs | 7 +- .../RetryFactAttribute.cs | 7 +- .../RetryTheoryAttribute.cs | 7 +- .../RetryTheoryDiscoverer.cs | 7 +- .../SkippableFactAttribute.cs | 7 +- .../SkippableTheoryAttribute.cs | 7 +- .../XunitAttributeSourceLocationTests.cs | 47 ++++++++++++++ 9 files changed, 123 insertions(+), 66 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XunitAttributeSourceLocationTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 6ad33ee100..92df5ece95 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -20,6 +21,9 @@ namespace Microsoft.Health.Extensions.Xunit /// internal sealed class CustomXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer { + private readonly ConcurrentDictionary _variantCollectionCache = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _variantClassCache = new(StringComparer.Ordinal); + public CustomXunitTestFrameworkDiscoverer(Assembly assembly, IXunitTestCollectionFactory collectionFactory = null) : base(new XunitTestAssembly(assembly, configFileName: null, assembly.GetName().Version, UniqueIDGenerator.ForAssembly(assembly.Location, null)), collectionFactory) { @@ -53,12 +57,6 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl classLevelClosedParameterSets = CartesianProduct(classLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); } - // Cache collections (and classes) per variant within a single test class so that all methods of the same class + variant - // share the same test collection instance. This matches the xunit v2 behavior where methods of a test class belong to - // a single collection and therefore run serially (and share IClassFixture lifetime). Without this caching, xunit v3 - // schedules each method as its own collection and runs them in parallel, breaking fixture/state sharing assumptions. - var collectionCache = new Dictionary(StringComparer.Ordinal); - foreach (var method in testClass.Methods) { var fixtureParameterAttribute = methodAttributes[method]; @@ -110,27 +108,25 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl foreach (SingleFlag[] closedVariant in closedSets) { - // Always scope the collection to the test class. In xunit v3 test collections are the unit of - // parallelism (methods within a collection run serially, collections run in parallel). If multiple - // classes share a collection (as they would under SharedPerVariant), every test across those classes - // is forced to run serially, which causes massive slowdowns in large integration test runs. - // Scoping per-class preserves xunit v2 behavior: one IClassFixture per class, methods within a - // class run serially, classes run in parallel. - var testClassName = testClass.Class.FullName; - - // Key the cache by the collection display identity (variant + class scoping). Methods that - // resolve to the same key share one collection instance. - var variantKey = testClassName + "|" + string.Join(",", closedVariant.Select(v => v.EnumValue)); - - if (!collectionCache.TryGetValue(variantKey, out var cached)) - { - var newCollection = new FixtureArgumentSetTestCollection(testClass.TestCollection, closedVariant, testClassName); - var newClass = new FixtureArgumentSetTestClass(testClass.Class, newCollection, closedVariant, UniqueIDGenerator.ForTestClass(newCollection.UniqueID, testClass.Class.FullName)); - cached = (newCollection, newClass); - collectionCache[variantKey] = cached; - } - - var closedVariantTestMethod = new FixtureArgumentSetTestMethod(cached.Class, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(cached.Class.UniqueID, method.Name)); + // Key variant collections by the source xUnit collection identity. Default xUnit collections are + // already per-class, while explicit [Collection] groups share one source UniqueID. Preserving that + // identity keeps explicit collection serialization intact without forcing unrelated classes to run + // serially. + var variantKey = BuildVariantCollectionKey(testClass.TestCollection, closedVariant); + var variantCollection = _variantCollectionCache.GetOrAdd( + variantKey, + _ => new FixtureArgumentSetTestCollection(testClass.TestCollection, closedVariant)); + + var classKey = BuildVariantClassKey(variantKey, testClass.Class); + var closedVariantTestClass = _variantClassCache.GetOrAdd( + classKey, + _ => new FixtureArgumentSetTestClass( + testClass.Class, + variantCollection, + closedVariant, + UniqueIDGenerator.ForTestClass(variantCollection.UniqueID, testClass.Class.FullName))); + + var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); closedVariantTestMethod.UpdateArgumentsFromMethod(); @@ -144,6 +140,20 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl return true; } + private static string BuildVariantCollectionKey(IXunitTestCollection sourceCollection, IReadOnlyList closedVariant) + { + var variantKey = string.Join( + ",", + closedVariant.Select(argument => $"{argument.EnumValue.GetType().AssemblyQualifiedName}={Convert.ToInt64(argument.EnumValue)}")); + + return $"{sourceCollection.UniqueID}|{variantKey}"; + } + + private static string BuildVariantClassKey(string variantCollectionKey, Type testClass) + { + return $"{variantCollectionKey}|{testClass.AssemblyQualifiedName}"; + } + private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(FixtureArgumentSetsAttribute attribute) { bool IsPowerOfTwo(long x) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index a97cdeb3ad..5927a5a23b 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -34,6 +34,7 @@ public override async ValueTask RunTestCases(IReadOnlyCollection private sealed class FixtureArgumentSetTestAssembly : XunitTestAssembly { private readonly Assembly _assembly; + private readonly object _assemblyFixtureTypesLock = new object(); private IReadOnlyCollection _assemblyFixtureTypes; private static readonly FieldInfo AssemblyFixtureTypesField = typeof(XunitTestAssembly).GetField("assemblyFixtureTypes", BindingFlags.Instance | BindingFlags.NonPublic); @@ -55,20 +56,26 @@ public FixtureArgumentSetTestAssembly() { if (_assemblyFixtureTypes == null) { + lock (_assemblyFixtureTypesLock) + { + if (_assemblyFixtureTypes == null) + { #pragma warning disable CS0618 // AssemblyFixtureAttribute is obsolete; usage is required for assembly fixture discovery. - _assemblyFixtureTypes = _assembly - .GetCustomAttributes(typeof(global::Xunit.AssemblyFixtureAttribute), inherit: false) - .Cast() - .Select(attribute => attribute.AssemblyFixtureType) - .ToArray(); + _assemblyFixtureTypes = _assembly + .GetCustomAttributes(typeof(global::Xunit.AssemblyFixtureAttribute), inherit: false) + .Cast() + .Select(attribute => attribute.AssemblyFixtureType) + .ToArray(); #pragma warning restore CS0618 - if (AssemblyFixtureTypesField == null) - { - throw new InvalidOperationException("Unable to initialize assembly fixtures because XunitTestAssembly.assemblyFixtureTypes was not found."); - } + if (AssemblyFixtureTypesField == null) + { + throw new InvalidOperationException("Unable to initialize assembly fixtures because XunitTestAssembly.assemblyFixtureTypes was not found."); + } - AssemblyFixtureTypesField.SetValue(this, new Lazy>(() => _assemblyFixtureTypes)); + AssemblyFixtureTypesField.SetValue(this, new Lazy>(() => _assemblyFixtureTypes)); + } + } } return _assemblyFixtureTypes; @@ -170,7 +177,12 @@ private static void InjectFixtureArguments(XunitTestClassRunnerContext context) if (FixtureCacheField == null) { - return; + throw new InvalidOperationException("Unable to inject fixture arguments because FixtureMappingManager.fixtureCache was not found."); + } + + if (ParentMappingManagerField == null) + { + throw new InvalidOperationException("Unable to inject fixture arguments because FixtureMappingManager.parentMappingManager was not found."); } var cacheOwner = context.ClassFixtureMappings; @@ -182,7 +194,7 @@ private static void InjectFixtureArguments(XunitTestClassRunnerContext context) var cache = FixtureCacheField.GetValue(cacheOwner) as IDictionary; if (cache == null) { - return; + throw new TestPipelineException("Unable to inject fixture arguments because the xUnit fixture cache could not be read."); } var fixtureArguments = new List(); diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs index a755aef50c..1c25784d0a 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs @@ -18,7 +18,7 @@ public FixtureArgumentSetTestCollection(IXunitTestCollection sourceCollection, I : base( EnsureArg.IsNotNull(sourceCollection, nameof(sourceCollection)).TestAssembly, sourceCollection.CollectionDefinition, - sourceCollection.DisableParallelization, + sourceCollection.DisableParallelization || IsExplicitCollection(sourceCollection), BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments, testClassName), uniqueID: BuildUniqueId(sourceCollection, fixtureArguments, testClassName)) { @@ -51,5 +51,10 @@ private static string BuildUniqueId(IXunitTestCollection sourceCollection, IRead var displayName = BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments, testClassName); return UniqueIDGenerator.ForTestCollection(sourceCollection.TestAssembly.UniqueID, displayName, sourceCollection.TestCollectionClassName); } + + private static bool IsExplicitCollection(IXunitTestCollection sourceCollection) + { + return sourceCollection.CollectionDefinition != null || sourceCollection.TestCollectionClassName == null; + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs index 1d9a7974ae..e19efa1824 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs @@ -19,15 +19,10 @@ namespace Microsoft.Health.Extensions.Xunit public sealed class RetryFactAttribute : FactAttribute { public RetryFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) { - SourceFilePath = sourceFilePath; - SourceLineNumber = sourceLineNumber; } - internal new string SourceFilePath { get; } - - internal new int SourceLineNumber { get; } - /// /// Gets or sets the maximum number of retry attempts (default is 3). /// diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs index 48ca973d04..5006abc022 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs @@ -19,15 +19,10 @@ namespace Microsoft.Health.Extensions.Xunit public sealed class RetryTheoryAttribute : TheoryAttribute { public RetryTheoryAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) { - SourceFilePath = sourceFilePath; - SourceLineNumber = sourceLineNumber; } - internal new string SourceFilePath { get; } - - internal new int SourceLineNumber { get; } - /// /// Gets or sets the maximum number of retry attempts (default is 3). /// diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index ee5445fe27..7564b283b8 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -55,9 +55,12 @@ protected override async ValueTask> CreateTe .ToArray(); } - private static RetryTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTestCase testCase, RetryTheoryAttribute attribute, int maxRetries, int delayMs, bool retryOnAssertionFailure) + private static IXunitTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTestCase testCase, RetryTheoryAttribute attribute, int maxRetries, int delayMs, bool retryOnAssertionFailure) { - var xunitTestCase = (XunitTestCase)testCase; + if (testCase is not XunitTestCase xunitTestCase) + { + return testCase; + } return new RetryTestCase( testMethod, diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs index 0cef141402..ff4e9333da 100644 --- a/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs @@ -15,13 +15,8 @@ namespace Xunit public sealed class SkippableFactAttribute : FactAttribute { public SkippableFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) { - SourceFilePath = sourceFilePath; - SourceLineNumber = sourceLineNumber; } - - internal new string SourceFilePath { get; } - - internal new int SourceLineNumber { get; } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs index bff37554fc..f86834daea 100644 --- a/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs @@ -15,13 +15,8 @@ namespace Xunit public sealed class SkippableTheoryAttribute : TheoryAttribute { public SkippableTheoryAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) { - SourceFilePath = sourceFilePath; - SourceLineNumber = sourceLineNumber; } - - internal new string SourceFilePath { get; } - - internal new int SourceLineNumber { get; } } } diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XunitAttributeSourceLocationTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XunitAttributeSourceLocationTests.cs new file mode 100644 index 0000000000..5c9c902752 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Extensions/XunitAttributeSourceLocationTests.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Extensions +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Web)] + public sealed class XunitAttributeSourceLocationTests + { + [Fact] + public void GivenCustomFactAttributes_WhenConstructedWithSourceLocation_ThenBaseSourceLocationIsPopulated() + { + const string sourceFilePath = "test-file.cs"; + const int sourceLineNumber = 123; + + var retryFact = new RetryFactAttribute(sourceFilePath, sourceLineNumber); + var skippableFact = new SkippableFactAttribute(sourceFilePath, sourceLineNumber); + + Assert.Equal(sourceFilePath, retryFact.SourceFilePath); + Assert.Equal(sourceLineNumber, retryFact.SourceLineNumber); + Assert.Equal(sourceFilePath, skippableFact.SourceFilePath); + Assert.Equal(sourceLineNumber, skippableFact.SourceLineNumber); + } + + [Fact] + public void GivenCustomTheoryAttributes_WhenConstructedWithSourceLocation_ThenBaseSourceLocationIsPopulated() + { + const string sourceFilePath = "test-file.cs"; + const int sourceLineNumber = 123; + + var retryTheory = new RetryTheoryAttribute(sourceFilePath, sourceLineNumber); + var skippableTheory = new SkippableTheoryAttribute(sourceFilePath, sourceLineNumber); + + Assert.Equal(sourceFilePath, retryTheory.SourceFilePath); + Assert.Equal(sourceLineNumber, retryTheory.SourceLineNumber); + Assert.Equal(sourceFilePath, skippableTheory.SourceFilePath); + Assert.Equal(sourceLineNumber, skippableTheory.SourceLineNumber); + } + } +} From c2ab3a2470a5d11d478b007484d8f6a4c8913c7a Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 28 May 2026 11:34:10 -0700 Subject: [PATCH 030/105] Fix xUnit3 review issues: IsExplicitCollection logic, dead code cleanup - Fix inverted IsExplicitCollection condition: TestCollectionClassName is null for implicit per-class collections, not explicit ones. The old condition disabled parallelization on all default collections. - Remove dead testClassName parameter from FixtureArgumentSetTestCollection (always null at call site). - Add Trace.WriteLine warning when RetryTheoryDiscoverer passes through a non-XunitTestCase without retry wrapping. - Remove dead null-conditional on ParentMappingManagerField after explicit null guard. Co-Authored-By: Claude Opus 4.6 --- .../CustomXunitTestFrameworkExecutor.cs | 2 +- .../FixtureArgumentSetTestCollection.cs | 23 ++++++++++--------- .../RetryTheoryDiscoverer.cs | 2 ++ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index 5927a5a23b..2e1f0b0f75 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -186,7 +186,7 @@ private static void InjectFixtureArguments(XunitTestClassRunnerContext context) } var cacheOwner = context.ClassFixtureMappings; - if (ParentMappingManagerField?.GetValue(cacheOwner) is FixtureMappingManager parentMappingManager) + if (ParentMappingManagerField.GetValue(cacheOwner) is FixtureMappingManager parentMappingManager) { cacheOwner = parentMappingManager; } diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs index 1c25784d0a..9a953154bc 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs @@ -14,13 +14,13 @@ namespace Microsoft.Health.Extensions.Xunit { internal sealed class FixtureArgumentSetTestCollection : XunitTestCollection { - public FixtureArgumentSetTestCollection(IXunitTestCollection sourceCollection, IReadOnlyList fixtureArguments, string testClassName = null) + public FixtureArgumentSetTestCollection(IXunitTestCollection sourceCollection, IReadOnlyList fixtureArguments) : base( EnsureArg.IsNotNull(sourceCollection, nameof(sourceCollection)).TestAssembly, sourceCollection.CollectionDefinition, sourceCollection.DisableParallelization || IsExplicitCollection(sourceCollection), - BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments, testClassName), - uniqueID: BuildUniqueId(sourceCollection, fixtureArguments, testClassName)) + BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments), + uniqueID: BuildUniqueId(sourceCollection, fixtureArguments)) { EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); } @@ -31,30 +31,31 @@ public FixtureArgumentSetTestCollection() } #pragma warning restore CS0618 - private static string BuildDisplayName(string baseDisplayName, IReadOnlyList fixtureArguments, string testClassName) + private static string BuildDisplayName(string baseDisplayName, IReadOnlyList fixtureArguments) { EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); if (fixtureArguments.Count == 0) { - return string.IsNullOrEmpty(testClassName) ? baseDisplayName : testClassName; + return baseDisplayName; } var argsLabel = string.Join(", ", fixtureArguments.Select(v => v.EnumValue)); - return string.IsNullOrEmpty(testClassName) - ? $"{baseDisplayName}({argsLabel})" - : $"{testClassName}({argsLabel})"; + return $"{baseDisplayName}({argsLabel})"; } - private static string BuildUniqueId(IXunitTestCollection sourceCollection, IReadOnlyList fixtureArguments, string testClassName) + private static string BuildUniqueId(IXunitTestCollection sourceCollection, IReadOnlyList fixtureArguments) { - var displayName = BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments, testClassName); + var displayName = BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments); return UniqueIDGenerator.ForTestCollection(sourceCollection.TestAssembly.UniqueID, displayName, sourceCollection.TestCollectionClassName); } private static bool IsExplicitCollection(IXunitTestCollection sourceCollection) { - return sourceCollection.CollectionDefinition != null || sourceCollection.TestCollectionClassName == null; + // A collection is explicit when it has a [CollectionDefinition] class or a named + // [Collection("...")] attribute (TestCollectionClassName is non-null for named collections, + // null for the implicit per-class default collection). + return sourceCollection.CollectionDefinition != null || sourceCollection.TestCollectionClassName != null; } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index 7564b283b8..2a00af1e0d 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Xunit; @@ -59,6 +60,7 @@ private static IXunitTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTe { if (testCase is not XunitTestCase xunitTestCase) { + Trace.WriteLine($"[RetryTheory] WARNING: Test case '{testCase.TestCaseDisplayName}' is {testCase.GetType().Name}, not XunitTestCase. Retry logic will NOT be applied."); return testCase; } From 4e6c3391229349a9224f8b6b272da09fdd69e263 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 13:41:53 -0700 Subject: [PATCH 031/105] Fix xUnit v3 compile errors in files merged from main SmartSearchSharedFixture must return ValueTask for IAsyncLifetime/IAsyncDisposable under xUnit v3, and ITestOutputHelper moved from Xunit.Abstractions to Xunit. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Features/Smart/SmartSearchSharedFixture.cs | 4 ++-- .../Persistence/SqlServerSchemaUpgradeTests.cs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs index 47e49d3b2e..5be079efbe 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs @@ -47,7 +47,7 @@ public SmartSearchSharedFixture(DataStore dataStore) public FhirStorageTestsFixture Fixture => _fixture; - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { if (ModelInfoProvider.Instance.Version != FhirSpecification.R4 && ModelInfoProvider.Instance.Version != FhirSpecification.R4B) @@ -79,7 +79,7 @@ public async Task InitializeAsync() await UpsertResource(Samples.GetJsonSample("Location-example-hq")); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_fixture != null) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs index d2b0807da0..e0ab029b0c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs @@ -41,7 +41,6 @@ using Microsoft.SqlServer.TransactSql.ScriptDom; using NSubstitute; using Xunit; -using Xunit.Abstractions; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { From 08fa85dfe7d4f203274f546f51c8c112cf52aef8 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 13:53:08 -0700 Subject: [PATCH 032/105] Align unit test result publishing with main's revert of testfx#7167 workaround main reverted failTaskOnFailedTests back to true for the integration/E2E publish tasks; keep build.yml consistent rather than leaving unit tests unable to fail the pipeline. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/build.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build/jobs/build.yml b/build/jobs/build.yml index 58d8aecb79..4166fdd9b1 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -48,8 +48,7 @@ steps: testResultsFiles: '$(Agent.TempDirectory)/unit-test-results/**/*.trx' mergeTestResults: true testRunTitle: 'Unit Tests' - # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed - failTaskOnFailedTests: false + failTaskOnFailedTests: true condition: succeededOrFailed() - ${{ if eq(parameters.codeCoverage, 'true') }}: @@ -71,8 +70,7 @@ steps: testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' mergeTestResults: true testRunTitle: 'Unit Tests' - # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed - failTaskOnFailedTests: false + failTaskOnFailedTests: true condition: succeededOrFailed() - task: reportgenerator@5 displayName: 'aggregate unit test coverage' From 5617602ee7693fdf56230d55f4d6f9521292cc55 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 14:40:03 -0700 Subject: [PATCH 033/105] Fix RetryFact failures silently disappearing from test results A [RetryFact]/[RetryTheory] test whose assertion failed was omitted from the test results entirely: it was absent from the TRX, the counters and stdout, and the process exited 0. Non-final attempts run on FailureInterceptingMessageBus, which swallowed the ITestFailed message so the attempt could be retried. Assertion failures are not retriable by default, so the run bailed out of the attempt loop and set RunSummary.Failed, but xUnit v3 reporters derive their results solely from the message bus and never saw the failure. The intercepting bus now defers the failure and the rest of the attempt's messages instead of dropping the failure outright, so the caller can replay them in their original order when it decides not to retry. Ordering matters: reporters finalize a test on ITestFinished, so a failure forwarded after its ITestFinished is ignored. Also: - Report RetryFact display names through TestIntrospectionHelper so they honor the configured MethodDisplay setting instead of being bare method names. - Write retry diagnostics to Console rather than Trace. Trace output only goes to an attached debugger, so none of this was visible in CI - which is why the dropped failures went unnoticed. - Run at least one attempt when MaxRetries is below one, which would otherwise skip the loop and report no result at all. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryFactDiscoverer.cs | 50 ++++++--- .../RetryTestCase.cs | 105 ++++++++++++++---- 2 files changed, 113 insertions(+), 42 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs index 9d30b5ca09..bf6a411b2c 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs @@ -16,6 +16,13 @@ namespace Microsoft.Health.Extensions.Xunit /// public sealed class RetryFactDiscoverer : IXunitTestCaseDiscoverer { + /// + /// Discovers the test case for a method marked with . + /// + /// The discovery options for the test assembly. + /// The test method being discovered. + /// The applied to the method. + /// A single wrapping the method. public ValueTask> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, @@ -23,28 +30,35 @@ public ValueTask> Discover( { var attribute = (RetryFactAttribute)factAttribute; - var maxRetries = attribute.MaxRetries; - var delayMs = attribute.DelayBetweenRetriesMs; - var retryOnAssertionFailure = attribute.RetryOnAssertionFailure; + // Let xUnit compute the test case details the same way it does for a plain [Fact], + // so that the display name honors the configured MethodDisplay setting instead of + // being reported as a bare, unqualified method name. + var details = TestIntrospectionHelper.GetTestCaseDetails( + discoveryOptions, + testMethod, + factAttribute, + testMethodArguments: null, + timeout: null, + baseDisplayName: null); var testCase = new RetryTestCase( - testMethod, - testMethod.GetDisplayName(testMethod.MethodName, label: null, testMethodArguments: null, methodGenericTypes: null), - UniqueIDGenerator.ForTestCase(testMethod.UniqueID, null, null), - @explicit: attribute.Explicit, - skipExceptions: attribute.SkipExceptions, - skipReason: attribute.Skip, - skipType: attribute.SkipType, - skipUnless: attribute.SkipUnless, - skipWhen: attribute.SkipWhen, + details.ResolvedTestMethod, + details.TestCaseDisplayName, + details.UniqueID, + @explicit: details.Explicit, + skipExceptions: details.SkipExceptions, + skipReason: details.SkipReason, + skipType: details.SkipType, + skipUnless: details.SkipUnless, + skipWhen: details.SkipWhen, traits: testMethod.Traits.ToDictionary(kvp => kvp.Key, kvp => new HashSet(kvp.Value)), testMethodArguments: null, - sourceFile: attribute.SourceFilePath, - sourceLine: attribute.SourceLineNumber, - timeout: attribute.Timeout, - maxRetries: maxRetries, - delayMs: delayMs, - retryOnAssertionFailure: retryOnAssertionFailure); + sourceFile: details.SourceFilePath, + sourceLine: details.SourceLineNumber, + timeout: details.Timeout, + maxRetries: attribute.MaxRetries, + delayMs: attribute.DelayBetweenRetriesMs, + retryOnAssertionFailure: attribute.RetryOnAssertionFailure); return new ValueTask>(new[] { testCase }); } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 3975e5624d..7c44bbb597 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -51,11 +51,23 @@ public RetryTestCase( bool retryOnAssertionFailure) : base(testMethod, displayName, uniqueId, @explicit, skipExceptions, skipReason, skipType, skipUnless, skipWhen, traits, testMethodArguments, sourceFile, sourceLine, timeout) { - _maxRetries = maxRetries; + // A retry count below one would skip the attempt loop entirely and report no result + // at all, silently dropping the test. Always run at least one attempt. + _maxRetries = Math.Max(1, maxRetries); _delayMs = delayMs; _retryOnAssertionFailure = retryOnAssertionFailure; } + /// + /// Runs the test case, retrying transient failures up to the configured number of attempts. + /// + /// Whether explicit tests should be run. + /// The message bus that test results are reported to. + /// The arguments to pass to the test class constructor. + /// The exception aggregator for the containing test class. + /// The cancellation token source for the test run. + /// A summary of the run. Reporters derive their results from + /// rather than from this summary, so every outcome must also be reported as a message. public async ValueTask Run( ExplicitOption explicitOption, IMessageBus messageBus, @@ -63,8 +75,9 @@ public async ValueTask Run( ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { - // Use System.Diagnostics.Trace for ADO visibility - Trace.WriteLine($"##vso[task.logdetail]RetryFact starting test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' with MaxRetries={_maxRetries}, DelayMs={_delayMs}, RetryOnAssertionFailure={_retryOnAssertionFailure}"); + // Trace output is only visible to an attached debugger, never in CI logs. Anything + // that needs to show up in a pipeline log has to go to Console instead. + Trace.WriteLine($"RetryFact starting test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' with MaxRetries={_maxRetries}, DelayMs={_delayMs}, RetryOnAssertionFailure={_retryOnAssertionFailure}"); var runSummary = new RunSummary { Total = 1 }; Exception lastException = null; @@ -73,7 +86,7 @@ public async ValueTask Run( { var isLastAttempt = attempt == _maxRetries; - Trace.WriteLine($"##vso[task.logdetail]RetryFact attempt {attempt}/{_maxRetries} for test '{TestMethod.MethodName}'"); + Trace.WriteLine($"RetryFact attempt {attempt}/{_maxRetries} for test '{TestMethod.MethodName}'"); // Create a fresh aggregator for each attempt var attemptAggregator = new ExceptionAggregator(); @@ -108,7 +121,7 @@ public async ValueTask Run( if (summary.Failed == 0) { // Test passed - success message already went through to Test Explorer - Trace.WriteLine($"##vso[task.logdetail]RetryFact test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' passed on attempt {attempt}/{_maxRetries}"); + Trace.WriteLine($"RetryFact test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' passed on attempt {attempt}/{_maxRetries}"); messageBus.QueueMessage( new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' passed on attempt {attempt}/{_maxRetries}")); @@ -140,23 +153,30 @@ public async ValueTask Run( lastException = new InvalidOperationException(fullMessage); } - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Test failed but exception is null, created placeholder exception (IsAssertion={isAssertionFailure})"); + Trace.WriteLine($"RetryFact: Test failed but exception is null, created placeholder exception (IsAssertion={isAssertionFailure})"); } - Trace.WriteLine($"##vso[task.logdetail]RetryFact test failed on attempt {attempt} with exception type: {lastException?.GetType().FullName ?? "null"}, Message: {lastException?.Message ?? "null"}"); + Trace.WriteLine($"RetryFact test failed on attempt {attempt} with exception type: {lastException?.GetType().FullName ?? "null"}, Message: {lastException?.Message ?? "null"}"); if (!isLastAttempt) { // Check if we should retry this exception (now handles null) var shouldRetry = ShouldRetry(lastException); - Trace.WriteLine($"##vso[task.logdetail]RetryFact ShouldRetry={shouldRetry} for exception type {lastException?.GetType().FullName ?? "null"}"); + Trace.WriteLine($"RetryFact ShouldRetry={shouldRetry} for exception type {lastException?.GetType().FullName ?? "null"}"); if (!shouldRetry) { - Trace.WriteLine($"##vso[task.logissue type=warning]Test '{TestMethod.MethodName}' failed with non-retriable exception. Skipping retries."); + Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed with a non-retriable exception on attempt {attempt}/{_maxRetries}. Skipping remaining retries."); messageBus.QueueMessage( new DiagnosticMessage($"[RetryFact] Test '{TestMethod.MethodName}' failed with non-retriable exception. Skipping retries.")); + // This attempt ran on the intercepting bus, which deferred the + // ITestFailed message so the test could be retried. We are not + // retrying, so replay it now: the reporters derive their results + // solely from the message bus, and without this the failure would + // disappear from the test results entirely and the run would pass. + interceptingBus?.ReplayDeferredMessages(); + if (lastException != null) { aggregator.Add(lastException); @@ -167,7 +187,7 @@ public async ValueTask Run( } // Not the last attempt - the failure was intercepted, so retry - Trace.WriteLine($"##vso[task.logissue type=warning]Test '{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}, will retry after {_delayMs}ms"); + Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}, retrying after {_delayMs}ms. Error: {lastException?.Message ?? "No exception message"}"); messageBus.QueueMessage( new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}. Retrying after {_delayMs}ms delay. Error: {lastException?.Message ?? "No exception message"}")); @@ -176,7 +196,7 @@ public async ValueTask Run( else { // Last attempt - failure message already went through to Test Explorer - Trace.WriteLine($"##vso[task.logissue type=error]Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed after {_maxRetries} attempts. Final error: {lastException?.Message ?? "No exception captured"}"); + Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed after all {_maxRetries} attempts. Final error: {lastException?.Message ?? "No exception captured"}"); messageBus.QueueMessage( new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed after {_maxRetries} attempts. Last exception: {lastException?.Message ?? "No exception message"}")); @@ -196,7 +216,7 @@ public async ValueTask Run( } catch (Exception ex) { - Trace.WriteLine($"##vso[task.logissue type=error]RetryFact unexpected exception: {ex.GetType().FullName}: {ex.Message}"); + Console.WriteLine($"[RetryFact] Unexpected exception while running '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}': {ex.GetType().FullName}: {ex.Message}"); throw; } finally @@ -207,7 +227,7 @@ public async ValueTask Run( } // Should never reach here - Trace.WriteLine($"##vso[task.logissue type=error]RetryFact WARNING: Reached end of retry loop unexpectedly"); + Console.WriteLine($"[RetryFact] Reached the end of the retry loop unexpectedly for '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}'. Reporting the test as failed."); runSummary.Failed = 1; return runSummary; } @@ -223,7 +243,7 @@ protected override void Serialize(IXunitSerializationInfo info) protected override void Deserialize(IXunitSerializationInfo info) { base.Deserialize(info); - _maxRetries = info.GetValue(nameof(_maxRetries)); + _maxRetries = Math.Max(1, info.GetValue(nameof(_maxRetries))); _delayMs = info.GetValue(nameof(_delayMs)); _retryOnAssertionFailure = info.GetValue(nameof(_retryOnAssertionFailure)); } @@ -236,7 +256,7 @@ private bool ShouldRetry(Exception ex) // If exception is null, we should retry (something went wrong with exception capture) if (ex == null) { - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Exception is null, will retry"); + Trace.WriteLine($"RetryFact: Exception is null, will retry"); return true; // Retry when we can't determine the exception type } @@ -251,30 +271,38 @@ private bool ShouldRetry(Exception ex) { if (!_retryOnAssertionFailure) { - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Not retrying XunitException because _retryOnAssertionFailure is false"); + Trace.WriteLine($"RetryFact: Not retrying XunitException because _retryOnAssertionFailure is false"); return false; } else { - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Retrying XunitException because _retryOnAssertionFailure is true"); + Trace.WriteLine($"RetryFact: Retrying XunitException because _retryOnAssertionFailure is true"); return true; } } // Retry everything else (network, timeout, SQL transient, etc.) - Trace.WriteLine($"##vso[task.logdetail]RetryFact: Retrying non-assertion exception of type {ex.GetType().FullName}"); + Trace.WriteLine($"RetryFact: Retrying non-assertion exception of type {ex.GetType().FullName}"); return true; } /// - /// Message bus that intercepts ONLY failure messages (ITestFailed). - /// Used on non-final retry attempts to suppress intermediate failures. - /// Success messages and all other messages always pass through. - /// Also captures failure details (messages and stack traces) for diagnostic purposes. + /// Message bus that defers failure messages (ITestFailed) so that an attempt which is + /// about to be retried leaves no trace in the test results. /// + /// + /// Once a failure is seen, every subsequent message for that attempt is buffered too. + /// Reporters finalize a test when they see ITestFinished, so forwarding a failure after + /// its ITestFinished has already gone through has no effect - the failure would be + /// dropped and the test would disappear from the run entirely. Buffering the tail of the + /// attempt keeps the messages in their original order, so the caller can either discard + /// them (retrying) or replay them intact (reporting the failure). + /// private class FailureInterceptingMessageBus : IMessageBus { private readonly IMessageBus _innerBus; + private readonly List _deferredMessages = new List(); + private bool _deferring; public FailureInterceptingMessageBus(IMessageBus innerBus) { @@ -287,9 +315,20 @@ public FailureInterceptingMessageBus(IMessageBus innerBus) public bool IsAssertionFailure { get; private set; } + /// + /// Gets a value indicating whether a failure was intercepted and is waiting to be + /// either discarded or replayed. + /// + public bool HasDeferredFailure => _deferredMessages.Count > 0; + public bool QueueMessage(IMessageSinkMessage message) { - // Intercept ONLY failure messages - suppress them for non-final attempts + if (_deferring) + { + _deferredMessages.Add(message); + return true; + } + if (message is ITestFailed failed) { // Capture failure details for diagnostics @@ -313,13 +352,31 @@ public bool QueueMessage(IMessageSinkMessage message) failed.ExceptionTypes[0].Contains("TrueException", StringComparison.Ordinal) || failed.ExceptionTypes[0].Contains("FalseException", StringComparison.Ordinal)); - return true; // Swallow the failure - we're going to retry + _deferring = true; + _deferredMessages.Add(message); + return true; } // All other messages (ITestPassed, ITestStarting, ITestFinished, etc.) pass through return _innerBus.QueueMessage(message); } + /// + /// Replays the deferred failure and everything that followed it to the underlying bus, + /// in their original order. Call this when the attempt will not be retried, otherwise + /// the failure is never reported and the test silently vanishes from the results. + /// + public void ReplayDeferredMessages() + { + foreach (IMessageSinkMessage message in _deferredMessages) + { + _innerBus.QueueMessage(message); + } + + _deferredMessages.Clear(); + _deferring = false; + } + public void Dispose() { // Don't dispose the inner bus - it's owned by the caller From 872507c01c6ec29f7f742a066335f5b4386de4b9 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 14:51:36 -0700 Subject: [PATCH 034/105] Fix fixture-variant display names and remove dead xUnit v3 shim code Review findings from the xUnit v2 -> v3 migration PR. F1: Fixture argument set variants all shared one display name, so a class with N variants produced N identical test names in TRX/console output and they were indistinguishable on failure. XunitTestClass has no writable class-name field in v3, so instead set XunitTestCase.testCaseDisplayName on each discovered case, appending the variant (e.g. "(CosmosDb)"). F3 / dead code: xUnit v3 seals XunitTestClass and XunitTestMethod (only ToString/Equals/GetHashCode are overridable) and dispatches through interfaces, so every "public new" member in these shims was unreachable. Removed them, along with the redundant nested FixtureArgumentSetTestAssembly and the empty [Obsolete] AssemblyFixtureAttribute shell (all repo usages already bind to Xunit.AssemblyFixtureAttribute). Reflection lookups now fail loudly via GetRequiredField instead of silently no-oping through "?.SetValue", so a future xUnit upgrade that renames a private field surfaces as an error rather than as silently mis-named or un-filterable tests. FixtureArgumentSetTestCollection: behavior is unchanged, but the comment on IsExplicitCollection was wrong -- TestCollectionClassName holds the [CollectionDefinition] class and is null for a name-only [Collection("...")], so that term never added anything. Renamed to ShouldDisableParallelization and documented what it actually covers. Verified: full solution builds clean; a probe assembly confirms variants now list as "VariantProbe.VariantTest (CosmosDb)" / "(SqlServer)" and that --filter-trait "DataStore=SqlServer" still selects exactly the SQL variants, which is what the CI filtering strategy depends on. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AssemblyFixtureAttribute.cs | 15 ---- .../CustomXunitTestFrameworkDiscoverer.cs | 45 +++++++++- .../CustomXunitTestFrameworkExecutor.cs | 54 +---------- .../FixtureArgumentSetTestClass.cs | 27 ++---- .../FixtureArgumentSetTestCollection.cs | 24 +++-- .../FixtureArgumentSetTestMethod.cs | 89 +++++++++---------- 6 files changed, 113 insertions(+), 141 deletions(-) delete mode 100644 src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs diff --git a/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs deleted file mode 100644 index 9737306b33..0000000000 --- a/src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs +++ /dev/null @@ -1,15 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; - -namespace Microsoft.Health.Extensions.Xunit -{ - [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true, Inherited = false)] - [Obsolete("Use Xunit.AssemblyFixtureAttribute instead.")] - public sealed class AssemblyFixtureAttribute : Attribute - { - } -} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 92df5ece95..ddc671db81 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -21,6 +21,8 @@ namespace Microsoft.Health.Extensions.Xunit /// internal sealed class CustomXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer { + private static readonly FieldInfo TestCaseDisplayNameField = typeof(XunitTestCase).GetField("testCaseDisplayName", BindingFlags.Instance | BindingFlags.NonPublic); + private readonly ConcurrentDictionary _variantCollectionCache = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _variantClassCache = new(StringComparer.Ordinal); @@ -130,7 +132,17 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl closedVariantTestMethod.UpdateArgumentsFromMethod(); - if (!await FindTestsForMethod(closedVariantTestMethod, discoveryOptions, callback)) + // xUnit builds the display name from the class and method names only, so every + // variant of a method would otherwise be reported under an identical name and + // a failure could not be attributed to a specific fixture argument set. + var variantArguments = closedVariant; + Func> variantCallback = testCase => + { + ApplyVariantDisplayName(testCase, variantArguments); + return callback(testCase); + }; + + if (!await FindTestsForMethod(closedVariantTestMethod, discoveryOptions, variantCallback)) { return false; } @@ -140,6 +152,37 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl return true; } + /// + /// Appends the fixture argument set to a discovered test case's display name, so that the + /// variants of a test can be told apart in test results. + /// + /// The discovered test case. + /// The fixture arguments the test case was discovered for. + private static void ApplyVariantDisplayName(ITestCase testCase, SingleFlag[] closedVariant) + { + if (closedVariant.Length == 0 || testCase is not XunitTestCase xunitTestCase) + { + return; + } + + if (TestCaseDisplayNameField == null) + { + throw new InvalidOperationException( + "Unable to name fixture argument set variants because XunitTestCase.testCaseDisplayName was not found. " + + "This usually means the xunit.v3 version changed; see Microsoft.Health.Extensions.Xunit."); + } + + var suffix = $" ({string.Join(", ", closedVariant.Select(argument => argument.EnumValue))})"; + var displayName = xunitTestCase.TestCaseDisplayName; + + if (displayName.EndsWith(suffix, StringComparison.Ordinal)) + { + return; + } + + TestCaseDisplayNameField.SetValue(xunitTestCase, displayName + suffix); + } + private static string BuildVariantCollectionKey(IXunitTestCollection sourceCollection, IReadOnlyList closedVariant) { var variantKey = string.Join( diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index 2e1f0b0f75..f3a69c2379 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -21,7 +21,7 @@ namespace Microsoft.Health.Extensions.Xunit internal sealed class CustomXunitTestFrameworkExecutor : XunitTestFrameworkExecutor { public CustomXunitTestFrameworkExecutor(Assembly assembly) - : base(new FixtureArgumentSetTestAssembly(assembly)) + : base(new XunitTestAssembly(assembly, configFileName: null, assembly.GetName().Version, UniqueIDGenerator.ForAssembly(assembly.Location, null))) { } @@ -31,58 +31,6 @@ public override async ValueTask RunTestCases(IReadOnlyCollection await runner.Run(TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); } - private sealed class FixtureArgumentSetTestAssembly : XunitTestAssembly - { - private readonly Assembly _assembly; - private readonly object _assemblyFixtureTypesLock = new object(); - private IReadOnlyCollection _assemblyFixtureTypes; - private static readonly FieldInfo AssemblyFixtureTypesField = typeof(XunitTestAssembly).GetField("assemblyFixtureTypes", BindingFlags.Instance | BindingFlags.NonPublic); - - public FixtureArgumentSetTestAssembly(Assembly assembly) - : base(assembly, configFileName: null, assembly.GetName().Version, UniqueIDGenerator.ForAssembly(assembly.Location, null)) - { - _assembly = assembly; - } - -#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes - public FixtureArgumentSetTestAssembly() - { - } -#pragma warning restore CS0618 - - public new IReadOnlyCollection AssemblyFixtureTypes - { - get - { - if (_assemblyFixtureTypes == null) - { - lock (_assemblyFixtureTypesLock) - { - if (_assemblyFixtureTypes == null) - { -#pragma warning disable CS0618 // AssemblyFixtureAttribute is obsolete; usage is required for assembly fixture discovery. - _assemblyFixtureTypes = _assembly - .GetCustomAttributes(typeof(global::Xunit.AssemblyFixtureAttribute), inherit: false) - .Cast() - .Select(attribute => attribute.AssemblyFixtureType) - .ToArray(); -#pragma warning restore CS0618 - - if (AssemblyFixtureTypesField == null) - { - throw new InvalidOperationException("Unable to initialize assembly fixtures because XunitTestAssembly.assemblyFixtureTypes was not found."); - } - - AssemblyFixtureTypesField.SetValue(this, new Lazy>(() => _assemblyFixtureTypes)); - } - } - } - - return _assemblyFixtureTypes; - } - } - } - private sealed class FixtureArgumentSetAssemblyRunner : XunitTestAssemblyRunner { protected override async ValueTask RunTestCollection(XunitTestAssemblyRunnerContext context, IXunitTestCollection testCollection, IReadOnlyCollection testCases) diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs index 698fb56f6d..073ab90282 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs @@ -15,9 +15,12 @@ namespace Microsoft.Health.Extensions.Xunit { internal sealed class FixtureArgumentSetTestClass : XunitTestClass { - private static readonly FieldInfo UniqueIdField = typeof(XunitTestClass).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic); - private readonly Type _testClassType; - private IReadOnlyList _fixtureArguments; + private static readonly FieldInfo UniqueIdField = typeof(XunitTestClass).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + "XunitTestClass.uniqueID was not found, so fixture argument set variants cannot be given distinct identities and " + + "would collapse into a single test class. This usually means the xunit.v3 version changed."); + + private readonly IReadOnlyList _fixtureArguments; public FixtureArgumentSetTestClass(Type testClassType, IXunitTestCollection testCollection, IReadOnlyList fixtureArguments, string uniqueId) : base(testClassType, testCollection, uniqueId) @@ -26,7 +29,6 @@ public FixtureArgumentSetTestClass(Type testClassType, IXunitTestCollection test EnsureArg.IsNotNull(testCollection, nameof(testCollection)); EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - _testClassType = testClassType; _fixtureArguments = fixtureArguments; UpdateUniqueId(); } @@ -38,26 +40,11 @@ public FixtureArgumentSetTestClass() } #pragma warning restore CS0618 -#pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists - public new Type Class => _testClassType ?? base.Class; -#pragma warning restore SA1100 - internal IReadOnlyList GetFixtureArguments() { return _fixtureArguments; } - internal void ApplyFixtureArguments(IReadOnlyList fixtureArguments) - { - if (fixtureArguments == null) - { - return; - } - - _fixtureArguments = fixtureArguments; - UpdateUniqueId(); - } - private void UpdateUniqueId() { if (_fixtureArguments.Count == 0) @@ -66,7 +53,7 @@ private void UpdateUniqueId() } var className = $"{TestClassName}({string.Join(", ", _fixtureArguments.Select(v => v.EnumValue))})"; - UniqueIdField?.SetValue(this, UniqueIDGenerator.ForTestClass(TestCollection.UniqueID, className)); + UniqueIdField.SetValue(this, UniqueIDGenerator.ForTestClass(TestCollection.UniqueID, className)); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs index 9a953154bc..09df7430b3 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs @@ -18,7 +18,7 @@ public FixtureArgumentSetTestCollection(IXunitTestCollection sourceCollection, I : base( EnsureArg.IsNotNull(sourceCollection, nameof(sourceCollection)).TestAssembly, sourceCollection.CollectionDefinition, - sourceCollection.DisableParallelization || IsExplicitCollection(sourceCollection), + ShouldDisableParallelization(sourceCollection), BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments), uniqueID: BuildUniqueId(sourceCollection, fixtureArguments)) { @@ -50,12 +50,24 @@ private static string BuildUniqueId(IXunitTestCollection sourceCollection, IRead return UniqueIDGenerator.ForTestCollection(sourceCollection.TestAssembly.UniqueID, displayName, sourceCollection.TestCollectionClassName); } - private static bool IsExplicitCollection(IXunitTestCollection sourceCollection) + /// + /// Determines whether the variants of a collection should be prevented from running concurrently. + /// + /// + /// Splitting a collection into one variant per fixture argument set would otherwise let the + /// variants run concurrently. Any collection backed by a [CollectionDefinition] class stays + /// serialized, because those group classes that were deliberately placed together. + /// + /// This does not cover a name-only [Collection("...")] with no matching [CollectionDefinition]: + /// TestCollectionClassName holds the definition class, so it is null for those and there is + /// nothing here to detect them by. + /// + /// + /// The collection the variant was derived from. + /// true if the variant collection should run serially. + private static bool ShouldDisableParallelization(IXunitTestCollection sourceCollection) { - // A collection is explicit when it has a [CollectionDefinition] class or a named - // [Collection("...")] attribute (TestCollectionClassName is non-null for named collections, - // null for the implicit per-class default collection). - return sourceCollection.CollectionDefinition != null || sourceCollection.TestCollectionClassName != null; + return sourceCollection.DisableParallelization || sourceCollection.CollectionDefinition != null; } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index 80134a09ab..c590ec9a3d 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -14,10 +14,10 @@ namespace Microsoft.Health.Extensions.Xunit { internal sealed class FixtureArgumentSetTestMethod : XunitTestMethod { - private static readonly FieldInfo TestMethodArgumentsField = typeof(XunitTestMethod).GetField("testMethodArguments", BindingFlags.Instance | BindingFlags.NonPublic); - private static readonly FieldInfo TraitsField = typeof(XunitTestMethod).GetField("traits", BindingFlags.Instance | BindingFlags.NonPublic); - private static readonly FieldInfo UniqueIdField = typeof(XunitTestMethod).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic); - private static readonly FieldInfo MethodField = typeof(XunitTestMethod).GetField("method", BindingFlags.Instance | BindingFlags.NonPublic); + private static readonly FieldInfo TestMethodArgumentsField = GetRequiredField("testMethodArguments"); + private static readonly FieldInfo TraitsField = GetRequiredField("traits"); + private static readonly FieldInfo UniqueIdField = GetRequiredField("uniqueID"); + private static readonly FieldInfo MethodField = GetRequiredField("method"); private readonly FixtureArgumentSetTestClass _testClass; private readonly MethodInfo _methodInfo; @@ -37,8 +37,8 @@ public FixtureArgumentSetTestMethod(FixtureArgumentSetTestClass testClass, Metho _uniqueId = uniqueId; UpdateMethodArguments(Array.Empty()); - UniqueIdField?.SetValue(this, _uniqueId); - MethodField?.SetValue(this, _methodInfo); + UniqueIdField.SetValue(this, _uniqueId); + MethodField.SetValue(this, _methodInfo); } #pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes @@ -47,21 +47,21 @@ public FixtureArgumentSetTestMethod() } #pragma warning restore CS0618 -#pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists - public new string GetDisplayName(string baseDisplayName, string displayName, object[] testMethodArguments, Type[] genericTypes) + /// + /// Resolves a private field on that this type has to write to + /// because xUnit v3 seals the corresponding members. + /// + /// The name of the private instance field. + /// The field. + /// Thrown when the field no longer exists. + private static FieldInfo GetRequiredField(string name) { - var combinedArguments = CombineFixtureAndMethodArguments(testMethodArguments ?? Array.Empty()); - return base.GetDisplayName(baseDisplayName, displayName, combinedArguments, genericTypes); + return typeof(XunitTestMethod).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + $"XunitTestMethod.{name} was not found. Fixture argument set traits and arguments cannot be applied, " + + "which would silently change which tests the CI filters select. This usually means the xunit.v3 version changed."); } - public new object[] ResolveMethodArguments(object[] arguments) - { - var combinedArguments = CombineFixtureAndMethodArguments(arguments ?? Array.Empty()); - UpdateMethodArguments(combinedArguments); - return base.ResolveMethodArguments(combinedArguments); - } -#pragma warning restore SA1100 - private object[] CombineFixtureAndMethodArguments(object[] methodArguments) { if (_fixtureArguments.Count == 0) @@ -90,7 +90,7 @@ private void UpdateMethodArguments(object[] methodArguments) return; } - TestMethodArgumentsField?.SetValue(this, methodArguments); + TestMethodArgumentsField.SetValue(this, methodArguments); } internal void UpdateArgumentsFromMethod() @@ -101,41 +101,38 @@ internal void UpdateArgumentsFromMethod() } var combinedArguments = CombineFixtureAndMethodArguments(Array.Empty()); - TestMethodArgumentsField?.SetValue(this, combinedArguments); + TestMethodArgumentsField.SetValue(this, combinedArguments); - if (TraitsField != null) - { - var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); #pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists - foreach (var kvp in base.Traits) - { - traits[kvp.Key] = new HashSet(kvp.Value, StringComparer.OrdinalIgnoreCase); - } + foreach (var kvp in base.Traits) + { + traits[kvp.Key] = new HashSet(kvp.Value, StringComparer.OrdinalIgnoreCase); + } #pragma warning restore SA1100 - for (int i = 0; i < _fixtureArguments.Count; i++) + for (int i = 0; i < _fixtureArguments.Count; i++) + { + var enumValue = _fixtureArguments[i].EnumValue; + var enumValueText = enumValue.ToString(); + + string key = $"{enumValue.GetType().Name}"; + if (!traits.TryGetValue(key, out var values)) { - var enumValue = _fixtureArguments[i].EnumValue; - var enumValueText = enumValue.ToString(); - - string key = $"{enumValue.GetType().Name}"; - if (!traits.TryGetValue(key, out var values)) - { - values = new HashSet(StringComparer.OrdinalIgnoreCase); - traits[key] = values; - } - - values.Add(enumValueText); - - if (TryGetFixtureTraitName(enumValue, out var traitName)) - { - AddTrait(traits, traitName, enumValueText); - } + values = new HashSet(StringComparer.OrdinalIgnoreCase); + traits[key] = values; } - var typedTraits = traits.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyCollection)kvp.Value, StringComparer.OrdinalIgnoreCase); - TraitsField.SetValue(this, new Lazy>>(() => typedTraits)); + values.Add(enumValueText); + + if (TryGetFixtureTraitName(enumValue, out var traitName)) + { + AddTrait(traits, traitName, enumValueText); + } } + + var typedTraits = traits.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyCollection)kvp.Value, StringComparer.OrdinalIgnoreCase); + TraitsField.SetValue(this, new Lazy>>(() => typedTraits)); } private static void AddTrait(Dictionary> traits, string key, string value) From 89cb4f6f1dbe4f37b810a2a2d92428619490983c Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 14:55:22 -0700 Subject: [PATCH 035/105] Stop suppressing xUnit1031 repo-wide The migration suppressed xUnit1031 and xUnit1051 together in Directory.Build.props and CustomAnalysisRules.Test.ruleset. Building the solution with both enabled shows xUnit1031 has zero violations -- the blocking-task call sites in ThreadSafeListTests and ThreadSafeHashSetTests already carry targeted #pragma disables -- so the global suppression was hiding nothing and would have masked new violations. xUnit1051 stays suppressed: it has ~7,400 call sites, which is far outside the scope of this migration. Documented that in the ruleset so the remaining suppression is deliberate and scoped rather than unexplained. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CustomAnalysisRules.Test.ruleset | 5 ++++- Directory.Build.props | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CustomAnalysisRules.Test.ruleset b/CustomAnalysisRules.Test.ruleset index 90f8f7625b..7315d765a7 100644 --- a/CustomAnalysisRules.Test.ruleset +++ b/CustomAnalysisRules.Test.ruleset @@ -130,7 +130,10 @@ - + diff --git a/Directory.Build.props b/Directory.Build.props index 7feace5e46..595e40a247 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -48,7 +48,7 @@ false Exe true - $(NoWarn);xUnit1031;xUnit1051;SA1208;SA1209;SA1518 + $(NoWarn);xUnit1051;SA1208;SA1209;SA1518 From 3133160e2f213a1fae2f7aaebd46122e227548fe Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 15:05:26 -0700 Subject: [PATCH 036/105] Add missing XML documentation to Extensions.Xunit public API .github/copilot-instructions.md requires XML documentation on public members. Seven public/protected constructors added or reshaped by the xUnit v3 migration had none, including the 17-parameter RetryTestCase constructor whose maxRetries clamping and retryOnAssertionFailure semantics are not self-evident. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFramework.cs | 3 +++ .../FixtureArgumentSetsAttribute.cs | 8 +++++++ .../RetryFactAttribute.cs | 5 ++++ .../RetryTestCase.cs | 23 +++++++++++++++++++ .../RetryTheoryAttribute.cs | 5 ++++ .../SkippableFactAttribute.cs | 5 ++++ .../SkippableTheoryAttribute.cs | 5 ++++ 7 files changed, 54 insertions(+) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs index c742fc013f..de8d6ce3ce 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs @@ -17,6 +17,9 @@ namespace Microsoft.Health.Extensions.Xunit /// public sealed class CustomXunitTestFramework : XunitTestFramework { + /// + /// Initializes a new instance of the class. + /// public CustomXunitTestFramework() : base(configFileName: null) { diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs index 42845e289f..a3414e5efb 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs @@ -18,6 +18,14 @@ public abstract class FixtureArgumentSetsAttribute : Attribute { private readonly Enum[] _argumentSets; + /// + /// Initializes a new instance of the class. + /// + /// + /// The flags enum values to pass to the class fixture's constructor. Each distinct enum type + /// contributes one argument, and the test class is expanded into one variant per combination + /// of the flags set within each type. + /// protected FixtureArgumentSetsAttribute(params Enum[] argumentSets) { EnsureArg.IsNotNull(argumentSets, nameof(argumentSets)); diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs index e19efa1824..9df4aa4510 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs @@ -18,6 +18,11 @@ namespace Microsoft.Health.Extensions.Xunit [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class RetryFactAttribute : FactAttribute { + /// + /// Initializes a new instance of the class. + /// + /// The source file containing the test method. Supplied by the compiler. + /// The line number of the test method. Supplied by the compiler. public RetryFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) : base(sourceFilePath, sourceLineNumber) { diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 7c44bbb597..b1745d505a 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -25,12 +25,35 @@ public sealed class RetryTestCase : XunitTestCase, ISelfExecutingXunitTestCase private int _delayMs; private bool _retryOnAssertionFailure; + /// + /// Initializes a new instance of the class. Used only by the deserializer. + /// [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] public RetryTestCase() { } + /// + /// Initializes a new instance of the class. + /// + /// The test method this test case belongs to. + /// The display name reported for the test case. + /// The unique ID of the test case. + /// Whether the test case is only run when explicitly selected. + /// Exception types that cause the test to be reported as skipped rather than failed. + /// The static reason the test is skipped, or null if it is not skipped. + /// The type containing the member named by or . + /// The name of a property that must be true for the test to run. + /// The name of a property that must be false for the test to run. + /// The traits associated with the test case. + /// The arguments passed to the test method. + /// The source file containing the test method. + /// The line number of the test method. + /// The per-attempt timeout in milliseconds, or null for none. + /// The maximum number of attempts. Values below one are clamped to one. + /// The delay in milliseconds between attempts. + /// Whether assertion failures should be retried, rather than only non-assertion exceptions. public RetryTestCase( IXunitTestMethod testMethod, string displayName, diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs index 5006abc022..8527d117e5 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs @@ -18,6 +18,11 @@ namespace Microsoft.Health.Extensions.Xunit [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class RetryTheoryAttribute : TheoryAttribute { + /// + /// Initializes a new instance of the class. + /// + /// The source file containing the test method. Supplied by the compiler. + /// The line number of the test method. Supplied by the compiler. public RetryTheoryAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) : base(sourceFilePath, sourceLineNumber) { diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs index ff4e9333da..bafc4b708a 100644 --- a/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs @@ -14,6 +14,11 @@ namespace Xunit [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class SkippableFactAttribute : FactAttribute { + /// + /// Initializes a new instance of the class. + /// + /// The source file containing the test method. Supplied by the compiler. + /// The line number of the test method. Supplied by the compiler. public SkippableFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) : base(sourceFilePath, sourceLineNumber) { diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs index f86834daea..cd0ef69c35 100644 --- a/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs @@ -14,6 +14,11 @@ namespace Xunit [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class SkippableTheoryAttribute : TheoryAttribute { + /// + /// Initializes a new instance of the class. + /// + /// The source file containing the test method. Supplied by the compiler. + /// The line number of the test method. Supplied by the compiler. public SkippableTheoryAttribute([CallerFilePath] string sourceFilePath = "", [CallerLineNumber] int sourceLineNumber = 0) : base(sourceFilePath, sourceLineNumber) { From 877a67152f3215bfdd297d6d46e9b53c316ca2bd Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 15:19:44 -0700 Subject: [PATCH 037/105] Stop RetryFact losing failures when a run is cancelled Re-review of the earlier RetryFact fix found the same silent-failure class on a second exit path. The retry loop treated the RunSummary of a cancelled attempt (Total=0, Failed=0) as a pass, and the failure deferred from the previous attempt had already been discarded, so the test disappeared from the console summary and the TRX entirely -- not passed, not failed, not skipped. Any cancellation source triggers it: stopOnFail, Ctrl+C or pipeline cancellation, run timeouts, --maximum-failed-tests, or a sibling collection cancelling the shared token. The deferred failure is now carried forward and discarded only once a later attempt has actually reported a result, and an attempt that reports nothing (or a cancelled token) is never treated as success. Task.Delay observes the cancellation token, so a cancelled run no longer sleeps out the remaining retry delays. Dispose replays anything still deferred instead of dropping it, so the failure mode is fail-safe rather than fail-silent, and discarding is now an explicit DiscardDeferredMessages call. The message bus is also wrapped on the final attempt in capture-only mode. It was previously bypassed there, so the one log line an engineer greps for always read "Final error: Test failed but no exception was captured" -- it now carries the real message. Negative retry delays are clamped, matching the MaxRetries clamp. Also corrects a comment (and the reasoning in commit 872507c01) claiming xUnit v3 seals XunitTestMethod. It does not -- these shims derive from it. The members are non-virtual and the state is in private fields with no setter, which is the actual reason reflection is required and why `new`-hiding silently did nothing. Verified with probe assemblies: under stopOnFail a cancelled RetryFact now reports "total: 2 / failed: 2" with its real exception in the TRX, where it previously vanished leaving "total: 1"; and a five-test matrix (non-retriable assertion, exhausted retries, flaky-then-passes, retriable assertion, plain pass) reports total 5 / failed 3 / succeeded 2 with exactly one TRX entry each. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../FixtureArgumentSetTestMethod.cs | 9 +- .../RetryTestCase.cs | 134 ++++++++++++++---- 2 files changed, 113 insertions(+), 30 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index c590ec9a3d..2514805bb0 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -48,9 +48,14 @@ public FixtureArgumentSetTestMethod() #pragma warning restore CS0618 /// - /// Resolves a private field on that this type has to write to - /// because xUnit v3 seals the corresponding members. + /// Resolves a private field on that this type has to write to. /// + /// + /// xUnit v3 holds this state in private fields with no public setter and no virtual member + /// that would let a derived type supply it, so reflection is the only way to set it. The + /// members are not virtual either, which is why hiding them with new does not work: + /// xUnit dispatches through and would never see the override. + /// /// The name of the private instance field. /// The field. /// Thrown when the field no longer exists. diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index b1745d505a..afd28c7ba8 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -77,7 +77,10 @@ public RetryTestCase( // A retry count below one would skip the attempt loop entirely and report no result // at all, silently dropping the test. Always run at least one attempt. _maxRetries = Math.Max(1, maxRetries); - _delayMs = delayMs; + + // A negative delay below -1 makes Task.Delay throw, turning a configuration slip into + // an unrelated crash inside the retry loop. + _delayMs = Math.Max(0, delayMs); _retryOnAssertionFailure = retryOnAssertionFailure; } @@ -105,6 +108,12 @@ public async ValueTask Run( var runSummary = new RunSummary { Total = 1 }; Exception lastException = null; + // Holds the deferred failure of an attempt that is about to be retried. It is discarded + // only once a later attempt has actually reported a result, so that a run cancelled + // between attempts still reports the failure that was already observed instead of + // silently dropping the test. + FailureInterceptingMessageBus pendingFailureBus = null; + for (int attempt = 1; attempt <= _maxRetries; attempt++) { var isLastAttempt = attempt == _maxRetries; @@ -114,26 +123,17 @@ public async ValueTask Run( // Create a fresh aggregator for each attempt var attemptAggregator = new ExceptionAggregator(); - // Only intercept failure messages on non-final attempts - // On the final attempt, let everything go through (both success and failure) - IMessageBus busToUse; FailureInterceptingMessageBus interceptingBus = null; - if (isLastAttempt) - { - busToUse = messageBus; - } - else - { - interceptingBus = new FailureInterceptingMessageBus(messageBus); - busToUse = interceptingBus; - } - try { + // Always wrap the bus so the failure details are available for the CI log, but only + // defer (hide) a failure while a further attempt could still supersede it. + interceptingBus = new FailureInterceptingMessageBus(messageBus, deferFailures: !isLastAttempt); + var summary = await XunitRunnerHelper.RunXunitTestCase( this, - busToUse, + interceptingBus, cancellationTokenSource, attemptAggregator, explicitOption, @@ -141,6 +141,31 @@ public async ValueTask Run( runSummary.Time = summary.Time; + // A cancelled run short-circuits and reports nothing, which is not the same as + // passing. Treating an empty summary as success would drop the test from the + // results entirely, taking any failure already seen on an earlier attempt with it. + if (summary.Total == 0 || cancellationTokenSource.IsCancellationRequested) + { + bool hasObservedFailure = (pendingFailureBus?.HasDeferredFailure ?? false) || interceptingBus.HasDeferredFailure; + + Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' did not complete attempt {attempt}/{_maxRetries} because the run was cancelled. Reporting the last observed result rather than a pass."); + + pendingFailureBus?.ReplayDeferredMessages(); + interceptingBus.ReplayDeferredMessages(); + + if (lastException != null) + { + aggregator.Add(lastException); + } + + runSummary.Failed = hasObservedFailure ? 1 : 0; + return runSummary; + } + + // This attempt reported a real result, so a failure held over from an earlier + // attempt has been superseded and must not also be reported. + pendingFailureBus?.DiscardDeferredMessages(); + if (summary.Failed == 0) { // Test passed - success message already went through to Test Explorer @@ -158,9 +183,9 @@ public async ValueTask Run( // If no exception was captured but test failed, create an exception using captured failure details if (lastException == null && summary.Failed > 0) { - string failureMsg = interceptingBus?.LastFailureMessage ?? "Test failed but no exception was captured."; - string stackTrace = interceptingBus?.LastFailureStackTrace; - bool isAssertionFailure = interceptingBus?.IsAssertionFailure ?? false; + string failureMsg = interceptingBus.LastFailureMessage ?? "Test failed but no exception was captured."; + string stackTrace = interceptingBus.LastFailureStackTrace; + bool isAssertionFailure = interceptingBus.IsAssertionFailure; string fullMessage = failureMsg + (stackTrace != null ? Environment.NewLine + "Stack Trace:" + Environment.NewLine + stackTrace : string.Empty); @@ -198,7 +223,7 @@ public async ValueTask Run( // retrying, so replay it now: the reporters derive their results // solely from the message bus, and without this the failure would // disappear from the test results entirely and the run would pass. - interceptingBus?.ReplayDeferredMessages(); + interceptingBus.ReplayDeferredMessages(); if (lastException != null) { @@ -209,12 +234,40 @@ public async ValueTask Run( return runSummary; } - // Not the last attempt - the failure was intercepted, so retry + // Not the last attempt - the failure was intercepted, so retry. + // Carry the deferred failure forward rather than discarding it now: if the + // run is cancelled during the delay, or the next attempt never reports a + // result, this is the only remaining record that the test failed. + pendingFailureBus = interceptingBus; + + // Ownership moves to pendingFailureBus. Clear the local so this iteration's + // finally does not dispose it, which would replay the failure immediately. + interceptingBus = null; + Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}, retrying after {_delayMs}ms. Error: {lastException?.Message ?? "No exception message"}"); messageBus.QueueMessage( new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}. Retrying after {_delayMs}ms delay. Error: {lastException?.Message ?? "No exception message"}")); - await Task.Delay(_delayMs); + try + { + await Task.Delay(_delayMs, cancellationTokenSource.Token); + } + catch (OperationCanceledException) + { + // Cancelled while waiting to retry. The failure from this attempt is + // still deferred, so report it instead of letting the test vanish. + Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' was cancelled during the retry delay after attempt {attempt}/{_maxRetries}. Reporting the failure from that attempt."); + + pendingFailureBus.ReplayDeferredMessages(); + + if (lastException != null) + { + aggregator.Add(lastException); + } + + runSummary.Failed = 1; + return runSummary; + } } else { @@ -240,11 +293,14 @@ public async ValueTask Run( catch (Exception ex) { Console.WriteLine($"[RetryFact] Unexpected exception while running '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}': {ex.GetType().FullName}: {ex.Message}"); + + // Disposing replays anything still deferred, so a crash cannot swallow a + // failure that an earlier attempt already produced. + pendingFailureBus?.Dispose(); throw; } finally { - // Dispose the intercepting bus if we created one interceptingBus?.Dispose(); } } @@ -267,7 +323,7 @@ protected override void Deserialize(IXunitSerializationInfo info) { base.Deserialize(info); _maxRetries = Math.Max(1, info.GetValue(nameof(_maxRetries))); - _delayMs = info.GetValue(nameof(_delayMs)); + _delayMs = Math.Max(0, info.GetValue(nameof(_delayMs))); _retryOnAssertionFailure = info.GetValue(nameof(_retryOnAssertionFailure)); } @@ -324,12 +380,14 @@ private bool ShouldRetry(Exception ex) private class FailureInterceptingMessageBus : IMessageBus { private readonly IMessageBus _innerBus; + private readonly bool _deferFailures; private readonly List _deferredMessages = new List(); private bool _deferring; - public FailureInterceptingMessageBus(IMessageBus innerBus) + public FailureInterceptingMessageBus(IMessageBus innerBus, bool deferFailures) { _innerBus = innerBus; + _deferFailures = deferFailures; } public string LastFailureMessage { get; private set; } @@ -375,9 +433,12 @@ public bool QueueMessage(IMessageSinkMessage message) failed.ExceptionTypes[0].Contains("TrueException", StringComparison.Ordinal) || failed.ExceptionTypes[0].Contains("FalseException", StringComparison.Ordinal)); - _deferring = true; - _deferredMessages.Add(message); - return true; + if (_deferFailures) + { + _deferring = true; + _deferredMessages.Add(message); + return true; + } } // All other messages (ITestPassed, ITestStarting, ITestFinished, etc.) pass through @@ -400,9 +461,26 @@ public void ReplayDeferredMessages() _deferring = false; } + /// + /// Drops the deferred failure without reporting it. Call this only when a later attempt + /// has actually reported a result that supersedes it. + /// + public void DiscardDeferredMessages() + { + _deferredMessages.Clear(); + _deferring = false; + } + public void Dispose() { - // Don't dispose the inner bus - it's owned by the caller + // Don't dispose the inner bus - it's owned by the caller. + // Anything still deferred was never explicitly replayed or discarded. Dropping it + // would remove the test from the run, so fail safe by reporting it. + if (_deferredMessages.Count > 0) + { + Console.WriteLine($"[RetryFact] Internal error: {_deferredMessages.Count} deferred test message(s) were neither replayed nor discarded. Replaying them so the failure is not lost."); + ReplayDeferredMessages(); + } } } } From ee25af0aec22cc9aa0518d270b69956f4341f17c Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 15:24:29 -0700 Subject: [PATCH 038/105] Fix using order instead of suppressing SA1208/SA1209/SA1518 These three StyleCop rules were disabled for every test project in the repo to accommodate violations this PR introduced: 25 files where the added "using System.Threading.Tasks;" was placed after the Microsoft.*/Xunit usings. Building with the rules enabled shows SA1209 and SA1518 have no violations at all, so those suppressions were pure collateral. Moved the misplaced usings into the System block and removed all three suppressions, rather than permanently lowering the bar for future test code to avoid touching 25 lines. Solution builds clean with the rules enforced. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CustomAnalysisRules.Test.ruleset | 4 ---- Directory.Build.props | 2 +- .../Rest/CompartmentTestFixture.cs | 2 +- .../Rest/Conformance/DocRefOperationTestFixture.cs | 2 +- .../Rest/Conformance/ExpandOperationTestFixture.cs | 2 +- .../Rest/Import/ImportBasicSearchTestFixture.cs | 2 +- .../Rest/Import/ImportCompositeSearchTestFixture.cs | 2 +- .../Rest/Import/ImportDateSearchTestFixture.cs | 2 +- .../Rest/Import/ImportNumberSearchTestFixture.cs | 2 +- .../Rest/Import/ImportQuantitySearchTestFixture.cs | 2 +- .../Rest/Import/ImportReferenceSearchTestFixture.cs | 2 +- .../Rest/Import/ImportStringSearchTestFixture.cs | 2 +- .../Rest/Import/ImportTokenSearchTestFixture.cs | 2 +- .../Rest/Import/ImportUriSearchTestFixture.cs | 2 +- .../Rest/MemberMatchTestFixture.cs | 2 +- .../Rest/Search/CanonicalSearchTestFixture.cs | 2 +- .../Rest/Search/DateSearchTestFixture.cs | 2 +- .../Rest/Search/IncludesOperationTestFixture.cs | 2 +- .../Rest/Search/NotReferencedSearchTestFixture.cs | 2 +- .../Rest/Search/NumberSearchTestFixture.cs | 2 +- .../Rest/Search/QuantitySearchTestFixture.cs | 2 +- .../Rest/Search/ReferenceSearchTestFixture.cs | 2 +- .../Rest/Search/StringSearchTestFixture.cs | 2 +- .../Rest/Search/TokenOverflowTests.cs | 2 +- .../Rest/Search/TokenSearchTestFixture.cs | 2 +- .../Rest/Search/UriSearchTestFixture.cs | 2 +- .../Rest/ValidateTestFixture.cs | 2 +- 27 files changed, 26 insertions(+), 30 deletions(-) diff --git a/CustomAnalysisRules.Test.ruleset b/CustomAnalysisRules.Test.ruleset index 7315d765a7..48edba4a52 100644 --- a/CustomAnalysisRules.Test.ruleset +++ b/CustomAnalysisRules.Test.ruleset @@ -136,8 +136,4 @@ blocking-task call sites use targeted #pragma disables instead. --> - - - - diff --git a/Directory.Build.props b/Directory.Build.props index 595e40a247..e105dc9d77 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -48,7 +48,7 @@ false Exe true - $(NoWarn);xUnit1051;SA1208;SA1209;SA1518 + $(NoWarn);xUnit1051 diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs index b5ad18ad2f..7eb5c4d694 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CompartmentTestFixture.cs @@ -3,10 +3,10 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs index 5771511618..54d0e1268a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/DocRefOperationTestFixture.cs @@ -7,11 +7,11 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Xunit; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Conformance diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs index 8cbcd1e60f..41118b1850 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTestFixture.cs @@ -7,11 +7,11 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Xunit; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Conformance diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs index b922f4fb0f..bd442252a5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportBasicSearchTestFixture.cs @@ -5,9 +5,9 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs index d8947fb014..d960d8dec5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportCompositeSearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs index 6a95c4ff96..d8d1aab03a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportDateSearchTestFixture.cs @@ -5,9 +5,9 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs index 165fdc9cef..6c505280aa 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportNumberSearchTestFixture.cs @@ -5,9 +5,9 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs index 7dba29f12d..78492a19c0 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportQuantitySearchTestFixture.cs @@ -5,9 +5,9 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs index 898afcc08e..2676e5bf11 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportReferenceSearchTestFixture.cs @@ -5,9 +5,9 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs index e676de800e..9a3392f138 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportStringSearchTestFixture.cs @@ -5,9 +5,9 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs index 9db6185d6c..504e583473 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTokenSearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs index f766781da1..2dfced61be 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportUriSearchTestFixture.cs @@ -5,9 +5,9 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Import diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs index 0f71f01a31..afd36799cb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/MemberMatchTestFixture.cs @@ -5,11 +5,11 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Fhir.Web; -using System.Threading.Tasks; namespace Microsoft.Health.Fhir.Tests.E2E.Rest { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs index b9af410287..98f88ff91d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CanonicalSearchTestFixture.cs @@ -4,11 +4,11 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs index d0222c3996..185b52f922 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs index 5f94c13adf..e9debb7f4c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludesOperationTestFixture.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Models; @@ -13,7 +14,6 @@ using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.SqlServer.Management.Smo; using NSubstitute.Core; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs index 684aec0448..858cfdaca1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NotReferencedSearchTestFixture.cs @@ -5,11 +5,11 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs index a06b58dc28..8c10b4e8c7 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/NumberSearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs index b8b910f6e9..a90340595a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/QuantitySearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs index ffb7bf2ef7..d4d72d36cc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ReferenceSearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs index fdb0f41951..aa73d2c48f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/StringSearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs index 2b4b41c983..4c2f88b40c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs @@ -6,6 +6,7 @@ using System; using System.Linq; using System.Net; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; using Microsoft.Health.Core.Extensions; @@ -16,7 +17,6 @@ using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs index df3d9cd99f..cb43ae1b21 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenSearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs index 66d851380c..509d4686e9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/UriSearchTestFixture.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs index 8806d286ea..31ef312adc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ValidateTestFixture.cs @@ -4,11 +4,11 @@ // ------------------------------------------------------------------------------------------------- using System.Collections.Generic; +using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Fhir.Web; -using System.Threading.Tasks; namespace Microsoft.Health.Fhir.Tests.E2E.Rest { From bbb52d8c56ef14d9e9c90a2baff29447d097d71c Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 15:53:53 -0700 Subject: [PATCH 039/105] Stop RetryFact reporting a phantom failure for a test that passed The cancellation guard added for the "test vanishes when cancelled" fix keyed off cancellationTokenSource.IsCancellationRequested as well as an empty RunSummary. Cancellation on its own is the wrong signal: an attempt can run to completion and report its own result through the message bus while cancellation is already requested, for example when another collection trips stopOnFail. In that case the guard fired anyway and replayed the earlier attempt's deferred failure on top of the result already reported, so a test that genuinely passed produced a second, orphaned failure with no display name and inflated the total. Verified with a probe that fails on attempt 1, passes on attempt 2, and has a sibling collection trip stopOnFail during attempt 2: before: total 3, failed 2 (including "failed ") after: total 2, failed 1, succeeded 1 An empty summary remains the signal that an attempt reported nothing, so the cancelled-during-retry case still reports its failure: re-verified at total 2, failed 2 with the real test name and message. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index afd28c7ba8..03acdb9c5f 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -144,11 +144,17 @@ public async ValueTask Run( // A cancelled run short-circuits and reports nothing, which is not the same as // passing. Treating an empty summary as success would drop the test from the // results entirely, taking any failure already seen on an earlier attempt with it. - if (summary.Total == 0 || cancellationTokenSource.IsCancellationRequested) + // + // An empty summary is the only reliable signal that the attempt reported + // nothing. Cancellation on its own is not: an attempt that ran to completion + // while cancellation was requested has already reported its own result through + // the bus, and replaying an earlier attempt's deferred failure on top of that + // adds a second, orphaned result for a test that actually finished. + if (summary.Total == 0) { bool hasObservedFailure = (pendingFailureBus?.HasDeferredFailure ?? false) || interceptingBus.HasDeferredFailure; - Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' did not complete attempt {attempt}/{_maxRetries} because the run was cancelled. Reporting the last observed result rather than a pass."); + Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted). Reporting the last observed result rather than a pass."); pendingFailureBus?.ReplayDeferredMessages(); interceptingBus.ReplayDeferredMessages(); From 07d42075d94230c80560751625d9852d41539584 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 15:54:07 -0700 Subject: [PATCH 040/105] Make xUnit shim fallbacks loud and drop unreachable code Two guards degraded silently when a test case was not an XunitTestCase: RetryTheoryDiscoverer skipped applying retry logic, and the discoverer skipped appending the fixture argument set to the display name, which leaves every variant of a test sharing one name. The theory warning went to Trace, which only an attached debugger sees, so in CI both were invisible. Both now report on Console, consistent with the other RetryFact diagnostics. Reflecting over xunit.v3.core confirms every concrete IXunitTestCase it produces (XunitTestCase, XunitDelayEnumeratedTheoryTestCase, ExecutionErrorTestCase and the Cultured variants) derives from XunitTestCase, so these paths are only reachable via a third-party discoverer. That is why they warn rather than throw: losing retries or a name suffix should not abort a run. Also removed code the same reflection shows can never execute: - StableFallbackTestCaseOrderer, reached only via "DefaultTestCaseOrderer.Instance ?? StableFallbackTestCaseOrderer.Instance", where DefaultTestCaseOrderer.Instance is a non-null static. - A CreateTestClassConstructorArguments override that only called base. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../CustomXunitTestFrameworkDiscoverer.cs | 14 +++++++- .../CustomXunitTestFrameworkExecutor.cs | 34 ++----------------- .../RetryTheoryDiscoverer.cs | 9 +++-- 3 files changed, 22 insertions(+), 35 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index ddc671db81..8e5bf17167 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -160,11 +160,23 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl /// The fixture arguments the test case was discovered for. private static void ApplyVariantDisplayName(ITestCase testCase, SingleFlag[] closedVariant) { - if (closedVariant.Length == 0 || testCase is not XunitTestCase xunitTestCase) + if (closedVariant.Length == 0) { return; } + if (testCase is not XunitTestCase xunitTestCase) + { + // Without the suffix every variant of this method reports under the same name, so + // a failure cannot be attributed to a fixture argument set. Every test case type + // xunit.v3 produces derives from XunitTestCase, so this only happens if a custom + // discoverer introduces its own type -- say so rather than silently mis-naming. + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: Test case '{testCase.TestCaseDisplayName}' is {testCase.GetType().Name}, not {nameof(XunitTestCase)}. " + + "Its fixture argument set will NOT be appended to the display name, so variants of this test will be indistinguishable in test results."); + return; + } + if (TestCaseDisplayNameField == null) { throw new InvalidOperationException( diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index f3a69c2379..7d101773c5 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -35,7 +35,7 @@ private sealed class FixtureArgumentSetAssemblyRunner : XunitTestAssemblyRunner { protected override async ValueTask RunTestCollection(XunitTestAssemblyRunnerContext context, IXunitTestCollection testCollection, IReadOnlyCollection testCases) { - var testCaseOrderer = context.AssemblyTestCaseOrderer ?? DefaultTestCaseOrderer.Instance ?? StableFallbackTestCaseOrderer.Instance; + var testCaseOrderer = context.AssemblyTestCaseOrderer ?? DefaultTestCaseOrderer.Instance; var runner = new FixtureArgumentSetCollectionRunner(); var summary = await runner.Run(testCollection, testCases, context.ExplicitOption, context.MessageBus, testCaseOrderer, context.Aggregator, context.CancellationTokenSource, context.AssemblyFixtureMappings); return summary; @@ -46,7 +46,7 @@ private sealed class FixtureArgumentSetCollectionRunner : XunitTestCollectionRun { protected override async ValueTask RunTestClass(XunitTestCollectionRunnerContext context, IXunitTestClass testClass, IReadOnlyCollection testCases) { - var testCaseOrderer = context.TestCaseOrderer ?? DefaultTestCaseOrderer.Instance ?? StableFallbackTestCaseOrderer.Instance; + var testCaseOrderer = context.TestCaseOrderer ?? DefaultTestCaseOrderer.Instance; var classRunner = new FixtureArgumentSetClassRunner(); var summary = await classRunner.Run(testClass, testCases, context.ExplicitOption, context.MessageBus, testCaseOrderer, context.Aggregator, context.CancellationTokenSource, context.CollectionFixtureMappings); return summary; @@ -111,11 +111,6 @@ protected override ValueTask GetConstructorArgument(XunitTestClassRunner return base.GetConstructorArgument(context, constructor, index, parameter); } - protected override ValueTask CreateTestClassConstructorArguments(XunitTestClassRunnerContext context) - { - return base.CreateTestClassConstructorArguments(context); - } - private static void InjectFixtureArguments(XunitTestClassRunnerContext context) { if (context?.TestClass == null) @@ -262,30 +257,5 @@ private static void MergeTraits(Dictionary> target, IRea } } } - - private sealed class StableFallbackTestCaseOrderer : ITestCaseOrderer - { - internal static ITestCaseOrderer Instance { get; } = new StableFallbackTestCaseOrderer(); - - public IReadOnlyCollection OrderTestCases(IReadOnlyCollection testCases) - where TTestCase : ITestCase - { - ArgumentNullException.ThrowIfNull(testCases); - - return testCases - .OrderBy(testCase => GetSortKey(testCase)) - .ToArray(); - } - - private static string GetSortKey(TTestCase testCase) - { - if (testCase is ITestCaseMetadata metadata) - { - return metadata.TestCaseDisplayName ?? string.Empty; - } - - return testCase?.ToString() ?? string.Empty; - } - } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index 2a00af1e0d..8089ed4943 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -3,8 +3,8 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Xunit; @@ -60,7 +60,12 @@ private static IXunitTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTe { if (testCase is not XunitTestCase xunitTestCase) { - Trace.WriteLine($"[RetryTheory] WARNING: Test case '{testCase.TestCaseDisplayName}' is {testCase.GetType().Name}, not XunitTestCase. Retry logic will NOT be applied."); + // Trace output only reaches an attached debugger, so a Trace-only warning here + // means retries are silently not applied in CI. Every test case type xunit.v3 + // produces derives from XunitTestCase, so this only happens if a custom + // discoverer introduces its own type. + Console.WriteLine( + $"[RetryTheory] WARNING: Test case '{testCase.TestCaseDisplayName}' is {testCase.GetType().Name}, not {nameof(XunitTestCase)}. Retry logic will NOT be applied to it."); return testCase; } From 316feca526d5f48b4173a4808b376d94a8e4fb57 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 15:54:19 -0700 Subject: [PATCH 041/105] Fail the build when a test run publishes no TRX file This PR moves result reporting onto explicit PublishTestResults@2 tasks fed by TRX files, with the dotnet test task's own publishing disabled. PublishTestResults defaults failTaskOnMissingResultsFile to false, so if the TRX glob ever stops matching -- a changed results directory, a runner that does not honour platformOptions__resultDirectory after an upgrade -- the task publishes nothing and still succeeds. failTaskOnFailedTests: true then has no results to act on, and the run looks green with an empty Tests tab. Setting failTaskOnMissingResultsFile: true makes that loud. Every one of these tasks is guarded by condition: succeededOrFailed(), and a run that legitimately executes no tests already fails earlier because Microsoft.Testing.Platform exits 8 on zero tests, so this cannot turn a healthy job red. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/build.yml | 6 ++++++ build/jobs/e2e-tests.yml | 3 +++ build/jobs/run-cosmos-tests.yml | 3 +++ build/jobs/run-export-tests.yml | 6 ++++++ build/jobs/run-sql-tests.yml | 3 +++ 5 files changed, 21 insertions(+) diff --git a/build/jobs/build.yml b/build/jobs/build.yml index 4166fdd9b1..d57e770ec4 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -49,6 +49,9 @@ steps: mergeTestResults: true testRunTitle: 'Unit Tests' failTaskOnFailedTests: true + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and failTaskOnFailedTests has no results to act on. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() - ${{ if eq(parameters.codeCoverage, 'true') }}: @@ -71,6 +74,9 @@ steps: mergeTestResults: true testRunTitle: 'Unit Tests' failTaskOnFailedTests: true + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and failTaskOnFailedTests has no results to act on. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() - task: reportgenerator@5 displayName: 'aggregate unit test coverage' diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index 0dbe9e5988..adac03a70d 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -101,4 +101,7 @@ steps: mergeTestResults: true testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}' failTaskOnFailedTests: true + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and failTaskOnFailedTests has no results to act on. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index 98ccc2c7f8..5635fedaa9 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -121,6 +121,9 @@ jobs: mergeTestResults: true testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' failTaskOnFailedTests: true + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and failTaskOnFailedTests has no results to act on. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() - task: reportgenerator@5 diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index 6f57f34d80..150e7023f7 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -149,6 +149,9 @@ jobs: mergeTestResults: true testRunTitle: 'Export ${{ parameters.version }} CosmosDB' failTaskOnFailedTests: true + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and failTaskOnFailedTests has no results to act on. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() - job: 'sqlE2eTests' @@ -291,5 +294,8 @@ jobs: mergeTestResults: true testRunTitle: 'Export ${{ parameters.version }} SQL' failTaskOnFailedTests: true + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and failTaskOnFailedTests has no results to act on. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 66a867853c..6bb35dc642 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -111,6 +111,9 @@ jobs: mergeTestResults: true testRunTitle: '${{ parameters.version }} SQL Integration Tests' failTaskOnFailedTests: true + # A missing TRX means the run reported nothing; without this the publish step + # succeeds silently and failTaskOnFailedTests has no results to act on. + failTaskOnMissingResultsFile: true condition: succeededOrFailed() - task: reportgenerator@5 From 9ce60dfc5a6162bfc8c782a27298200415bc606d Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 16:41:33 -0700 Subject: [PATCH 042/105] Add regression tests for the xUnit extensions Microsoft.Health.Extensions.Xunit had no tests, which is why three separate bugs survived in it: a retrying test could vanish from the results entirely, a deferred failure could be lost when a run was cancelled, and a test that passed could be reported as an extra failure with no name. None of that is observable from inside the run doing the asserting. It is a property of a whole xUnit run: how many results were published, what they were named, and whether one was published at all. So the scenarios live in a separate assembly that the tests launch as a child process, once per scenario, and assert on the TRX it produces. The assets are launched through Microsoft Testing Platform because that is the runner CI uses; reporters differ in how they handle a result published for a test they have already finished. They are kept out of the solution and marked as a non-test project so that neither Test Explorer nor a solution-wide `dotnet test` runs their deliberate failures. Each test was checked by reverting the fix it covers and confirming it fails. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Microsoft.Health.Fhir.sln | 7 + .../AssemblyInfo.cs | 11 + .../Directory.Build.props | 15 ++ ....Health.Extensions.Xunit.TestAssets.csproj | 33 +++ .../CancellationTriggerTests.cs | 28 +++ .../PassesWhileCancellingTests.cs | 41 ++++ .../CancellationTriggerTests.cs | 28 +++ .../DeferredFailureTests.cs | 34 +++ .../AssetArgumentSetsAttribute.cs | 22 ++ .../FixtureVariants/AssetDataStore.cs | 27 +++ .../Scenarios/FixtureVariants/AssetFixture.cs | 28 +++ .../FixtureVariants/FixtureVariantTests.cs | 41 ++++ .../RetryOutcomes/RetryOutcomeTests.cs | 84 +++++++ .../FixtureVariantExpansionTests.cs | 38 ++++ ...t.Health.Extensions.Xunit.UnitTests.csproj | 40 ++++ .../RetryFactCancellationTests.cs | 59 +++++ .../RetryFactOutcomeTests.cs | 43 ++++ .../TestAssetResult.cs | 17 ++ .../TestAssetRun.cs | 69 ++++++ .../TestAssetRunAssertions.cs | 72 ++++++ .../TestAssetRunner.cs | 209 ++++++++++++++++++ 21 files changed, 946 insertions(+) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Directory.Build.props create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetArgumentSetsAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetDataStore.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetFixture.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/FixtureVariantExpansionTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetResult.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index 970f580b03..ba3e55cbb6 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -215,6 +215,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.R5.Fi EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Ignixa", "src\Microsoft.Health.Fhir.Ignixa\Microsoft.Health.Fhir.Ignixa.csproj", "{87B98ECB-D37E-4DD7-A51C-1AE7CB8765F7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Extensions.Xunit.UnitTests", "src\Microsoft.Health.Extensions.Xunit.UnitTests\Microsoft.Health.Extensions.Xunit.UnitTests.csproj", "{43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -517,6 +519,10 @@ Global {87B98ECB-D37E-4DD7-A51C-1AE7CB8765F7}.Debug|Any CPU.Build.0 = Debug|Any CPU {87B98ECB-D37E-4DD7-A51C-1AE7CB8765F7}.Release|Any CPU.ActiveCfg = Release|Any CPU {87B98ECB-D37E-4DD7-A51C-1AE7CB8765F7}.Release|Any CPU.Build.0 = Release|Any CPU + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}.Debug|Any CPU.Build.0 = Debug|Any CPU + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}.Release|Any CPU.ActiveCfg = Release|Any CPU + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -537,6 +543,7 @@ Global {C6759C03-6060-44D7-B44D-0BD89908B741} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} {87849B3F-5D12-41CA-A082-FAC065EF9FD8} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {4DDF9576-E22C-460A-937F-0EE0FEA6DB87} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} + {43708DAB-AB99-4F6B-8A6F-7AAE94C7C720} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} {E02E5224-32CD-490F-B1E5-8509AD669334} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {070759A9-51D9-4967-8651-39CCA8288C93} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} {D933AC8A-3CE5-4F03-AEE4-9AC0D0274D80} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs new file mode 100644 index 0000000000..e576b6fa90 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs @@ -0,0 +1,11 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit; +using Xunit; + +// Exercise the same custom framework the real test projects use, so these assets +// cover the discoverer and executor shims and not just the retry test case. +[assembly: TestFramework(typeof(CustomXunitTestFramework))] diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Directory.Build.props b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Directory.Build.props new file mode 100644 index 0000000000..6f4b5a0b30 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Directory.Build.props @@ -0,0 +1,15 @@ + + + + + false + + + + + diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj new file mode 100644 index 0000000000..f8aee0e228 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj @@ -0,0 +1,33 @@ + + + + + Exe + true + false + $(NoWarn);xUnit1051 + + + + + + + + + + + + + diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs new file mode 100644 index 0000000000..76e4e09e8a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt +{ + /// + /// Fails partway through the retrying test's second attempt, tripping --stop-on-fail so the + /// run is cancelled while is still executing. + /// + [Collection("CancellationTrigger")] + public class CancellationTriggerTests + { + /// + /// Deliberately fails to trigger cancellation of the whole run. + /// + [Fact] + public void FailsToTriggerCancellation() + { + Thread.Sleep(800); + Assert.Fail("ASSET: deliberate failure that cancels the run"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs new file mode 100644 index 0000000000..0446309681 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt +{ + /// + /// An attempt that runs to completion while cancellation is already requested has reported + /// its own result, so an earlier attempt's deferred failure must not also be reported. + /// + /// + /// Run this namespace with --stop-on-fail on. When cancellation alone was treated as "the + /// attempt reported nothing", the earlier failure was replayed on top of the pass, + /// producing a second orphaned result with no display name and inflating the run total. + /// + [Collection("PassesWhileCancelling")] + public class PassesWhileCancellingTests + { + private static int _attempts; + + /// + /// Fails on the first attempt, then stays alive long enough on the second attempt for + /// the sibling collection to cancel the run, and passes anyway. + /// + [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = 50, RetryOnAssertionFailure = true)] + public void PassingAttemptIsNotOverriddenByEarlierFailure() + { + if (Interlocked.Increment(ref _attempts) == 1) + { + Assert.Fail("ASSET: first attempt fails so a failure is deferred"); + } + + // Stay in the second attempt while the sibling collection cancels the run. + Thread.Sleep(3000); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs new file mode 100644 index 0000000000..23fca9965a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay +{ + /// + /// Fails shortly after the run starts so that --stop-on-fail cancels the run while + /// is still waiting to retry. + /// + [Collection("CancellationTrigger")] + public class CancellationTriggerTests + { + /// + /// Deliberately fails to trigger cancellation of the whole run. + /// + [Fact] + public void FailsToTriggerCancellation() + { + Thread.Sleep(700); + Assert.Fail("ASSET: deliberate failure that cancels the run"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs new file mode 100644 index 0000000000..0fdedee152 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay +{ + /// + /// A retrying test whose failure is deferred must still be reported when the run is + /// cancelled before a later attempt can supersede it. + /// + /// + /// Run this namespace with --stop-on-fail on, so the sibling collection cancels the run + /// while this test is waiting to retry. Before the deferred failure was carried across + /// attempts, this test vanished from the results entirely -- neither passed, failed nor + /// skipped -- and the run reported success. + /// + [Collection("DeferredFailure")] + public class DeferredFailureTests + { + /// + /// Fails on every attempt. The long delay guarantees the run is cancelled while this + /// test is waiting to retry, so the deferred first-attempt failure is the only record + /// that it ever ran. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 30000, RetryOnAssertionFailure = true)] + public void FailureSurvivesCancellationDuringRetryDelay() + { + Assert.Fail("ASSET: failure that must survive cancellation"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetArgumentSetsAttribute.cs new file mode 100644 index 0000000000..0c1deb12dc --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetArgumentSetsAttribute.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants +{ + /// + /// Declares the fixture argument sets a test class is expanded over. + /// + public sealed class AssetArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The data stores to expand the test class over. + public AssetArgumentSetsAttribute(AssetDataStore dataStore) + : base(dataStore) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetDataStore.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetDataStore.cs new file mode 100644 index 0000000000..af4cd3842c --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetDataStore.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants +{ + /// + /// The data stores a fixture variant can be built for. Fixture argument sets are required + /// to be flags enums. + /// + [Flags] + public enum AssetDataStore + { + /// + /// SQL Server. + /// + Sql = 1, + + /// + /// Cosmos DB. + /// + Cosmos = 2, + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetFixture.cs new file mode 100644 index 0000000000..a4c9603fb2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/AssetFixture.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants +{ + /// + /// A class fixture whose constructor argument is supplied by the fixture argument set, + /// which is what the custom executor injects. + /// + public class AssetFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The data store this variant was constructed for. + public AssetFixture(AssetDataStore dataStore) + { + DataStore = dataStore; + } + + /// + /// Gets the data store this variant was constructed for. + /// + public AssetDataStore DataStore { get; } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs new file mode 100644 index 0000000000..686cd24816 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants +{ + /// + /// Expanded into one variant per data store. Each variant must report under a distinct + /// display name, otherwise a failure cannot be attributed to a fixture argument set. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class FixtureVariantTests : IClassFixture + { + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public FixtureVariantTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// Passes for every variant. The harness asserts on the reported display names, which + /// must carry the fixture argument set as a suffix, and on the fixture actually + /// receiving its argument. + /// + [Fact] + public void EachVariantIsReportedUnderItsOwnName() + { + Assert.True( + _fixture.DataStore == AssetDataStore.Sql || _fixture.DataStore == AssetDataStore.Cosmos, + $"ASSET: fixture received an unexpected data store: {_fixture.DataStore}"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs new file mode 100644 index 0000000000..b5797349a5 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs @@ -0,0 +1,84 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryOutcomes +{ + /// + /// The full outcome matrix, run without cancellation. + /// Expected: 6 results, 3 failed and 3 passed, with exactly one result per test. + /// + public class RetryOutcomeTests + { + private static int _flakyAttempts; + private static int _clampedAttempts; + + /// + /// An assertion failure with retries disabled must be reported once and not retried. + /// This is the case that originally disappeared from the results entirely, because the + /// failure was deferred for a retry that then never happened. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = false)] + public void NonRetriableAssertionFailure_IsReportedOnce() + { + Assert.Fail("ASSET: non-retriable assertion failure"); + } + + /// + /// A test that fails every attempt must be reported as failed once, after the retries + /// are exhausted, rather than losing the final attempt's failure. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + public void ExhaustedRetries_IsReportedFailedOnce() + { + Assert.Fail("ASSET: fails on every attempt"); + } + + /// + /// A test that fails once then succeeds must report a single pass, with the superseded + /// failure discarded rather than also reported. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + public void FlakyThenPasses_IsReportedPassedOnce() + { + if (Interlocked.Increment(ref _flakyAttempts) == 1) + { + Assert.Fail("ASSET: first attempt fails, second succeeds"); + } + } + + /// + /// Non-assertion exceptions are retried regardless of RetryOnAssertionFailure, and must + /// still be reported once the attempts are exhausted. + /// + [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = 10)] + public void NonAssertionException_IsReportedFailedOnce() + { + throw new InvalidOperationException("ASSET: transient-style exception"); + } + + /// + /// A plain passing test must be reported as passed. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void AlwaysPasses_IsReportedPassed() + { + Assert.True(true); + } + + /// + /// A MaxRetries below one would skip the attempt loop and report nothing, and a negative + /// delay would make Task.Delay throw. Both are clamped, so the test runs exactly once. + /// + [RetryFact(MaxRetries = 0, DelayBetweenRetriesMs = -500, RetryOnAssertionFailure = true)] + public void ClampedRetryConfiguration_RunsExactlyOnce() + { + Assert.Equal(1, Interlocked.Increment(ref _clampedAttempts)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/FixtureVariantExpansionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FixtureVariantExpansionTests.cs new file mode 100644 index 0000000000..d809641bad --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FixtureVariantExpansionTests.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a test class carrying fixture argument sets is expanded into one variant per + /// argument set, and that each variant reports under its own display name. + /// + public class FixtureVariantExpansionTests + { + private const string TestName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants.FixtureVariantTests.EachVariantIsReportedUnderItsOwnName"; + + /// + /// Without the argument set suffix every variant reports under the same name, so the + /// variants collide in the results and a failure cannot be attributed to the data store + /// that produced it. + /// + [Fact] + public void GivenAClassWithFixtureArgumentSets_WhenTheRunCompletes_ThenEachVariantIsNamedAfterItsArgumentSet() + { + TestAssetRun run = TestAssetRunner.Run("FixtureVariants"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [TestName + " (Sql)"] = "Passed", + [TestName + " (Cosmos)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj b/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj new file mode 100644 index 0000000000..7f5cc418ea --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj @@ -0,0 +1,40 @@ + + + + + $(MSBuildThisFileDirectory)..\Microsoft.Health.Extensions.Xunit.TestAssets\bin\$(Configuration)\$(TargetFramework)\Microsoft.Health.Extensions.Xunit.TestAssets.dll + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + <_Parameter1>TestAssetsAssembly + <_Parameter2>$(TestAssetsAssembly) + + + + diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs new file mode 100644 index 0000000000..9cf9305b44 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies how resolves a deferred failure when the run is + /// cancelled part way through, which is where a retrying test can either lose a real failure + /// or invent one that never happened. + /// + public class RetryFactCancellationTests + { + private const string RetryDelayPrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay."; + private const string PassingAttemptPrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt."; + + /// + /// A failing attempt's result is held back in case a later attempt supersedes it. If the + /// run is cancelled during the retry delay there is no later attempt, so the held-back + /// failure is the only record the test ever ran and must still be published. + /// + [Fact] + public void GivenARetryingTest_WhenTheRunIsCancelledDuringTheRetryDelay_ThenTheDeferredFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("CancelledDuringRetryDelay", stopOnFail: true); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [RetryDelayPrefix + "CancellationTriggerTests.FailsToTriggerCancellation"] = "Failed", + [RetryDelayPrefix + "DeferredFailureTests.FailureSurvivesCancellationDuringRetryDelay"] = "Failed", + }); + } + + /// + /// An attempt can run to completion while cancellation is already requested. It has + /// published its own result, so the earlier attempt's deferred failure must be discarded: + /// republishing it would add a second, unattributable result for a test that passed. + /// + [Fact] + public void GivenARetryingTest_WhenAnAttemptPassesWhileCancellationIsRequested_ThenOnlyThePassIsReported() + { + TestAssetRun run = TestAssetRunner.Run("CancelledDuringPassingAttempt", stopOnFail: true); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [PassingAttemptPrefix + "CancellationTriggerTests.FailsToTriggerCancellation"] = "Failed", + [PassingAttemptPrefix + "PassesWhileCancellingTests.PassingAttemptIsNotOverriddenByEarlierFailure"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs new file mode 100644 index 0000000000..0120875a3c --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that publishes exactly one result per test, with + /// the outcome of the attempt that decided it. + /// + public class RetryFactOutcomeTests + { + private const string Prefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryOutcomes.RetryOutcomeTests."; + + /// + /// Covers the outcome matrix: a non-retriable failure, an exhausted retry, a flaky test + /// that eventually passes, a non-assertion exception, a plain pass, and a clamped + /// configuration. Every one of these must appear exactly once, and the failures must not + /// be swallowed by the retry bookkeeping. + /// + [Fact] + public void GivenRetryingTests_WhenTheRunCompletes_ThenEachTestIsReportedExactlyOnce() + { + TestAssetRun run = TestAssetRunner.Run("RetryOutcomes"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "NonRetriableAssertionFailure_IsReportedOnce"] = "Failed", + [Prefix + "ExhaustedRetries_IsReportedFailedOnce"] = "Failed", + [Prefix + "NonAssertionException_IsReportedFailedOnce"] = "Failed", + [Prefix + "FlakyThenPasses_IsReportedPassedOnce"] = "Passed", + [Prefix + "AlwaysPasses_IsReportedPassed"] = "Passed", + [Prefix + "ClampedRetryConfiguration_RunsExactlyOnce"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetResult.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetResult.cs new file mode 100644 index 0000000000..86db377a3b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetResult.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// A single test result reported by a test asset run, as recorded in the TRX report. + /// + /// + /// The fully qualified display name the result was published under, or null when the + /// runner published a result it could not attribute to a test. + /// + /// The TRX outcome, for example Passed or Failed. + public sealed record TestAssetResult(string Name, string Outcome); +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs new file mode 100644 index 0000000000..1d8f8d5da4 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs @@ -0,0 +1,69 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// The outcome of running one test asset scenario in a child process. + /// + public sealed class TestAssetRun + { + /// + /// Initializes a new instance of the class. + /// + /// The exit code of the runner process. + /// The combined standard output and standard error of the runner. + /// The results the runner published to its TRX report. + /// + /// The number of runner-level errors reported, which are counted separately from failed + /// tests and do not appear as results. + /// + public TestAssetRun(int exitCode, string output, IReadOnlyList results, int errorCount) + { + ExitCode = exitCode; + Output = output; + Results = results; + ErrorCount = errorCount; + } + + /// + /// Gets the exit code of the runner process. + /// + public int ExitCode { get; } + + /// + /// Gets the combined standard output and standard error of the runner, for diagnostics. + /// + public string Output { get; } + + /// + /// Gets the results the runner published to its TRX report. + /// + public IReadOnlyList Results { get; } + + /// + /// Gets the number of runner-level errors reported. These are counted separately from + /// failed tests and are not published as results, so a run can be wrong in ways that the + /// result list alone does not show. + /// + public int ErrorCount { get; } + + /// + /// Renders the run for use in assertion messages. + /// + /// The exit code, every result, and the runner output. + public override string ToString() + { + var lines = Results.Select(r => $" {r.Outcome} :: {r.Name ?? ""}"); + return $"exit code {ExitCode}, {Results.Count} result(s), {ErrorCount} runner error(s):{Environment.NewLine}" + + string.Join(Environment.NewLine, lines) + + $"{Environment.NewLine}--- runner output ---{Environment.NewLine}{Output}"; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs new file mode 100644 index 0000000000..525fb22611 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs @@ -0,0 +1,72 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Assertions over what a test asset run published. + /// + public static class TestAssetRunAssertions + { + /// + /// Asserts that a run published exactly the expected results and nothing else. + /// + /// + /// Comparing the whole result set, rather than counting outcomes, is what catches the + /// failure modes these tests exist for: a result that was never published, a result + /// published twice, and a result published without a display name because it was + /// attributed to a test the runner had already finished. + /// + /// The run to assert on. + /// + /// The expected results, as display name to outcome. Order is not significant. + /// + public static void PublishedExactly(TestAssetRun run, IReadOnlyDictionary expected) + { + EnsureNoRunnerErrors(run); + EnsureNoOrphanedResults(run); + + string[] actualEntries = run.Results + .Select(r => $"{r.Outcome} :: {r.Name}") + .OrderBy(e => e, StringComparer.Ordinal) + .ToArray(); + + string[] expectedEntries = expected + .Select(kvp => $"{kvp.Value} :: {kvp.Key}") + .OrderBy(e => e, StringComparer.Ordinal) + .ToArray(); + + string message = $"The run did not publish the expected results.{Environment.NewLine}" + + $"Expected:{Environment.NewLine} {string.Join(Environment.NewLine + " ", expectedEntries)}{Environment.NewLine}" + + $"Actual: {run}"; + + Assert.True(actualEntries.SequenceEqual(expectedEntries, StringComparer.Ordinal), message); + } + + private static void EnsureNoRunnerErrors(TestAssetRun run) + { + string message = $"The run reported {run.ErrorCount} runner-level error(s). These are counted separately " + + $"from failed tests and are never published as results, so the result list below can look correct " + + $"even though the run went wrong.{Environment.NewLine}Actual: {run}"; + + Assert.True(run.ErrorCount == 0, message); + } + + private static void EnsureNoOrphanedResults(TestAssetRun run) + { + int orphaned = run.Results.Count(r => string.IsNullOrWhiteSpace(r.Name)); + + string message = $"The run published {orphaned} result(s) with no display name, which means a result was " + + $"attributed to a test the runner had already finished.{Environment.NewLine}Actual: {run}"; + + Assert.True(orphaned == 0, message); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs new file mode 100644 index 0000000000..46834ad900 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -0,0 +1,209 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Xml.Linq; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Runs one scenario from the test assets assembly in a child process and reads back the + /// results it published. + /// + /// + /// The behaviours under test -- how many results a run publishes, what they are named, and + /// whether a result is published at all -- are properties of a whole xUnit run, so they cannot + /// be observed from inside the run that is asserting on them. + /// + public static class TestAssetRunner + { + private const string TrxNamespace = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + private const string ScenarioNamespacePrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios."; + + private static readonly TimeSpan RunTimeout = TimeSpan.FromMinutes(2); + + /// + /// Runs a single scenario and returns everything it reported. + /// + /// + /// The scenario folder name, which is also the last segment of the scenario's namespace. + /// + /// + /// Whether to cancel the run as soon as a test fails. Scenarios that exercise cancellation + /// rely on this to cancel the run from inside it. + /// + /// The exit code, output and published results of the run. + public static TestAssetRun Run(string scenario, bool stopOnFail = false) + { + string assetsAssembly = ResolveAssetsAssembly(); + string resultsDirectory = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(resultsDirectory); + + try + { + const string trxFileName = "scenario.trx"; + + // The assets are launched through Microsoft Testing Platform because that is the + // runner CI uses: reporters differ in how they handle a result published for a + // test they have already finished, so asserting against a different runner would + // not say anything about the behaviour that actually ships. + var arguments = new List + { + "exec", + assetsAssembly, + "--filter-namespace", + ScenarioNamespacePrefix + scenario, + "--parallel", + "collections", + "--report-trx", + "--report-trx-filename", + trxFileName, + "--results-directory", + resultsDirectory, + }; + + if (stopOnFail) + { + arguments.Add("--stop-on-fail"); + arguments.Add("on"); + } + + (int exitCode, string output) = Execute(arguments); + + string trxPath = Path.Combine(resultsDirectory, trxFileName); + if (!File.Exists(trxPath)) + { + throw new InvalidOperationException( + $"The '{scenario}' test asset run published no TRX report. Exit code {exitCode}.{Environment.NewLine}{output}"); + } + + return ParseTrx(trxPath, exitCode, output); + } + finally + { + try + { + Directory.Delete(resultsDirectory, recursive: true); + } + catch (IOException) + { + // A leaked temp directory is not worth failing an otherwise good test over. + } + } + } + + private static string ResolveAssetsAssembly() + { + string path = typeof(TestAssetRunner).Assembly + .GetCustomAttributes() + .FirstOrDefault(a => a.Key == "TestAssetsAssembly") + ?.Value; + + if (string.IsNullOrEmpty(path)) + { + throw new InvalidOperationException( + "The TestAssetsAssembly metadata attribute is missing. It is injected by the project file and identifies the assembly holding the scenarios."); + } + + if (!File.Exists(path)) + { + throw new InvalidOperationException( + $"The test assets assembly was not found at '{path}'. It is built by the project reference to Microsoft.Health.Extensions.Xunit.TestAssets."); + } + + return path; + } + + private static (int ExitCode, string Output) Execute(IReadOnlyList arguments) + { + var startInfo = new ProcessStartInfo("dotnet") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + foreach (string argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + // The pipeline sets this so its own test runs land in a known place. The child would + // inherit it and could write its TRX somewhere other than --results-directory, which + // would look exactly like a run that produced no report at all. + startInfo.Environment.Remove("platformOptions__resultDirectory"); + + var output = new StringBuilder(); + using var outputComplete = new ManualResetEventSlim(false); + using var errorComplete = new ManualResetEventSlim(false); + using var process = new Process { StartInfo = startInfo }; + + void Append(DataReceivedEventArgs e, ManualResetEventSlim complete) + { + if (e.Data == null) + { + complete.Set(); + } + else + { + lock (output) + { + output.AppendLine(e.Data); + } + } + } + + process.OutputDataReceived += (_, e) => Append(e, outputComplete); + process.ErrorDataReceived += (_, e) => Append(e, errorComplete); + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + if (!process.WaitForExit((int)RunTimeout.TotalMilliseconds)) + { + process.Kill(entireProcessTree: true); + throw new TimeoutException( + $"A test asset run did not finish within {RunTimeout}.{Environment.NewLine}{output}"); + } + + outputComplete.Wait(RunTimeout); + errorComplete.Wait(RunTimeout); + + lock (output) + { + return (process.ExitCode, output.ToString()); + } + } + + private static TestAssetRun ParseTrx(string trxPath, int exitCode, string output) + { + XNamespace trx = TrxNamespace; + XDocument document = XDocument.Load(trxPath); + + List results = document + .Descendants(trx + "UnitTestResult") + .Select(e => new TestAssetResult((string)e.Attribute("testName"), (string)e.Attribute("outcome"))) + .ToList(); + + // Runner-level errors are counted here and nowhere else: they never become results, so + // a run that mishandled its own bookkeeping still looks tidy in the result list. + int errorCount = document + .Descendants(trx + "Counters") + .Select(e => (int?)e.Attribute("error") ?? 0) + .FirstOrDefault(); + + return new TestAssetRun(exitCode, output, results, errorCount); + } + } +} From a0198573c53b6cfd2585b03f4d70c9b5c494ca41 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 20:47:34 -0700 Subject: [PATCH 043/105] Address multi-model PR review findings for the xUnit v3 extensions Correctness: - RetryTestCase: when an attempt reports no result, replay at most one deferred failure bus so a single test can never publish two results, and stop counting a never-reported test as a pass. - TestAssetRunner: snapshot child-process output under the lock before throwing TimeoutException, fail loudly when a TRX has no error counter, and surface output-drain timeouts instead of silently truncating. Test strength: - Assert the child process exit code agrees with the published results. - Pin the cancellation scenarios: budget the retry-delay test's wall clock and make the passing-attempt asset verify cancellation was actually requested. - Replace the tautological fixture-variant assertion with one that catches a transposed name-to-fixture mapping. - Add PassthroughDiscoveryTests covering the new method-level-attribute passthrough branch in CustomXunitTestFrameworkDiscoverer. Clarity: - Correct the MaxRetries and assembly-fixture docs, describe the Skippable attributes as durable source-compatibility aliases, and simplify RetryTheoryDiscoverer.WrapTestCase. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../PassesWhileCancellingTests.cs | 8 ++- .../FixtureVariants/FixtureVariantTests.cs | 17 ++++-- .../MixedAttributeTests.cs | 53 ++++++++++++++++ .../PassthroughDiscoveryTests.cs | 45 ++++++++++++++ .../RetryFactCancellationTests.cs | 17 ++++++ .../TestAssetRun.cs | 13 +++- .../TestAssetRunAssertions.cs | 19 ++++++ .../TestAssetRunner.cs | 61 +++++++++++++++---- .../CustomXunitTestFramework.cs | 5 +- .../FixtureArgumentSetTestMethod.cs | 2 +- .../RetryFactAttribute.cs | 3 +- .../RetryTestCase.cs | 32 ++++++++-- .../RetryTheoryAttribute.cs | 3 +- .../RetryTheoryDiscoverer.cs | 22 +++---- .../SkippableFactAttribute.cs | 4 +- .../SkippableTheoryAttribute.cs | 4 +- 16 files changed, 260 insertions(+), 48 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MixedFixtureAttributes/MixedAttributeTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs index 0446309681..34670ccaf9 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs @@ -34,8 +34,14 @@ public void PassingAttemptIsNotOverriddenByEarlierFailure() Assert.Fail("ASSET: first attempt fails so a failure is deferred"); } - // Stay in the second attempt while the sibling collection cancels the run. + // Stay in the second attempt while the sibling collection cancels the run, then prove + // that is really what happened. Without this the attempt simply passes on its own and + // the scenario would report the expected result while exercising nothing. Thread.Sleep(3000); + + Assert.True( + TestContext.Current.CancellationToken.IsCancellationRequested, + "ASSET: the run was not cancelled while this attempt was running, so this scenario did not exercise a pass during cancellation."); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs index 686cd24816..4aee5eb2dd 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FixtureVariants/FixtureVariantTests.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using Xunit; namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants @@ -26,16 +27,20 @@ public FixtureVariantTests(AssetFixture fixture) } /// - /// Passes for every variant. The harness asserts on the reported display names, which - /// must carry the fixture argument set as a suffix, and on the fixture actually - /// receiving its argument. + /// Passes for every variant, and ties the reported name to the fixture it actually got. + /// Asserting only that the fixture holds some valid store would hold equally for both + /// variants, so a transposed mapping - or the same value injected into both - would go + /// unnoticed while the display names still looked right. /// [Fact] public void EachVariantIsReportedUnderItsOwnName() { - Assert.True( - _fixture.DataStore == AssetDataStore.Sql || _fixture.DataStore == AssetDataStore.Cosmos, - $"ASSET: fixture received an unexpected data store: {_fixture.DataStore}"); + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.Contains( + $"({_fixture.DataStore})", + displayName, + StringComparison.Ordinal); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MixedFixtureAttributes/MixedAttributeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MixedFixtureAttributes/MixedAttributeTests.cs new file mode 100644 index 0000000000..70a73db739 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MixedFixtureAttributes/MixedAttributeTests.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MixedFixtureAttributes +{ + /// + /// A class carrying no class-level argument sets, where only some methods declare their own. + /// A class shaped like this bypasses the discoverer's fast path yet has no class-level sets to + /// expand, so its undecorated methods take a passthrough path that hands them to xUnit directly. + /// That path has to preserve ordinary discovery, including expanding a theory's data rows: + /// dropping them would silently shrink the suite rather than fail it. + /// + public class MixedAttributeTests + { + /// + /// Declares its own argument sets, so this method is expanded into one variant per store. + /// Its presence is what keeps the class off the wholly-undecorated fast path. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + [Fact] + public void ExpandedMethod() + { + Assert.True(true); + } + + /// + /// Takes the passthrough path. A plain fact must still be discovered exactly once. + /// + [Fact] + public void PassthroughFact() + { + Assert.True(true); + } + + /// + /// Takes the passthrough path while carrying data rows, which must all be discovered. + /// + /// The row value supplied by the data attribute. + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public void PassthroughTheory(int value) + { + Assert.True(value >= 1, "ASSET: unexpected theory row value."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs new file mode 100644 index 0000000000..70323b19fa --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies discovery for a class that declares fixture argument sets on some methods but not on + /// the class itself. Such a class cannot take the fast path that hands wholly undecorated classes + /// straight to xUnit, yet it has no class-level sets to expand either, so its undecorated methods + /// take a passthrough path of their own. + /// + public class PassthroughDiscoveryTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MixedFixtureAttributes.MixedAttributeTests."; + + /// + /// A method on the passthrough path is handed to xUnit with its arguments left unresolved, so + /// that data attributes are still expanded downstream. Resolving them here instead would cost + /// a theory every one of its rows, and losing rows shrinks the suite silently: the run stays + /// green while the cases that would have failed simply never execute. + /// + [Fact] + public void GivenAClassWithArgumentSetsOnOnlySomeMethods_WhenTheRunCompletes_ThenUndecoratedMethodsAreDiscoveredInFull() + { + TestAssetRun run = TestAssetRunner.Run("MixedFixtureAttributes"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "ExpandedMethod (Sql)"] = "Passed", + [ClassName + "ExpandedMethod (Cosmos)"] = "Passed", + [ClassName + "PassthroughFact"] = "Passed", + [ClassName + "PassthroughTheory(value: 1)"] = "Passed", + [ClassName + "PassthroughTheory(value: 2)"] = "Passed", + [ClassName + "PassthroughTheory(value: 3)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs index 9cf9305b44..b241285735 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using Xunit; @@ -18,6 +19,13 @@ public class RetryFactCancellationTests private const string RetryDelayPrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay."; private const string PassingAttemptPrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt."; + /// + /// Comfortably above what a cancelled run needs, and comfortably below the 30s retry delay + /// the scenario would otherwise sit through, so the assertion distinguishes the two without + /// being sensitive to how quickly the run starts up. + /// + private static readonly TimeSpan CancellationBudget = TimeSpan.FromSeconds(20); + /// /// A failing attempt's result is held back in case a later attempt supersedes it. If the /// run is cancelled during the retry delay there is no later attempt, so the held-back @@ -35,6 +43,15 @@ public void GivenARetryingTest_WhenTheRunIsCancelledDuringTheRetryDelay_ThenTheD [RetryDelayPrefix + "CancellationTriggerTests.FailsToTriggerCancellation"] = "Failed", [RetryDelayPrefix + "DeferredFailureTests.FailureSurvivesCancellationDuringRetryDelay"] = "Failed", }); + + // The same failed result would be reported by simply exhausting every retry, so without + // this the scenario would still pass if cancellation never happened. Three attempts + // spaced by the 30s delay cannot complete in anything close to this budget. + string message = $"The run took {run.Duration.TotalSeconds:F1}s, which is long enough to have exhausted " + + $"the retry delays rather than being cancelled during one. The expected results were reported, but " + + $"not for the reason this scenario exists to check.{Environment.NewLine}Actual: {run}"; + + Assert.True(run.Duration < CancellationBudget, message); } /// diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs index 1d8f8d5da4..9d39525524 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs @@ -24,12 +24,14 @@ public sealed class TestAssetRun /// The number of runner-level errors reported, which are counted separately from failed /// tests and do not appear as results. /// - public TestAssetRun(int exitCode, string output, IReadOnlyList results, int errorCount) + /// The wall-clock time the runner process took. + public TestAssetRun(int exitCode, string output, IReadOnlyList results, int errorCount, TimeSpan duration) { ExitCode = exitCode; Output = output; Results = results; ErrorCount = errorCount; + Duration = duration; } /// @@ -54,6 +56,13 @@ public TestAssetRun(int exitCode, string output, IReadOnlyList /// public int ErrorCount { get; } + /// + /// Gets the wall-clock time the runner process took. A scenario that cancels its own run + /// asserts on this to show the run was actually cut short, rather than reaching the same + /// outcome by running every attempt to completion. + /// + public TimeSpan Duration { get; } + /// /// Renders the run for use in assertion messages. /// @@ -61,7 +70,7 @@ public TestAssetRun(int exitCode, string output, IReadOnlyList public override string ToString() { var lines = Results.Select(r => $" {r.Outcome} :: {r.Name ?? ""}"); - return $"exit code {ExitCode}, {Results.Count} result(s), {ErrorCount} runner error(s):{Environment.NewLine}" + return $"exit code {ExitCode}, {Results.Count} result(s), {ErrorCount} runner error(s), took {Duration.TotalSeconds:F1}s:{Environment.NewLine}" + string.Join(Environment.NewLine, lines) + $"{Environment.NewLine}--- runner output ---{Environment.NewLine}{Output}"; } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs index 525fb22611..b4155eb422 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs @@ -15,6 +15,9 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests /// public static class TestAssetRunAssertions { + private const int SuccessExitCode = 0; + private const int FailedTestsExitCode = 2; + /// /// Asserts that a run published exactly the expected results and nothing else. /// @@ -32,6 +35,7 @@ public static void PublishedExactly(TestAssetRun run, IReadOnlyDictionary $"{r.Outcome} :: {r.Name}") @@ -68,5 +72,20 @@ private static void EnsureNoOrphanedResults(TestAssetRun run) Assert.True(orphaned == 0, message); } + + private static void EnsureExitCodeMatchesResults(TestAssetRun run, IReadOnlyDictionary expected) + { + // The published results and the exit code are reported through different paths, so a + // child that writes a correct-looking report and then dies -- in a fixture teardown, + // say -- would otherwise be indistinguishable from a clean run. + bool expectingFailures = expected.Values.Any(outcome => !string.Equals(outcome, "Passed", StringComparison.Ordinal)); + int expectedExitCode = expectingFailures ? FailedTestsExitCode : SuccessExitCode; + + string message = $"The run exited with code {run.ExitCode}, but {expectedExitCode} was expected for these " + + $"results. An unexpected code means the run ended for a reason its published results do not " + + $"show.{Environment.NewLine}Actual: {run}"; + + Assert.True(run.ExitCode == expectedExitCode, message); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index 46834ad900..0a48fb24af 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -77,7 +77,7 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false) arguments.Add("on"); } - (int exitCode, string output) = Execute(arguments); + (int exitCode, string output, TimeSpan duration) = Execute(arguments); string trxPath = Path.Combine(resultsDirectory, trxFileName); if (!File.Exists(trxPath)) @@ -86,7 +86,7 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false) $"The '{scenario}' test asset run published no TRX report. Exit code {exitCode}.{Environment.NewLine}{output}"); } - return ParseTrx(trxPath, exitCode, output); + return ParseTrx(trxPath, exitCode, output, duration); } finally { @@ -94,9 +94,11 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false) { Directory.Delete(resultsDirectory, recursive: true); } - catch (IOException) + catch (Exception e) when (e is IOException or UnauthorizedAccessException) { // A leaked temp directory is not worth failing an otherwise good test over. + // Windows in particular reports a file still held open as an access violation + // rather than an IO error, so both have to be tolerated here. } } } @@ -123,7 +125,7 @@ private static string ResolveAssetsAssembly() return path; } - private static (int ExitCode, string Output) Execute(IReadOnlyList arguments) + private static (int ExitCode, string Output, TimeSpan Duration) Execute(IReadOnlyList arguments) { var startInfo = new ProcessStartInfo("dotnet") { @@ -170,23 +172,46 @@ void Append(DataReceivedEventArgs e, ManualResetEventSlim complete) process.BeginOutputReadLine(); process.BeginErrorReadLine(); + long startedAt = Stopwatch.GetTimestamp(); + if (!process.WaitForExit((int)RunTimeout.TotalMilliseconds)) { process.Kill(entireProcessTree: true); + + // The child is still writing on its background threads, so the buffer has to be + // snapshotted under the same lock its callbacks take. Reading it directly here + // would race those appends and surface as an obscure StringBuilder crash instead + // of the timeout that actually happened. + string captured; + lock (output) + { + captured = output.ToString(); + } + throw new TimeoutException( - $"A test asset run did not finish within {RunTimeout}.{Environment.NewLine}{output}"); + $"A test asset run did not finish within {RunTimeout}.{Environment.NewLine}{captured}"); } - outputComplete.Wait(RunTimeout); - errorComplete.Wait(RunTimeout); + TimeSpan duration = Stopwatch.GetElapsedTime(startedAt); + + // Exit does not imply the redirected streams have been drained. Waiting for the EOF + // signal on each is what makes the captured output complete; if that wait itself times + // out the output is truncated, which would otherwise quietly weaken every diagnostic + // built from it. + bool drained = outputComplete.Wait(RunTimeout) & errorComplete.Wait(RunTimeout); lock (output) { - return (process.ExitCode, output.ToString()); + if (!drained) + { + output.AppendLine("[TestAssetRunner] The child process output was not fully drained before this run was read, so the text above may be truncated."); + } + + return (process.ExitCode, output.ToString(), duration); } } - private static TestAssetRun ParseTrx(string trxPath, int exitCode, string output) + private static TestAssetRun ParseTrx(string trxPath, int exitCode, string output, TimeSpan duration) { XNamespace trx = TrxNamespace; XDocument document = XDocument.Load(trxPath); @@ -198,12 +223,22 @@ private static TestAssetRun ParseTrx(string trxPath, int exitCode, string output // Runner-level errors are counted here and nowhere else: they never become results, so // a run that mishandled its own bookkeeping still looks tidy in the result list. - int errorCount = document + // + // A missing counter is not the same as a count of zero. Defaulting it to zero would + // turn the guard built on it into a no-op the moment the report shape changed, and the + // guard would keep passing while checking nothing. + XAttribute errorAttribute = document .Descendants(trx + "Counters") - .Select(e => (int?)e.Attribute("error") ?? 0) - .FirstOrDefault(); + .Select(e => e.Attribute("error")) + .FirstOrDefault(a => a != null); + + if (errorAttribute == null) + { + throw new InvalidOperationException( + $"The TRX report at '{trxPath}' has no Counters/@error attribute, so runner-level errors cannot be read from it."); + } - return new TestAssetRun(exitCode, output, results, errorCount); + return new TestAssetRun(exitCode, output, results, (int)errorAttribute, duration); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs index de8d6ce3ce..0b5ddcf000 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFramework.cs @@ -12,8 +12,9 @@ namespace Microsoft.Health.Extensions.Xunit /// An XunitTestFramework implementation that allows parameterizing class fixtures with combinations of argument values. /// To use, decorate the test assembly with /// [assembly: TestFramework(typeof(CustomXunitTestFramework))] - /// Also allows an assembly to declare one or more , which are created before any tests - /// are executed and disposed at the end of the test run. + /// Assembly fixtures declared with are created before any tests run and + /// disposed at the end of the run. That is xUnit v3 behaviour inherited from the base framework, not something this + /// class adds: it previously needed a local implementation, which this assembly no longer carries. /// public sealed class CustomXunitTestFramework : XunitTestFramework { diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index 2514805bb0..4285cff326 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -121,7 +121,7 @@ internal void UpdateArgumentsFromMethod() var enumValue = _fixtureArguments[i].EnumValue; var enumValueText = enumValue.ToString(); - string key = $"{enumValue.GetType().Name}"; + string key = enumValue.GetType().Name; if (!traits.TryGetValue(key, out var values)) { values = new HashSet(StringComparer.OrdinalIgnoreCase); diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs index 9df4aa4510..bdbac82fad 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs @@ -29,7 +29,8 @@ public RetryFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLi } /// - /// Gets or sets the maximum number of retry attempts (default is 3). + /// Gets or sets the maximum number of attempts, including the first run (default is 3). + /// A value of 3 therefore runs a failing test three times in total, not four. /// public int MaxRetries { get; set; } = 3; diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 03acdb9c5f..a5700915fe 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -156,15 +156,36 @@ public async ValueTask Run( Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted). Reporting the last observed result rather than a pass."); - pendingFailureBus?.ReplayDeferredMessages(); - interceptingBus.ReplayDeferredMessages(); + // Replay at most one bus. A failure this attempt already saw supersedes anything + // held over from an earlier one, and replaying both would publish two results for + // a single test. + if (interceptingBus.HasDeferredFailure) + { + pendingFailureBus?.DiscardDeferredMessages(); + interceptingBus.ReplayDeferredMessages(); + } + else + { + pendingFailureBus?.ReplayDeferredMessages(); + } if (lastException != null) { aggregator.Add(lastException); } - runSummary.Failed = hasObservedFailure ? 1 : 0; + if (hasObservedFailure) + { + runSummary.Failed = 1; + } + else + { + // No attempt ever published a result for this test. Counting it as a pass would + // claim an outcome that was never reported, so contribute nothing to the totals. + runSummary.Total = 0; + runSummary.Failed = 0; + } + return runSummary; } @@ -288,7 +309,10 @@ public async ValueTask Run( } else if (summary.Failed > 0) { - // Add an exception with captured failure details if test failed but no exception was captured + // Defensive only: a failing attempt always has an exception synthesized + // for it above, so reaching this means that invariant has been broken. + // Reporting something generic is still better than reporting a failure + // with nothing attached to explain it. aggregator.Add(new InvalidOperationException($"Test failed after {_maxRetries} attempts but no exception was captured")); } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs index 8527d117e5..624cb8234e 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs @@ -29,7 +29,8 @@ public RetryTheoryAttribute([CallerFilePath] string sourceFilePath = "", [Caller } /// - /// Gets or sets the maximum number of retry attempts (default is 3). + /// Gets or sets the maximum number of attempts, including the first run (default is 3). + /// A value of 3 therefore runs a failing test three times in total, not four. /// public int MaxRetries { get; set; } = 3; diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index 8089ed4943..e1f614f5e0 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -29,13 +29,9 @@ protected override async ValueTask> CreateTe { var attribute = (RetryTheoryAttribute)theoryAttribute; - var maxRetries = attribute.MaxRetries; - var delayMs = attribute.DelayBetweenRetriesMs; - var retryOnAssertionFailure = attribute.RetryOnAssertionFailure; - var baseCases = await base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments); return baseCases - .Select(testCase => WrapTestCase(testMethod, testCase, attribute, maxRetries, delayMs, retryOnAssertionFailure)) + .Select(testCase => WrapTestCase(testMethod, testCase, attribute)) .ToArray(); } @@ -46,17 +42,13 @@ protected override async ValueTask> CreateTe { var attribute = (RetryTheoryAttribute)theoryAttribute; - var maxRetries = attribute.MaxRetries; - var delayMs = attribute.DelayBetweenRetriesMs; - var retryOnAssertionFailure = attribute.RetryOnAssertionFailure; - var baseCases = await base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute); return baseCases - .Select(testCase => WrapTestCase(testMethod, testCase, attribute, maxRetries, delayMs, retryOnAssertionFailure)) + .Select(testCase => WrapTestCase(testMethod, testCase, attribute)) .ToArray(); } - private static IXunitTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTestCase testCase, RetryTheoryAttribute attribute, int maxRetries, int delayMs, bool retryOnAssertionFailure) + private static IXunitTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTestCase testCase, RetryTheoryAttribute attribute) { if (testCase is not XunitTestCase xunitTestCase) { @@ -79,14 +71,14 @@ private static IXunitTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTe xunitTestCase.SkipType, xunitTestCase.SkipUnless, xunitTestCase.SkipWhen, - new Dictionary>(xunitTestCase.Traits.ToDictionary(kvp => kvp.Key, kvp => new HashSet(kvp.Value))), + xunitTestCase.Traits.ToDictionary(kvp => kvp.Key, kvp => new HashSet(kvp.Value)), xunitTestCase.TestMethodArguments, xunitTestCase.SourceFilePath, xunitTestCase.SourceLineNumber, xunitTestCase.Timeout == 0 ? null : xunitTestCase.Timeout, - maxRetries, - delayMs, - retryOnAssertionFailure); + attribute.MaxRetries, + attribute.DelayBetweenRetriesMs, + attribute.RetryOnAssertionFailure); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs index bafc4b708a..b327ae1a35 100644 --- a/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableFactAttribute.cs @@ -9,7 +9,9 @@ namespace Xunit { /// - /// Compatibility attribute for legacy SkippableFact usage (temporary shim to reduce PR review churn). + /// Compatibility attribute for legacy SkippableFact usage. In xUnit v3 a dynamically skipped test is expressed + /// through Assert.Skip rather than a separate attribute, so this is a source-compatibility alias for the plain + /// fact attribute and exists only so existing call sites keep compiling. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class SkippableFactAttribute : FactAttribute diff --git a/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs index cd0ef69c35..2d22200042 100644 --- a/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/SkippableTheoryAttribute.cs @@ -9,7 +9,9 @@ namespace Xunit { /// - /// Compatibility attribute for legacy SkippableTheory usage (temporary shim to reduce PR review churn). + /// Compatibility attribute for legacy SkippableTheory usage. In xUnit v3 a dynamically skipped test is expressed + /// through Assert.Skip rather than a separate attribute, so this is a source-compatibility alias for the plain + /// theory attribute and exists only so existing call sites keep compiling. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public sealed class SkippableTheoryAttribute : TheoryAttribute From fcf793787b85e69ea050e313b115709d6150f378 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 21:37:56 -0700 Subject: [PATCH 044/105] Harden the xUnit v3 extensions against round-2 review findings Correctness: - TestAssetRunner: a timed-out run disposed its completion events while the child was still writing, so the EOF callback could set a disposed event and take down the test host from a thread pool thread. Also tolerate the child exiting between the wait giving up and the kill landing, which would otherwise replace the timeout report with an unrelated exception. - RetryTestCase: the fail-safe replay in Dispose no longer masks an exception that is already unwinding. - FixtureArgumentSetTestMethod: drop TryGetFixtureTraitName, which hardcoded a FHIR namespace into a general-purpose library to add traits the enum type name already produced. Test strength: - RetryTheory had no coverage at all. Add a scenario whose rows retry independently, pinning per-row discovery and the retry wrapper. - The no-result accounting cannot be reached deterministically from a scenario, so express it as two pure helpers and pin every combination directly. Both new suites were mutation-tested. - TestAssetRunAssertions: a skipped test is not a failure, so it must not expect the failure exit code. Clarity: - Record why fixture argument variants cannot round-trip through xUnit serialization: the base Serialize and Deserialize members are not virtual. - Drop assertion-exception checks already covered by the Xunit namespace check, and a constructor call that re-set an argument array to its existing value. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryTheoryOutcomeTests.cs | 45 +++++++ .../RetryTestCaseNoResultTests.cs | 118 ++++++++++++++++++ .../RetryTheoryOutcomeTests.cs | 40 ++++++ .../TestAssetRunAssertions.cs | 7 +- .../TestAssetRunner.cs | 24 +++- .../FixtureArgumentSetTestClass.cs | 13 ++ .../FixtureArgumentSetTestMethod.cs | 46 ------- .../Microsoft.Health.Extensions.Xunit.csproj | 6 + .../NoResultOutcome.cs | 34 +++++ .../RetryTestCase.cs | 104 ++++++++++----- 10 files changed, 357 insertions(+), 80 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTheoryOutcomes/RetryTheoryOutcomeTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit/NoResultOutcome.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTheoryOutcomes/RetryTheoryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTheoryOutcomes/RetryTheoryOutcomeTests.cs new file mode 100644 index 0000000000..8e2507ae11 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTheoryOutcomes/RetryTheoryOutcomeTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Concurrent; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryTheoryOutcomes +{ + /// + /// The outcome matrix. Expected: 3 results, one per data row, + /// with the rows that recover reported as passed and the row that never recovers reported as + /// failed. + /// + /// + /// Each row keeps its own attempt count, which only works if every row is discovered as its own + /// test case and wrapped individually. A discoverer that collapsed the rows would either share + /// the counter or lose rows outright, and both show up in the published results. + /// + public class RetryTheoryOutcomeTests + { + private static readonly ConcurrentDictionary Attempts = new ConcurrentDictionary(); + + /// + /// Rows that recover on a later attempt must be reported as passed exactly once, and the row + /// that fails every attempt must be reported as failed exactly once. + /// + /// Identifies the row so each keeps a separate attempt count. + /// Whether this row should stop failing after its first attempt. + [RetryTheory(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + [InlineData(1, true)] + [InlineData(2, false)] + [InlineData(3, true)] + public void FlakyRow(int row, bool recovers) + { + int attempt = Attempts.AddOrUpdate(row, 1, (_, current) => current + 1); + + if (!recovers || attempt == 1) + { + Assert.Fail($"ASSET: row {row} failing on attempt {attempt}"); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs new file mode 100644 index 0000000000..26aa777026 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs @@ -0,0 +1,118 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; +using Xunit.Sdk; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers how accounts for an attempt that published no result of + /// its own, which happens when the run is cancelled or aborted underneath it. + /// + /// + /// This decision cannot be reached deterministically from a scenario asset, because it needs the + /// runner to abandon an attempt mid-flight at an exact moment. The decision is therefore made by + /// two pure helpers so that every combination can be pinned directly. Without these tests the + /// branch that stops a never-reported test from counting as a pass is unprotected. + /// + public class RetryTestCaseNoResultTests + { + /// + /// A failure seen by the attempt that was cut short supersedes anything an earlier attempt + /// deferred, so only the current attempt's failure is replayed. Replaying both would publish + /// two results for one test. + /// + [Fact] + public void GivenFailuresOnBothTheCurrentAndAnEarlierAttempt_WhenNothingWasReported_ThenOnlyTheCurrentAttemptIsReplayed() + { + NoResultOutcome outcome = RetryTestCase.DecideNoResultOutcome( + currentAttemptObservedFailure: true, + earlierAttemptObservedFailure: true); + + Assert.Equal(NoResultOutcome.ReplayCurrentAttempt, outcome); + } + + /// + /// A failure seen only by the attempt that was cut short is still a real observed failure. + /// + [Fact] + public void GivenAFailureOnTheCurrentAttemptOnly_WhenNothingWasReported_ThenTheCurrentAttemptIsReplayed() + { + NoResultOutcome outcome = RetryTestCase.DecideNoResultOutcome( + currentAttemptObservedFailure: true, + earlierAttemptObservedFailure: false); + + Assert.Equal(NoResultOutcome.ReplayCurrentAttempt, outcome); + } + + /// + /// A failure deferred while waiting for a retry that then reported nothing must still reach + /// the results, rather than being dropped along with the abandoned attempt. + /// + [Fact] + public void GivenAFailureOnAnEarlierAttemptOnly_WhenNothingWasReported_ThenTheEarlierAttemptIsReplayed() + { + NoResultOutcome outcome = RetryTestCase.DecideNoResultOutcome( + currentAttemptObservedFailure: false, + earlierAttemptObservedFailure: true); + + Assert.Equal(NoResultOutcome.ReplayEarlierAttempt, outcome); + } + + /// + /// When no attempt ever saw a failure and none reported a result, there is nothing to replay. + /// + [Fact] + public void GivenNoObservedFailure_WhenNothingWasReported_ThenNothingIsReplayed() + { + NoResultOutcome outcome = RetryTestCase.DecideNoResultOutcome( + currentAttemptObservedFailure: false, + earlierAttemptObservedFailure: false); + + Assert.Equal(NoResultOutcome.ReportNothing, outcome); + } + + /// + /// A test that no attempt ever reported must contribute nothing to the totals. Reporting one + /// test with no failures is what previously turned an abandoned test into a silent pass. + /// + [Fact] + public void GivenNothingToReplay_WhenTheSummaryIsBuilt_ThenTheTestContributesNothing() + { + RunSummary summary = RetryTestCase.CreateNoResultSummary(NoResultOutcome.ReportNothing); + + Assert.Equal(0, summary.Total); + Assert.Equal(0, summary.Failed); + } + + /// + /// A failure replayed from the attempt that was cut short is a published result, so it must + /// be counted as one failed test. + /// + [Fact] + public void GivenTheCurrentAttemptIsReplayed_WhenTheSummaryIsBuilt_ThenTheTestCountsAsOneFailure() + { + RunSummary summary = RetryTestCase.CreateNoResultSummary(NoResultOutcome.ReplayCurrentAttempt); + + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Failed); + } + + /// + /// A failure replayed from an earlier attempt is equally a published result, so it counts the + /// same way. + /// + [Fact] + public void GivenAnEarlierAttemptIsReplayed_WhenTheSummaryIsBuilt_ThenTheTestCountsAsOneFailure() + { + RunSummary summary = RetryTestCase.CreateNoResultSummary(NoResultOutcome.ReplayEarlierAttempt); + + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Failed); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs new file mode 100644 index 0000000000..f5d3cb5d21 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that publishes exactly one result per data row, + /// with the outcome of the attempt that decided that row. + /// + public class RetryTheoryOutcomeTests + { + private const string Prefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryTheoryOutcomes.RetryTheoryOutcomeTests."; + + /// + /// Retries are applied per row, not per method: two rows recover on their second attempt and + /// must be reported as passed, while the row that fails every attempt must be reported as + /// failed. A discoverer that lost rows, shared a single test case between them, or dropped + /// the retry wrapper would change this result set. + /// + [Fact] + public void GivenARetryingTheory_WhenTheRunCompletes_ThenEachRowIsReportedExactlyOnce() + { + TestAssetRun run = TestAssetRunner.Run("RetryTheoryOutcomes"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "FlakyRow(row: 1, recovers: True)"] = "Passed", + [Prefix + "FlakyRow(row: 2, recovers: False)"] = "Failed", + [Prefix + "FlakyRow(row: 3, recovers: True)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs index b4155eb422..0e4aa6216e 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs @@ -78,7 +78,12 @@ private static void EnsureExitCodeMatchesResults(TestAssetRun run, IReadOnlyDict // The published results and the exit code are reported through different paths, so a // child that writes a correct-looking report and then dies -- in a fixture teardown, // say -- would otherwise be indistinguishable from a clean run. - bool expectingFailures = expected.Values.Any(outcome => !string.Equals(outcome, "Passed", StringComparison.Ordinal)); + // + // A skip is not a failure: the runner exits 0 for a run whose tests all passed or were + // skipped, so only a genuinely failing outcome may expect the failure exit code. + bool expectingFailures = expected.Values.Any(outcome => + !string.Equals(outcome, "Passed", StringComparison.Ordinal) + && !string.Equals(outcome, "NotExecuted", StringComparison.Ordinal)); int expectedExitCode = expectingFailures ? FailedTestsExitCode : SuccessExitCode; string message = $"The run exited with code {run.ExitCode}, but {expectedExitCode} was expected for these " diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index 0a48fb24af..3d913ba872 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -154,7 +154,18 @@ void Append(DataReceivedEventArgs e, ManualResetEventSlim complete) { if (e.Data == null) { - complete.Set(); + try + { + complete.Set(); + } + catch (ObjectDisposedException) + { + // The only way this event is disposed while a callback is still in flight is + // that Execute has already abandoned the wait and is unwinding on the timeout + // path. Nothing is listening for the signal any more, and letting it escape + // would take down the whole test host from a thread pool thread, replacing the + // timeout report with an unrelated crash. + } } else { @@ -176,7 +187,16 @@ void Append(DataReceivedEventArgs e, ManualResetEventSlim complete) if (!process.WaitForExit((int)RunTimeout.TotalMilliseconds)) { - process.Kill(entireProcessTree: true); + try + { + process.Kill(entireProcessTree: true); + } + catch (Exception e) when (e is InvalidOperationException or System.ComponentModel.Win32Exception) + { + // The child can exit on its own between the wait giving up and the kill landing. + // Letting that race escape would replace the timeout report -- which carries the + // child's output -- with an unrelated and far less useful exception. + } // The child is still writing on its background threads, so the buffer has to be // snapshotted under the same lock its callbacks take. Reading it directly here diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs index 073ab90282..c3c21d1584 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestClass.cs @@ -13,6 +13,19 @@ namespace Microsoft.Health.Extensions.Xunit { + /// + /// A test class bound to one fixture argument set, so that a single class type can be discovered + /// once per variant with a distinct identity. + /// + /// + /// The fixture arguments are held in memory only. implements + /// IXunitSerializable non-virtually, so a derived type has no way to add its own state to + /// the serialized form, and a runner that discovered tests in one process and ran them in another + /// would hand the executor a plain rather than this type. Both the + /// Microsoft Testing Platform runner used by CI and the Visual Studio test explorer discover and + /// run in the same process, so the variant survives; a runner that did not would fall back to the + /// unvaried class rather than silently mixing variants up. + /// internal sealed class FixtureArgumentSetTestClass : XunitTestClass { private static readonly FieldInfo UniqueIdField = typeof(XunitTestClass).GetField("uniqueID", BindingFlags.Instance | BindingFlags.NonPublic) diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index 4285cff326..2b7493c17d 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -36,7 +36,6 @@ public FixtureArgumentSetTestMethod(FixtureArgumentSetTestClass testClass, Metho _fixtureArguments = fixtureArguments; _uniqueId = uniqueId; - UpdateMethodArguments(Array.Empty()); UniqueIdField.SetValue(this, _uniqueId); MethodField.SetValue(this, _methodInfo); } @@ -129,55 +128,10 @@ internal void UpdateArgumentsFromMethod() } values.Add(enumValueText); - - if (TryGetFixtureTraitName(enumValue, out var traitName)) - { - AddTrait(traits, traitName, enumValueText); - } } var typedTraits = traits.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyCollection)kvp.Value, StringComparer.OrdinalIgnoreCase); TraitsField.SetValue(this, new Lazy>>(() => typedTraits)); } - - private static void AddTrait(Dictionary> traits, string key, string value) - { - if (!traits.TryGetValue(key, out var values)) - { - values = new HashSet(StringComparer.OrdinalIgnoreCase); - traits[key] = values; - } - - values.Add(value); - } - - private static bool TryGetFixtureTraitName(Enum enumValue, out string traitName) - { - traitName = null; - if (enumValue == null) - { - return false; - } - - var enumType = enumValue.GetType(); - if (!string.Equals(enumType.Namespace, "Microsoft.Health.Fhir.Tests.Common.FixtureParameters", StringComparison.Ordinal)) - { - return false; - } - - if (string.Equals(enumType.Name, "DataStore", StringComparison.Ordinal)) - { - traitName = "DataStore"; - return true; - } - - if (string.Equals(enumType.Name, "Format", StringComparison.Ordinal)) - { - traitName = "Format"; - return true; - } - - return false; - } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj b/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj index 8769cf00d2..ffba2980ab 100644 --- a/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj +++ b/src/Microsoft.Health.Extensions.Xunit/Microsoft.Health.Extensions.Xunit.csproj @@ -12,6 +12,12 @@ $(NoWarn);NU5104;NU5100;CA1812;CA1515 + + + <_Parameter1>Microsoft.Health.Extensions.Xunit.UnitTests + + + diff --git a/src/Microsoft.Health.Extensions.Xunit/NoResultOutcome.cs b/src/Microsoft.Health.Extensions.Xunit/NoResultOutcome.cs new file mode 100644 index 0000000000..caa31d5147 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/NoResultOutcome.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit +{ + /// + /// How a retry attempt that published no result of its own should be accounted for. + /// + /// + /// An attempt reports nothing when the run is cancelled or aborted underneath it. That is not + /// the same as passing, and the choice made here decides whether an already observed failure + /// still reaches the results. + /// + internal enum NoResultOutcome + { + /// + /// No attempt ever published a result, so the test contributes nothing to the totals. + /// + ReportNothing, + + /// + /// This attempt observed a failure before it was cut short, so that failure is replayed. + /// + ReplayCurrentAttempt, + + /// + /// An earlier attempt deferred a failure while waiting to be retried, and the retry never + /// reported anything, so the earlier failure is replayed. + /// + ReplayEarlierAttempt, + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index a5700915fe..8f4345c41a 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -152,21 +152,28 @@ public async ValueTask Run( // adds a second, orphaned result for a test that actually finished. if (summary.Total == 0) { - bool hasObservedFailure = (pendingFailureBus?.HasDeferredFailure ?? false) || interceptingBus.HasDeferredFailure; + NoResultOutcome outcome = DecideNoResultOutcome( + currentAttemptObservedFailure: interceptingBus.HasDeferredFailure, + earlierAttemptObservedFailure: pendingFailureBus?.HasDeferredFailure ?? false); Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted). Reporting the last observed result rather than a pass."); - // Replay at most one bus. A failure this attempt already saw supersedes anything - // held over from an earlier one, and replaying both would publish two results for - // a single test. - if (interceptingBus.HasDeferredFailure) + // At most one bus is replayed. A failure this attempt saw supersedes anything held + // over from an earlier one, and replaying both would publish two results for a + // single test. + switch (outcome) { - pendingFailureBus?.DiscardDeferredMessages(); - interceptingBus.ReplayDeferredMessages(); - } - else - { - pendingFailureBus?.ReplayDeferredMessages(); + case NoResultOutcome.ReplayCurrentAttempt: + pendingFailureBus?.DiscardDeferredMessages(); + interceptingBus.ReplayDeferredMessages(); + break; + + case NoResultOutcome.ReplayEarlierAttempt: + pendingFailureBus.ReplayDeferredMessages(); + break; + + default: + break; } if (lastException != null) @@ -174,19 +181,9 @@ public async ValueTask Run( aggregator.Add(lastException); } - if (hasObservedFailure) - { - runSummary.Failed = 1; - } - else - { - // No attempt ever published a result for this test. Counting it as a pass would - // claim an outcome that was never reported, so contribute nothing to the totals. - runSummary.Total = 0; - runSummary.Failed = 0; - } - - return runSummary; + RunSummary noResultSummary = CreateNoResultSummary(outcome); + noResultSummary.Time = runSummary.Time; + return noResultSummary; } // This attempt reported a real result, so a failure held over from an earlier @@ -341,6 +338,41 @@ public async ValueTask Run( return runSummary; } + /// + /// Decides how an attempt that published no result of its own should be accounted for. + /// + /// Whether this attempt saw a failure before it was cut short. + /// Whether an earlier attempt deferred a failure that has not been superseded. + /// Which deferred failure, if any, should reach the results. + internal static NoResultOutcome DecideNoResultOutcome(bool currentAttemptObservedFailure, bool earlierAttemptObservedFailure) + { + if (currentAttemptObservedFailure) + { + return NoResultOutcome.ReplayCurrentAttempt; + } + + return earlierAttemptObservedFailure + ? NoResultOutcome.ReplayEarlierAttempt + : NoResultOutcome.ReportNothing; + } + + /// + /// Builds the summary for an attempt that published no result of its own. + /// + /// The outcome chosen by . + /// The totals this test contributes to the run. + /// + /// A replayed failure is a published result, so it counts as one failed test. When nothing + /// was ever published the test contributes nothing at all: counting it would claim an + /// outcome that no attempt reported, and a total of one with no failures reads as a pass. + /// + internal static RunSummary CreateNoResultSummary(NoResultOutcome outcome) + { + return outcome == NoResultOutcome.ReportNothing + ? new RunSummary { Total = 0, Failed = 0 } + : new RunSummary { Total = 1, Failed = 1 }; + } + protected override void Serialize(IXunitSerializationInfo info) { base.Serialize(info); @@ -453,15 +485,13 @@ public bool QueueMessage(IMessageSinkMessage message) ? string.Join(Environment.NewLine, failed.StackTraces) : null; - // Detect if this is an assertion failure by checking exception types - // XUnit assertion exceptions typically have types containing "Xunit" or "Assert" + // Detect if this is an assertion failure by checking exception types. + // Every xUnit assertion exception lives under the Xunit.Sdk namespace, so the + // namespace check already covers the individual assertion exception types. IsAssertionFailure = failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 && (failed.ExceptionTypes[0].Contains("Xunit", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("Assert", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("EqualException", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("TrueException", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("FalseException", StringComparison.Ordinal)); + failed.ExceptionTypes[0].Contains("Assert", StringComparison.Ordinal)); if (_deferFailures) { @@ -509,7 +539,19 @@ public void Dispose() if (_deferredMessages.Count > 0) { Console.WriteLine($"[RetryFact] Internal error: {_deferredMessages.Count} deferred test message(s) were neither replayed nor discarded. Replaying them so the failure is not lost."); - ReplayDeferredMessages(); + + try + { + ReplayDeferredMessages(); + } + catch (Exception e) + { + // Dispose runs on the way out of the attempt, including while an exception is + // already unwinding. Throwing from here would replace whatever actually went + // wrong with a reporting error, so the failure to replay is logged and the + // original exception is left to propagate. + Console.WriteLine($"[RetryFact] Failed to replay {_deferredMessages.Count} deferred test message(s): {e.GetType().FullName}: {e.Message}"); + } } } } From 0e85232d686854b32e899d1e9b64c33dddd01d72 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 20 Aug 2026 21:59:03 -0700 Subject: [PATCH 045/105] Keep fixture argument set variants in their original test collection A class expanded into fixture argument set variants was given a new test collection per variant. xUnit parallelizes by collection, so classes an explicit [Collection] had deliberately grouped together started running concurrently, which is the opposite of what the grouping asks for. The xUnit v2 implementation kept every variant in the class's own collection; this restores that and drops the variant collection type entirely. Sharing a collection exposed the fixture argument cache: it is scoped to the collection and was only written to when a key was absent, so the first variant's arguments were handed to every later variant. Each class now writes over the cache and evicts what it cannot resolve. Also honour the message bus asking for the run to stop when a deferred failure is replayed, use the test method xUnit resolved for a theory row so generic arguments are not discarded, warn when a class expands to no argument sets instead of silently contributing no tests, and stop claiming the TRX error counter can catch anything -- the pinned reporter writes it as a constant zero. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SerializedVariants/ConcurrencyProbe.cs | 50 +++++++++++++ .../FirstSerializedVariantTests.cs | 53 ++++++++++++++ .../SecondSerializedVariantTests.cs | 59 +++++++++++++++ .../SerializedVariantCollectionTests.cs | 44 +++++++++++ .../TestAssetRun.cs | 12 ++- .../TestAssetRunAssertions.cs | 8 ++ .../CustomXunitTestFrameworkDiscoverer.cs | 55 ++++++++++---- .../CustomXunitTestFrameworkExecutor.cs | 19 ++++- .../FixtureArgumentSetTestCollection.cs | 73 ------------------- .../RetryTestCase.cs | 44 +++++++++-- .../RetryTheoryDiscoverer.cs | 11 ++- .../Rest/Search/TokenOverflowTests.cs | 1 - 12 files changed, 324 insertions(+), 105 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/ConcurrencyProbe.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/SecondSerializedVariantTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/SerializedVariantCollectionTests.cs delete mode 100644 src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/ConcurrencyProbe.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/ConcurrencyProbe.cs new file mode 100644 index 0000000000..797b6dce90 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/ConcurrencyProbe.cs @@ -0,0 +1,50 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants +{ + /// + /// Records whether any two tests that used it were ever inside their bodies at the same time. + /// + internal static class ConcurrencyProbe + { + private static int _active; + private static int _observedOverlap; + + /// + /// Gets a value indicating whether two tests were ever active at once. + /// + internal static bool ObservedOverlap => Volatile.Read(ref _observedOverlap) != 0; + + /// + /// Marks the caller as active for a fixed window, so that a concurrent caller overlaps it. + /// + /// + /// The window has to be long enough that a genuinely parallel run overlaps rather than + /// merely interleaving between the increment and the decrement. Tests in one xUnit + /// collection run one after another, so the window costs a serialized run only its own + /// duration and never makes the assertion flaky in the direction that matters: a serial + /// run cannot observe an overlap however long the window is. + /// + internal static void Occupy() + { + if (Interlocked.Increment(ref _active) > 1) + { + Interlocked.Exchange(ref _observedOverlap, 1); + } + + try + { + Thread.Sleep(250); + } + finally + { + Interlocked.Decrement(ref _active); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs new file mode 100644 index 0000000000..05771f0b6a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants +{ + /// + /// One of two classes placed in the same name-only collection, each expanded into one variant + /// per data store. A name-only [Collection] has no [CollectionDefinition] class + /// behind it, which is the case the fixture argument set machinery has to detect by other means. + /// + /// + /// Classes are put in a collection to stop them running at the same time, usually because they + /// share something external. Expanding a class into variants must not quietly undo that: if the + /// variants land in collections of their own, xUnit is free to run them concurrently and the + /// grouping the author asked for is lost. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + [Collection("SerializedVariants")] + public class FirstSerializedVariantTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public FirstSerializedVariantTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Fails if any other test in this collection was running at the same time. + /// + [Fact] + public void RunsWithoutOverlappingTheOtherVariants() + { + ConcurrencyProbe.Occupy(); + + Assert.False( + ConcurrencyProbe.ObservedOverlap, + "Two tests from the same [Collection] ran concurrently, so expanding a class into fixture argument set variants broke the collection's serialization."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/SecondSerializedVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/SecondSerializedVariantTests.cs new file mode 100644 index 0000000000..f629b0fbb3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/SecondSerializedVariantTests.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants +{ + /// + /// The second class in the same name-only collection as , + /// so that the scenario covers variants of different classes and not only variants of one class. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + [Collection("SerializedVariants")] + public class SecondSerializedVariantTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public SecondSerializedVariantTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Fails if any other test in this collection was running at the same time. + /// + [Fact] + public void RunsWithoutOverlappingTheOtherVariants() + { + ConcurrencyProbe.Occupy(); + + Assert.False( + ConcurrencyProbe.ObservedOverlap, + "Two tests from the same [Collection] ran concurrently, so expanding a class into fixture argument set variants broke the collection's serialization."); + } + + /// + /// Ties the variant's reported name to the fixture it was actually given, so that sharing a + /// collection between variants cannot quietly hand them each other's fixture. + /// + [Fact] + public void ReceivesTheFixtureItsNameClaims() + { + Assert.Contains( + $"({Fixture.DataStore})", + TestContext.Current.Test.TestDisplayName, + System.StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SerializedVariantCollectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SerializedVariantCollectionTests.cs new file mode 100644 index 0000000000..dcc152414c --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SerializedVariantCollectionTests.cs @@ -0,0 +1,44 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that expanding a test class into fixture argument set variants leaves the class in + /// the collection its author put it in. + /// + public class SerializedVariantCollectionTests + { + private const string FirstClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants.FirstSerializedVariantTests"; + private const string SecondClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants.SecondSerializedVariantTests"; + + /// + /// A collection is how an author says "these must not run at the same time". Giving each + /// variant its own collection would let xUnit run them concurrently, and because the + /// collection is named rather than backed by a [CollectionDefinition] class there is + /// no definition to inspect and notice the grouping. + /// + [Fact] + public void GivenClassesInOneNamedCollection_WhenTheirVariantsRun_ThenNoTwoRunConcurrently() + { + TestAssetRun run = TestAssetRunner.Run("SerializedVariants"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstClass + ".RunsWithoutOverlappingTheOtherVariants (Sql)"] = "Passed", + [FirstClass + ".RunsWithoutOverlappingTheOtherVariants (Cosmos)"] = "Passed", + [SecondClass + ".RunsWithoutOverlappingTheOtherVariants (Sql)"] = "Passed", + [SecondClass + ".RunsWithoutOverlappingTheOtherVariants (Cosmos)"] = "Passed", + [SecondClass + ".ReceivesTheFixtureItsNameClaims (Sql)"] = "Passed", + [SecondClass + ".ReceivesTheFixtureItsNameClaims (Cosmos)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs index 9d39525524..87224de342 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRun.cs @@ -50,10 +50,16 @@ public TestAssetRun(int exitCode, string output, IReadOnlyList public IReadOnlyList Results { get; } /// - /// Gets the number of runner-level errors reported. These are counted separately from - /// failed tests and are not published as results, so a run can be wrong in ways that the - /// result list alone does not show. + /// Gets the number of runner-level errors the TRX report recorded, which are counted + /// separately from failed tests and are never published as results. /// + /// + /// The Microsoft Testing Platform TRX reporter builds its summary from passed, failed, + /// skipped and timed-out counts only, and writes this counter as a constant zero, so today + /// this is a tripwire for a reporter that starts populating it rather than a check that can + /// currently fire. It is kept because a run that is wrong in this way shows nothing at all + /// in the result list, which is exactly the failure these tests would otherwise miss. + /// public int ErrorCount { get; } /// diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs index 0e4aa6216e..03e2d928d9 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunAssertions.cs @@ -54,6 +54,14 @@ public static void PublishedExactly(TestAssetRun run, IReadOnlyDictionary + /// Fails when the TRX report recorded runner-level errors. + /// + /// + /// See : the pinned reporter writes this counter as a + /// constant zero, so this cannot fire today and is not what keeps these tests honest. + /// + /// The run to assert on. private static void EnsureNoRunnerErrors(TestAssetRun run) { string message = $"The run reported {run.ErrorCount} runner-level error(s). These are counted separately " diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 8e5bf17167..6aa1946224 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -22,8 +22,6 @@ namespace Microsoft.Health.Extensions.Xunit internal sealed class CustomXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer { private static readonly FieldInfo TestCaseDisplayNameField = typeof(XunitTestCase).GetField("testCaseDisplayName", BindingFlags.Instance | BindingFlags.NonPublic); - - private readonly ConcurrentDictionary _variantCollectionCache = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _variantClassCache = new(StringComparer.Ordinal); public CustomXunitTestFrameworkDiscoverer(Assembly assembly, IXunitTestCollectionFactory collectionFactory = null) @@ -108,25 +106,33 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl } } - foreach (SingleFlag[] closedVariant in closedSets) + if (closedSets.Length == 0) { - // Key variant collections by the source xUnit collection identity. Default xUnit collections are - // already per-class, while explicit [Collection] groups share one source UniqueID. Preserving that - // identity keeps explicit collection serialization intact without forcing unrelated classes to run - // serially. - var variantKey = BuildVariantCollectionKey(testClass.TestCollection, closedVariant); - var variantCollection = _variantCollectionCache.GetOrAdd( - variantKey, - _ => new FixtureArgumentSetTestCollection(testClass.TestCollection, closedVariant)); + // A dimension that contributes no flags -- an argument set of zero, a value that + // names no single flag, or a null entry -- collapses the cartesian product to + // nothing, and this method then produces no test cases at all. The run stays + // green with the tests simply absent, so say so rather than let them vanish. + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: '{testClass.TestClassName}.{method.Name}' expanded to no fixture argument sets, so none of its tests will run. " + + "Check that every argument set names at least one single-valued flag of a [Flags] enum."); + } + foreach (SingleFlag[] closedVariant in closedSets) + { + // Every variant stays in the class's original collection. xUnit's unit of + // parallelization is the collection, so giving each variant its own would let + // classes an explicit [Collection] deliberately grouped together -- and the + // variants of a single class -- start running concurrently. Only the identifiers + // below carry the variant, so the collection keeps its original grouping. + var variantKey = BuildVariantKey(testClass.TestCollection, closedVariant); var classKey = BuildVariantClassKey(variantKey, testClass.Class); var closedVariantTestClass = _variantClassCache.GetOrAdd( classKey, _ => new FixtureArgumentSetTestClass( testClass.Class, - variantCollection, + testClass.TestCollection, closedVariant, - UniqueIDGenerator.ForTestClass(variantCollection.UniqueID, testClass.Class.FullName))); + UniqueIDGenerator.ForTestClass(testClass.TestCollection.UniqueID, classKey))); var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); @@ -195,7 +201,13 @@ private static void ApplyVariantDisplayName(ITestCase testCase, SingleFlag[] clo TestCaseDisplayNameField.SetValue(xunitTestCase, displayName + suffix); } - private static string BuildVariantCollectionKey(IXunitTestCollection sourceCollection, IReadOnlyList closedVariant) + /// + /// Builds a key that distinguishes one fixture argument set from another within a collection. + /// + /// The collection the test class belongs to. + /// The fixture arguments the variant was closed over. + /// A stable key for the variant. + private static string BuildVariantKey(IXunitTestCollection sourceCollection, IReadOnlyList closedVariant) { var variantKey = string.Join( ",", @@ -204,9 +216,20 @@ private static string BuildVariantCollectionKey(IXunitTestCollection sourceColle return $"{sourceCollection.UniqueID}|{variantKey}"; } - private static string BuildVariantClassKey(string variantCollectionKey, Type testClass) + /// + /// Builds a key that identifies one variant of one test class. + /// + /// The key returned by . + /// The test class the variant was derived from. + /// A stable key for the variant class, also used to seed its unique ID. + /// + /// The variants of a class share a collection and a class name, so the class name alone no + /// longer separates them. Seeding the unique ID from this key instead keeps every variant + /// distinct to the runner and to the reporters. + /// + private static string BuildVariantClassKey(string variantKey, Type testClass) { - return $"{variantCollectionKey}|{testClass.AssemblyQualifiedName}"; + return $"{variantKey}|{testClass.AssemblyQualifiedName}"; } private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(FixtureArgumentSetsAttribute attribute) diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index 7d101773c5..d372e52ecd 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -212,10 +212,23 @@ private static void InjectFixtureArguments(XunitTestClassRunnerContext context) } } - foreach (var resolvedArgument in resolvedArguments - .Where(resolvedArgument => !cache.ContainsKey(resolvedArgument.Key))) + // The cache being written to is scoped to the collection, and the variants of a class + // share a collection so that an explicit [Collection] keeps its classes serialized. + // Every variant therefore writes over the previous one's arguments: adding only what + // is missing would leave the first variant's values in place and hand every later + // variant the wrong fixture. Types this class expects but could not resolve are + // evicted for the same reason -- a stale value is worse than no value, because it + // silently constructs a fixture for the wrong argument set. + foreach (Type parameterType in fixtureParameterTypes) { - cache[resolvedArgument.Key] = resolvedArgument.Value; + if (resolvedArguments.TryGetValue(parameterType, out var resolvedValue)) + { + cache[parameterType] = resolvedValue; + } + else + { + cache.Remove(parameterType); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs deleted file mode 100644 index 09df7430b3..0000000000 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestCollection.cs +++ /dev/null @@ -1,73 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using EnsureThat; -using Xunit.Sdk; -using Xunit.v3; - -namespace Microsoft.Health.Extensions.Xunit -{ - internal sealed class FixtureArgumentSetTestCollection : XunitTestCollection - { - public FixtureArgumentSetTestCollection(IXunitTestCollection sourceCollection, IReadOnlyList fixtureArguments) - : base( - EnsureArg.IsNotNull(sourceCollection, nameof(sourceCollection)).TestAssembly, - sourceCollection.CollectionDefinition, - ShouldDisableParallelization(sourceCollection), - BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments), - uniqueID: BuildUniqueId(sourceCollection, fixtureArguments)) - { - EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - } - -#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes - public FixtureArgumentSetTestCollection() - { - } -#pragma warning restore CS0618 - - private static string BuildDisplayName(string baseDisplayName, IReadOnlyList fixtureArguments) - { - EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - - if (fixtureArguments.Count == 0) - { - return baseDisplayName; - } - - var argsLabel = string.Join(", ", fixtureArguments.Select(v => v.EnumValue)); - return $"{baseDisplayName}({argsLabel})"; - } - - private static string BuildUniqueId(IXunitTestCollection sourceCollection, IReadOnlyList fixtureArguments) - { - var displayName = BuildDisplayName(sourceCollection.TestCollectionDisplayName, fixtureArguments); - return UniqueIDGenerator.ForTestCollection(sourceCollection.TestAssembly.UniqueID, displayName, sourceCollection.TestCollectionClassName); - } - - /// - /// Determines whether the variants of a collection should be prevented from running concurrently. - /// - /// - /// Splitting a collection into one variant per fixture argument set would otherwise let the - /// variants run concurrently. Any collection backed by a [CollectionDefinition] class stays - /// serialized, because those group classes that were deliberately placed together. - /// - /// This does not cover a name-only [Collection("...")] with no matching [CollectionDefinition]: - /// TestCollectionClassName holds the definition class, so it is null for those and there is - /// nothing here to detect them by. - /// - /// - /// The collection the variant was derived from. - /// true if the variant collection should run serially. - private static bool ShouldDisableParallelization(IXunitTestCollection sourceCollection) - { - return sourceCollection.DisableParallelization || sourceCollection.CollectionDefinition != null; - } - } -} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 8f4345c41a..d566a5a2ec 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -161,21 +161,25 @@ public async ValueTask Run( // At most one bus is replayed. A failure this attempt saw supersedes anything held // over from an earlier one, and replaying both would publish two results for a // single test. + bool continueRunning = true; + switch (outcome) { case NoResultOutcome.ReplayCurrentAttempt: pendingFailureBus?.DiscardDeferredMessages(); - interceptingBus.ReplayDeferredMessages(); + continueRunning = interceptingBus.ReplayDeferredMessages(); break; case NoResultOutcome.ReplayEarlierAttempt: - pendingFailureBus.ReplayDeferredMessages(); + continueRunning = pendingFailureBus.ReplayDeferredMessages(); break; default: break; } + StopRunIfRequested(continueRunning, cancellationTokenSource); + if (lastException != null) { aggregator.Add(lastException); @@ -247,7 +251,7 @@ public async ValueTask Run( // retrying, so replay it now: the reporters derive their results // solely from the message bus, and without this the failure would // disappear from the test results entirely and the run would pass. - interceptingBus.ReplayDeferredMessages(); + StopRunIfRequested(interceptingBus.ReplayDeferredMessages(), cancellationTokenSource); if (lastException != null) { @@ -284,6 +288,8 @@ public async ValueTask Run( pendingFailureBus.ReplayDeferredMessages(); + // The run is already cancelling, so there is nothing further to stop and + // the replay result is deliberately ignored here. if (lastException != null) { aggregator.Add(lastException); @@ -338,6 +344,24 @@ public async ValueTask Run( return runSummary; } + /// + /// Cancels the run when the message bus asked for it to stop. + /// + /// The result of replaying deferred messages. + /// The run's cancellation source. + /// + /// A deferred failure answers true to the runner when it is queued, because at that + /// point the test may still be retried. Replaying it is therefore the first and only chance + /// to honour a request to stop, such as the one --stop-on-fail makes. + /// + private static void StopRunIfRequested(bool continueRunning, CancellationTokenSource cancellationTokenSource) + { + if (!continueRunning) + { + cancellationTokenSource.Cancel(); + } + } + /// /// Decides how an attempt that published no result of its own should be accounted for. /// @@ -510,15 +534,25 @@ public bool QueueMessage(IMessageSinkMessage message) /// in their original order. Call this when the attempt will not be retried, otherwise /// the failure is never reported and the test silently vanishes from the results. /// - public void ReplayDeferredMessages() + /// + /// false when the underlying bus asked for the run to stop. Deferring a failure + /// forces a true back to the runner at the time it is queued, so this is the only + /// point at which a request to stop -- from --stop-on-fail, for instance -- can still be + /// honoured. + /// + public bool ReplayDeferredMessages() { + bool continueRunning = true; + foreach (IMessageSinkMessage message in _deferredMessages) { - _innerBus.QueueMessage(message); + continueRunning &= _innerBus.QueueMessage(message); } _deferredMessages.Clear(); _deferring = false; + + return continueRunning; } /// diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index e1f614f5e0..40dc41eed6 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -31,7 +31,7 @@ protected override async ValueTask> CreateTe var baseCases = await base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments); return baseCases - .Select(testCase => WrapTestCase(testMethod, testCase, attribute)) + .Select(testCase => WrapTestCase(testCase, attribute)) .ToArray(); } @@ -44,11 +44,11 @@ protected override async ValueTask> CreateTe var baseCases = await base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute); return baseCases - .Select(testCase => WrapTestCase(testMethod, testCase, attribute)) + .Select(testCase => WrapTestCase(testCase, attribute)) .ToArray(); } - private static IXunitTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTestCase testCase, RetryTheoryAttribute attribute) + private static IXunitTestCase WrapTestCase(IXunitTestCase testCase, RetryTheoryAttribute attribute) { if (testCase is not XunitTestCase xunitTestCase) { @@ -61,8 +61,11 @@ private static IXunitTestCase WrapTestCase(IXunitTestMethod testMethod, IXunitTe return testCase; } + // The base case carries the method xUnit resolved for this row, including any generic + // arguments closed from the row's data. The method passed to the discoverer is still + // open, so using it here would discard that resolution. return new RetryTestCase( - testMethod, + xunitTestCase.TestMethod, xunitTestCase.TestCaseDisplayName, xunitTestCase.UniqueID, xunitTestCase.Explicit, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs index 4c2f88b40c..15995f3e6d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/TokenOverflowTests.cs @@ -578,4 +578,3 @@ await ExecuteAndValidateBundle( public ValueTask DisposeAsync() => ValueTask.CompletedTask; } } - From 1650509071787ac61eac73b44532b6ca51283949 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 07:52:55 -0700 Subject: [PATCH 046/105] Report discovery faults, keep theory rows, and stop retries being cancelled out Round 4 of the review of the xunit.v3 migration. Each item below was reproduced before it was changed. Discovery faults were invisible. An exception thrown while expanding a class into its fixture argument set variants is reported by xunit.v3 only as an internal diagnostic, which is suppressed by default, and the class is dropped. A run with any other healthy class still reported success and exit 0, so a broken expansion looked exactly like a class with no tests. Such a fault is now published as a failing test case, which puts it in the results and in the exit code. The new DiscoveryFault asset provokes a real one by declaring more dimensions on a method than its class has. Theory rows resolved at run time were lost. A delay-enumerated theory case is a XunitTestCase, so it passed the retry wrapper's type check and was rebuilt as a plain RetryTestCase carrying no arguments; every row then failed with an arity error instead of running. Such cases are now left unwrapped, which costs the retries but keeps the tests. Retries were cancelled out in CI. --retry-failed-tests writes one TRX per attempt under Retries///, all matched by the publish globs, so attempt 1 kept a Failed record for a test that later passed and failTaskOnFailedTests failed the task even though the runner exited 0. Measured: a test that recovers exits 0 with a Failed record in attempt 1's TRX, and one that exhausts its retries exits 2. The runner's exit code is the retry-aware verdict and the step running the tests already fails on it, so the attempts are now published for visibility only. Execution context flow was silently gone. Under xunit.v2 this runner captured the context immediately after each class fixture was constructed, so an AsyncLocal set in a fixture constructor was readable from the tests. xunit.v3 builds fixtures inside an async method, whose state machine restores the caller's context as it returns, so the write is discarded before any capture point outside that method can see it -- the ported machinery captured after an await and preserved nothing. It is removed rather than left claiming a guarantee it does not provide, the loss is recorded by an asset, and the two fixture comments that described the old behaviour are corrected. ResourceIdProvider, which one of them named, holds its factory in an ordinary field and never depended on the flow. Also: a passing RetryFact no longer drops the attempt's Skipped and NotRun counts, which made a skipped test read as one that ran and passed; the no-result console message no longer claims a result was reported when there was nothing to report; and the comments describing the assertion-failure match, the variant class key and the serialized variant collection now match what the code does. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/build.yml | 30 +++++++--- build/jobs/e2e-tests.yml | 11 +++- build/jobs/run-cosmos-tests.yml | 15 +++-- build/jobs/run-export-tests.yml | 22 ++++++-- build/jobs/run-sql-tests.yml | 15 +++-- .../DiscoveryFault/AssetOtherDimension.cs | 28 ++++++++++ .../DiscoveryFault/DiscoveryFaultTests.cs | 48 ++++++++++++++++ .../TwoDimensionArgumentSetsAttribute.cs | 26 +++++++++ .../AsyncLocalFlowTests.cs | 48 ++++++++++++++++ .../ExecutionContextFlow/AsyncLocalProbe.cs | 29 ++++++++++ .../AsyncLocalWritingFixture.cs | 24 ++++++++ .../FirstSerializedVariantTests.cs | 6 +- .../DiscoveryFaultTests.cs | 39 +++++++++++++ .../ExecutionContextFlowTests.cs | 39 +++++++++++++ .../RetryTheoryOutcomeTests.cs | 23 ++++++++ .../TestAssetRunner.cs | 17 +++++- .../CustomXunitTestFrameworkDiscoverer.cs | 55 ++++++++++++++++++- .../CustomXunitTestFrameworkExecutor.cs | 39 +++++-------- .../FixtureArgumentSetTestMethod.cs | 10 ---- .../RetryTestCase.cs | 20 +++++-- .../RetryTheoryDiscoverer.cs | 15 +++++ .../Persistence/FhirStorageTestsFixture.cs | 12 ++-- 22 files changed, 498 insertions(+), 73 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/AssetOtherDimension.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/TwoDimensionArgumentSetsAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalFlowTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalProbe.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/ExecutionContextFlowTests.cs diff --git a/build/jobs/build.yml b/build/jobs/build.yml index d57e770ec4..e141473d81 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -36,8 +36,8 @@ steps: command: test projects: '**/*UnitTests/*.csproj' arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx' - # Disable the task's built-in (non-retry-aware) result publishing so the - # explicit retry-aware PublishTestResults@2 task below is the sole publisher. + # Disable the task's built-in result publishing so the PublishTestResults@2 task below, + # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false env: platformOptions__resultDirectory: '$(Agent.TempDirectory)/unit-test-results' @@ -48,9 +48,16 @@ steps: testResultsFiles: '$(Agent.TempDirectory)/unit-test-results/**/*.trx' mergeTestResults: true testRunTitle: 'Unit Tests' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, + # non-zero when they are exhausted -- and the step that runs the tests already fails on + # it, so the attempts are published for visibility only. + failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step - # succeeds silently and failTaskOnFailedTests has no results to act on. + # succeeds silently and there is nothing to notice. failTaskOnMissingResultsFile: true condition: succeededOrFailed() @@ -61,8 +68,8 @@ steps: command: test projects: '**/*UnitTests/*.csproj' arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"' - # Disable the task's built-in (non-retry-aware) result publishing so the - # explicit retry-aware PublishTestResults@2 task below is the sole publisher. + # Disable the task's built-in result publishing so the PublishTestResults@2 task below, + # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false env: platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' @@ -73,9 +80,16 @@ steps: testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' mergeTestResults: true testRunTitle: 'Unit Tests' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, + # non-zero when they are exhausted -- and the step that runs the tests already fails on + # it, so the attempts are published for visibility only. + failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step - # succeeds silently and failTaskOnFailedTests has no results to act on. + # succeeds silently and there is nothing to notice. failTaskOnMissingResultsFile: true condition: succeededOrFailed() - task: reportgenerator@5 diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index adac03a70d..ced72768f1 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -100,8 +100,15 @@ steps: testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx' mergeTestResults: true testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, + # non-zero when they are exhausted -- and the step that runs the tests already fails on + # it, so the attempts are published for visibility only. + failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step - # succeeds silently and failTaskOnFailedTests has no results to act on. + # succeeds silently and there is nothing to notice. failTaskOnMissingResultsFile: true condition: succeededOrFailed() diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index 5635fedaa9..d7d2cdf624 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -100,8 +100,8 @@ jobs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - # Disable the task's built-in (non-retry-aware) result publishing so the - # explicit retry-aware PublishTestResults@2 task below is the sole publisher. + # Disable the task's built-in result publishing so the PublishTestResults@2 task below, + # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false env: 'CosmosDb__Host': $(CosmosDb--Host) @@ -120,9 +120,16 @@ jobs: testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' mergeTestResults: true testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, + # non-zero when they are exhausted -- and the step that runs the tests already fails on + # it, so the attempts are published for visibility only. + failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step - # succeeds silently and failTaskOnFailedTests has no results to act on. + # succeeds silently and there is nothing to notice. failTaskOnMissingResultsFile: true condition: succeededOrFailed() diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index 150e7023f7..4ebd2adb77 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -148,9 +148,16 @@ jobs: testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx' mergeTestResults: true testRunTitle: 'Export ${{ parameters.version }} CosmosDB' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, + # non-zero when they are exhausted -- and the step that runs the tests already fails on + # it, so the attempts are published for visibility only. + failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step - # succeeds silently and failTaskOnFailedTests has no results to act on. + # succeeds silently and there is nothing to notice. failTaskOnMissingResultsFile: true condition: succeededOrFailed() @@ -293,9 +300,16 @@ jobs: testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx' mergeTestResults: true testRunTitle: 'Export ${{ parameters.version }} SQL' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, + # non-zero when they are exhausted -- and the step that runs the tests already fails on + # it, so the attempts are published for visibility only. + failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step - # succeeds silently and failTaskOnFailedTests has no results to act on. + # succeeds silently and there is nothing to notice. failTaskOnMissingResultsFile: true condition: succeededOrFailed() diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 6bb35dc642..a96669f282 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -92,8 +92,8 @@ jobs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - # Disable the task's built-in (non-retry-aware) result publishing so the - # explicit retry-aware PublishTestResults@2 task below is the sole publisher. + # Disable the task's built-in result publishing so the PublishTestResults@2 task below, + # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false env: 'SqlServer:ConnectionString': 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);' @@ -110,9 +110,16 @@ jobs: testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' mergeTestResults: true testRunTitle: '${{ parameters.version }} SQL Integration Tests' - failTaskOnFailedTests: true + # --retry-failed-tests re-runs the failures in a fresh process and writes one TRX per + # attempt under Retries///, all of which this glob matches. Attempt 1 therefore + # keeps a Failed record for a test that later passed, and mergeTestResults does not + # resolve the two, so failing the task on published results would cancel out every + # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, + # non-zero when they are exhausted -- and the step that runs the tests already fails on + # it, so the attempts are published for visibility only. + failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step - # succeeds silently and failTaskOnFailedTests has no results to act on. + # succeeds silently and there is nothing to notice. failTaskOnMissingResultsFile: true condition: succeededOrFailed() diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/AssetOtherDimension.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/AssetOtherDimension.cs new file mode 100644 index 0000000000..9a81f23b70 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/AssetOtherDimension.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault +{ + /// + /// A second fixture argument set dimension. Only the zero value is used, because a dimension + /// that names no flag is what makes the discoverer fall back to the class-level dimension of + /// the same position -- a position the class does not have. + /// + [Flags] + public enum AssetOtherDimension + { + /// + /// No flags, so this dimension contributes no values of its own. + /// + None = 0, + + /// + /// An arbitrary flag, present so the type is a usable flags enum. + /// + Some = 1, + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs new file mode 100644 index 0000000000..2b334de698 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault +{ + /// + /// A class whose fixture argument sets cannot be expanded, so discovering it throws. + /// + /// + /// The class declares one dimension and the method below declares two, the second of which + /// names no flag. A dimension that names no flag means "use the class-level dimension in this + /// position", and the class has no second position, so the expansion reads past the end of the + /// class-level dimensions. The misconfiguration itself is not the point: it is simply the + /// cheapest way to make discovery of a real class throw, which is what this scenario is for. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class DiscoveryFaultTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public DiscoveryFaultTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Never runs: discovery of this class throws before any test case is produced. + /// + [Fact] + [TwoDimensionArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos, AssetOtherDimension.None)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/TwoDimensionArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/TwoDimensionArgumentSetsAttribute.cs new file mode 100644 index 0000000000..82e8eec26a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/TwoDimensionArgumentSetsAttribute.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault +{ + /// + /// Declares two fixture argument set dimensions, so a method carrying it declares more + /// dimensions than a class that declares only one. + /// + public sealed class TwoDimensionArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The first dimension. + /// The second dimension. + public TwoDimensionArgumentSetsAttribute(AssetDataStore first, AssetOtherDimension second) + : base(first, second) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalFlowTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalFlowTests.cs new file mode 100644 index 0000000000..f32b285026 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalFlowTests.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContextFlow +{ + /// + /// Pins what happens to a value the class fixture constructor wrote into the execution context. + /// + /// + /// Under xunit.v2 the custom executor ran test methods in the execution context captured just + /// after each class fixture was built, so this value was readable here. xunit.v3 builds fixtures + /// inside an async method, and an async method's state machine restores the caller's execution + /// context as it returns, so the write is discarded before any runner code can capture it. This + /// test records that loss deliberately: it fails if the flow ever comes back, which would mean + /// the note in FixtureArgumentSetClassRunner and the fixtures written against it are out + /// of date. + /// + public class AsyncLocalFlowTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The fixture whose constructor writes the probe value. + public AsyncLocalFlowTests(AsyncLocalWritingFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the fixture this class was constructed with. + /// + protected AsyncLocalWritingFixture Fixture { get; } + + /// + /// Fails if a value written to the execution context by a class fixture constructor becomes + /// readable from a test method again. + /// + [Fact] + public void DoesNotSeeTheValueWrittenByTheFixtureConstructor() + { + Assert.Null(AsyncLocalProbe.Value.Value); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalProbe.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalProbe.cs new file mode 100644 index 0000000000..71a24a56e0 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalProbe.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContextFlow +{ + /// + /// Carries a value from a class fixture constructor to a test method through the execution + /// context, which is the only channel that can show whether the context the fixture was built + /// in is still the one the tests run in. + /// + public static class AsyncLocalProbe + { + /// + /// The value the fixture constructor writes. + /// + public const string ExpectedValue = "set-in-fixture-constructor"; + + /// + /// The value written by the fixture constructor. An lives in the + /// execution context rather than in a field, so a test reads back what the fixture wrote + /// only if the runner deliberately restores that context. + /// + public static readonly AsyncLocal Value = new AsyncLocal(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs new file mode 100644 index 0000000000..dbe1d2baf8 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContextFlow +{ + /// + /// A class fixture that writes to an from its + /// constructor, the way FhirStorageTestsFixture builds its ResourceIdProvider + /// there rather than in InitializeAsync. + /// + public sealed class AsyncLocalWritingFixture + { + /// + /// Initializes a new instance of the class, writing + /// the probe value into the execution context that is current while the fixture is built. + /// + public AsyncLocalWritingFixture() + { + AsyncLocalProbe.Value.Value = AsyncLocalProbe.ExpectedValue; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs index 05771f0b6a..c16581807a 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SerializedVariants/FirstSerializedVariantTests.cs @@ -9,9 +9,9 @@ namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SerializedVariants { /// - /// One of two classes placed in the same name-only collection, each expanded into one variant - /// per data store. A name-only [Collection] has no [CollectionDefinition] class - /// behind it, which is the case the fixture argument set machinery has to detect by other means. + /// One of two classes placed in the same collection, each expanded into one variant per data + /// store. The collection is declared by name only, without a [CollectionDefinition] + /// class behind it, which xUnit accepts and treats like any other collection. /// /// /// Classes are put in a collection to stop them running at the same time, usually because they diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs new file mode 100644 index 0000000000..01677f25c9 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs @@ -0,0 +1,39 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a class whose fixture argument sets cannot be expanded is reported as a failure + /// rather than dropped from the run. + /// + public class DiscoveryFaultTests + { + private const string ErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests (fixture argument set discovery)"; + + /// + /// xUnit reports an exception thrown out of discovery only as an internal diagnostic, which + /// is suppressed by default, and carries on without the class. A run containing any other + /// healthy class therefore still reports success, so a broken expansion is indistinguishable + /// from a class that has no tests. The discoverer turns the fault into a failing test case + /// instead, which puts it in the results and in the exit code. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheFaultIsReportedAsAFailedTest() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ErrorCaseName] = "Failed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ExecutionContextFlowTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ExecutionContextFlowTests.cs new file mode 100644 index 0000000000..74c072929a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ExecutionContextFlowTests.cs @@ -0,0 +1,39 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Records what the custom executor does with the execution context a class fixture is built in. + /// + public class ExecutionContextFlowTests + { + private const string Prefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContextFlow.AsyncLocalFlowTests."; + + /// + /// A value written to an by a class fixture + /// constructor is not readable from the test methods of that class. xunit.v2 preserved that + /// flow and xunit.v3 cannot, because it builds fixtures inside an async method whose state + /// machine restores the caller's execution context on return. The scenario asserts the value + /// is absent, so this run turning red means the flow came back and the notes describing its + /// absence need revisiting. + /// + [Fact] + public void GivenAFixtureThatWritesToTheExecutionContext_WhenItsTestsRun_ThenTheWriteIsNotVisibleToThem() + { + TestAssetRun run = TestAssetRunner.Run("ExecutionContextFlow"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "DoesNotSeeTheValueWrittenByTheFixtureConstructor"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs index f5d3cb5d21..040429c6a2 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTheoryOutcomeTests.cs @@ -36,5 +36,28 @@ public void GivenARetryingTheory_WhenTheRunCompletes_ThenEachRowIsReportedExactl [Prefix + "FlakyRow(row: 3, recovers: True)"] = "Passed", }); } + + /// + /// A theory whose data is resolved at run time rather than at discovery cannot be wrapped for + /// retry, because the wrapper would supply the arguments it was built with -- none -- and every + /// row would be lost to an arity error instead of running. The discoverer leaves such a case + /// alone, so the rows still run, each with its own arguments, and simply do not retry: all + /// three fail here because each one fails its first attempt. Names carrying the argument values + /// are what separates "ran without retrying" from "lost its data". + /// + [Fact] + public void GivenARetryingTheoryResolvedAtRunTime_WhenTheRunCompletes_ThenEveryRowStillRunsWithItsArguments() + { + TestAssetRun run = TestAssetRunner.Run("RetryTheoryOutcomes", preEnumerateTheories: false); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "FlakyRow(row: 1, recovers: True)"] = "Failed", + [Prefix + "FlakyRow(row: 2, recovers: False)"] = "Failed", + [Prefix + "FlakyRow(row: 3, recovers: True)"] = "Failed", + }); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index 3d913ba872..8f03c9958b 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -41,8 +41,13 @@ public static class TestAssetRunner /// Whether to cancel the run as soon as a test fails. Scenarios that exercise cancellation /// rely on this to cancel the run from inside it. /// + /// + /// Whether the runner may resolve theory data at discovery time. Turning this off is the + /// supported way to reach the delay-enumerated code path, which a theory also reaches on + /// its own when its data cannot be serialized. + /// /// The exit code, output and published results of the run. - public static TestAssetRun Run(string scenario, bool stopOnFail = false) + public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true) { string assetsAssembly = ResolveAssetsAssembly(); string resultsDirectory = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", Guid.NewGuid().ToString("N")); @@ -77,6 +82,12 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false) arguments.Add("on"); } + if (!preEnumerateTheories) + { + arguments.Add("--pre-enumerate-theories"); + arguments.Add("off"); + } + (int exitCode, string output, TimeSpan duration) = Execute(arguments); string trxPath = Path.Combine(resultsDirectory, trxFileName); @@ -249,8 +260,8 @@ private static TestAssetRun ParseTrx(string trxPath, int exitCode, string output // guard would keep passing while checking nothing. XAttribute errorAttribute = document .Descendants(trx + "Counters") - .Select(e => e.Attribute("error")) - .FirstOrDefault(a => a != null); + .Attributes("error") + .FirstOrDefault(); if (errorAttribute == null) { diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 6aa1946224..6935eb3077 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -35,6 +35,53 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl EnsureArg.IsNotNull(callback, nameof(callback)); EnsureArg.IsNotNull(discoveryOptions, nameof(discoveryOptions)); + try + { + return await FindTestsForTypeCore(testClass, discoveryOptions, callback); + } + catch (Exception ex) + { + // An exception thrown out of discovery is reported by xunit.v3 only as an internal + // diagnostic message, which is suppressed unless the run was started with + // --xunit-internal-diagnostics on. The class is dropped and the run still reports + // success, so a broken expansion would look exactly like a class that has no tests. + // Reporting a test case that fails on execution instead puts the fault in the results + // and in the exit code, where it cannot be missed. + Console.WriteLine( + $"[FixtureArgumentSets] ERROR: discovery of '{testClass.TestClassName}' failed, so its tests were replaced by a single failing test case. {ex}"); + + MethodInfo firstMethod = testClass.Methods.FirstOrDefault(); + if (firstMethod == null) + { + // Nothing to hang a test case off. Rethrowing keeps the original xunit behaviour, + // which is all that is left, and the console line above is the only warning. + throw; + } + + var errorTestMethod = new XunitTestMethod( + testClass, + firstMethod, + Array.Empty(), + uniqueID: UniqueIDGenerator.ForTestMethod(testClass.UniqueID, firstMethod.Name)); + + // The test case is handed to xunit through the callback, which owns it from then on + // and disposes it with the rest of the discovered cases. +#pragma warning disable CA2000 + var errorTestCase = new ExecutionErrorTestCase( + errorTestMethod, + $"{testClass.TestClassName} (fixture argument set discovery)", + $"{testClass.UniqueID}-fixture-argument-set-discovery-error", + sourceFilePath: null, + sourceLineNumber: null, + errorMessage: $"Discovering the fixture argument set variants of '{testClass.TestClassName}' failed, so none of its tests ran. {ex}"); +#pragma warning restore CA2000 + + return await callback(errorTestCase); + } + } + + private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, ITestFrameworkDiscoveryOptions discoveryOptions, Func> callback) + { var attribute = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; var methodAttributes = testClass.Methods.ToDictionary( method => method, @@ -221,11 +268,13 @@ private static string BuildVariantKey(IXunitTestCollection sourceCollection, IRe /// /// The key returned by . /// The test class the variant was derived from. - /// A stable key for the variant class, also used to seed its unique ID. + /// A stable key for the variant class. /// /// The variants of a class share a collection and a class name, so the class name alone no - /// longer separates them. Seeding the unique ID from this key instead keeps every variant - /// distinct to the runner and to the reporters. + /// longer separates them and this key is what the variant class cache is keyed on. It is + /// also passed as the seed for the class's unique ID, but only reaches it for a variant that + /// carries no arguments: recomputes the unique ID + /// from its own argument values whenever it has any, which is the usual case. /// private static string BuildVariantClassKey(string variantKey, Type testClass) { diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index d372e52ecd..e5205996a9 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -53,6 +53,20 @@ protected override async ValueTask RunTestClass(XunitTestCollectionR } } + /// + /// Runs a test class, injecting the fixture argument set values its variant was expanded with. + /// + /// + /// Under xunit.v2 this runner also ran test methods in the execution context captured right + /// after each class fixture was constructed, so an written by a + /// fixture constructor could be read by the tests. That is not reproducible on xunit.v3: + /// v2 built fixtures in a synchronous CreateClassFixture override, while v3 builds + /// them inside FixtureMappingManager.GetFixture, an async method whose state machine + /// restores the caller's execution context as it returns. Any such write is therefore + /// already discarded before this runner regains control, and no capture point outside that + /// method can see it. Fixtures that need to share ambient state with their tests have to + /// expose it as a member instead of relying on the execution context. + /// private sealed class FixtureArgumentSetClassRunner : XunitTestClassRunner { private static readonly FieldInfo FixtureCacheField = typeof(FixtureMappingManager) @@ -61,33 +75,10 @@ private sealed class FixtureArgumentSetClassRunner : XunitTestClassRunner private static readonly FieldInfo ParentMappingManagerField = typeof(FixtureMappingManager) .GetField("parentMappingManager", BindingFlags.Instance | BindingFlags.NonPublic); - private ExecutionContext _executionContext; - protected override async ValueTask OnTestClassStarting(XunitTestClassRunnerContext context) { InjectFixtureArguments(context); - var result = await base.OnTestClassStarting(context); - _executionContext = ExecutionContext.Capture(); - return result; - } - - protected override async ValueTask OnTestClassFinished(XunitTestClassRunnerContext context, RunSummary summary) - { - _executionContext?.Dispose(); - _executionContext = null; - return await base.OnTestClassFinished(context, summary); - } - - protected override ValueTask RunTestMethod(XunitTestClassRunnerContext context, IXunitTestMethod testMethod, IReadOnlyCollection testCases, object[] constructorArguments) - { - if (_executionContext == null) - { - return base.RunTestMethod(context, testMethod, testCases, constructorArguments); - } - - ValueTask summary = default; - ExecutionContext.Run(_executionContext, _ => summary = base.RunTestMethod(context, testMethod, testCases, constructorArguments), null); - return summary; + return await base.OnTestClassStarting(context); } protected override ValueTask GetConstructorArgument(XunitTestClassRunnerContext context, ConstructorInfo constructor, int index, ParameterInfo parameter) diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index 2b7493c17d..f0ad9aa762 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -87,16 +87,6 @@ private object[] CombineFixtureAndMethodArguments(object[] methodArguments) return result; } - private void UpdateMethodArguments(object[] methodArguments) - { - if (_fixtureArguments.Count == 0) - { - return; - } - - TestMethodArgumentsField.SetValue(this, methodArguments); - } - internal void UpdateArgumentsFromMethod() { if (_fixtureArguments.Count == 0) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index d566a5a2ec..d256b052bc 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -156,7 +156,10 @@ public async ValueTask Run( currentAttemptObservedFailure: interceptingBus.HasDeferredFailure, earlierAttemptObservedFailure: pendingFailureBus?.HasDeferredFailure ?? false); - Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted). Reporting the last observed result rather than a pass."); + Console.WriteLine( + outcome == NoResultOutcome.ReportNothing + ? $"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted), and no earlier attempt observed a failure, so there is nothing to report for it." + : $"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted). Reporting the last observed failure rather than a pass."); // At most one bus is replayed. A failure this attempt saw supersedes anything held // over from an earlier one, and replaying both would publish two results for a @@ -202,6 +205,12 @@ public async ValueTask Run( new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' passed on attempt {attempt}/{_maxRetries}")); runSummary.Failed = 0; + + // A test that was skipped or left unrun did not pass, and the counts saying so + // live on the attempt's summary. Returning only Total and Failed would report + // it to anything reading this summary as a test that ran and passed. + runSummary.Skipped = summary.Skipped; + runSummary.NotRun = summary.NotRun; return runSummary; } @@ -509,9 +518,12 @@ public bool QueueMessage(IMessageSinkMessage message) ? string.Join(Environment.NewLine, failed.StackTraces) : null; - // Detect if this is an assertion failure by checking exception types. - // Every xUnit assertion exception lives under the Xunit.Sdk namespace, so the - // namespace check already covers the individual assertion exception types. + // Classify the failure as an assertion failure so RetryOnAssertionFailure can decide + // whether it is worth another attempt. This is a substring match, not a namespace + // check: xUnit's own assertion exceptions live under Xunit.Sdk, but assertion + // libraries used alongside it do not, and matching "Assert" anywhere in the type + // name catches those too. The trade is that an unrelated type whose name happens to + // contain either word is classified the same way, which only costs an attempt. IsAssertionFailure = failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 && (failed.ExceptionTypes[0].Contains("Xunit", StringComparison.Ordinal) || diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index 40dc41eed6..dc92808114 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -50,6 +50,21 @@ protected override async ValueTask> CreateTe private static IXunitTestCase WrapTestCase(IXunitTestCase testCase, RetryTheoryAttribute attribute) { + if (testCase is IXunitDelayEnumeratedTestCase) + { + // A delay-enumerated case has not resolved its data rows yet, and it resolves them + // itself at run time. RetryTestCase cannot stand in for that: it would run the + // method with the arguments it was constructed with -- none -- and every row of the + // theory would be lost with an arity error rather than run. Returning the case + // unwrapped costs the retries but keeps the tests, which is the better trade. + // Theories reach this path when their data cannot be pre-enumerated, or when + // pre-enumeration is turned off. + Console.WriteLine( + $"[RetryTheory] WARNING: Test case '{testCase.TestCaseDisplayName}' resolves its data at run time, so retry logic will NOT be applied to it. " + + "Use pre-enumerable theory data (for example [InlineData], or [MemberData] whose values are serializable) if the rows need to be retried."); + return testCase; + } + if (testCase is not XunitTestCase xunitTestCase) { // Trace output only reaches an attached debugger, so a Trace-only warning here diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs index 3a067e0f16..74aaafdb4b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs @@ -94,8 +94,10 @@ internal FhirStorageTestsFixture(IServiceProvider fixture) { _fixture = fixture; - // This step has to be done in the constructor because it uses an AsyncLocal and the tests run with the same - // execution context as the fixture constructor, but not the same as InitializeAsync(). + // ResourceIdProvider holds its factory in an ordinary field, so where it is built does + // not affect what the tests see. The comment that used to be here claimed it had to be + // built in the constructor to keep an AsyncLocal alive; that flow no longer exists under + // xunit.v3 (see FixtureArgumentSetClassRunner) and this type never depended on it. _resourceIdProvider = new ResourceIdProvider(); _dataResourceFilter = new DataResourceFilter(MissingDataFilterCriteria.Default); @@ -212,8 +214,10 @@ public async ValueTask InitializeAsync() await asyncLifetime.InitializeAsync(); } - // Initialize FhirRequestContext to ensure pending status updates are captured - // This needs to be here (like ResourceIdProvider) because it uses AsyncLocal + // Initialize FhirRequestContext to ensure pending status updates are captured. + // FhirRequestContextAccessor stores the context in an AsyncLocal, so this assignment is + // only visible to callers that share this execution context; tests that need a request + // context of their own set it themselves. FhirRequestContextAccessor.RequestContext = new DefaultFhirRequestContext { BaseUri = new Uri("http://localhost/"), From 60fe3aa02e0cdc26f5ff241db43c61564d6c4aa7 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 08:43:52 -0700 Subject: [PATCH 047/105] Stop trait filters and cancellation from hiding discovery faults Round 4 of the pre-merge review found two ways the discovery fault reporting added earlier could still go quiet, and one claim in a comment that overstated what the code does. A discovery fault was reported as a failing test case, but that case carried no traits, while the variants it stood in for would each have carried the argument set value that produced them. Every CI leg selects its tests with a positive trait filter, so the case was filtered out and the leg passed with the class silently missing - the same silent drop the mechanism was added to prevent, only harder to notice. Measured before the fix: --filter-trait AssetDataStore=Sql over the assets gave 5 passed and exit 0 with the fault nowhere in the report. The case now declares every argument set value its class asked for, so any filter that would have selected any variant also selects the failure. ExecutionErrorTestCase takes no traits and XunitTestCase.Traits is not virtual, so the values go into the private backing field, as FixtureArgumentSetTestMethod already does for real variants; a rename in xunit throws a descriptive error rather than quietly reverting to the old behaviour. The blanket catch around discovery also swallowed cancellation, turning a run the user stopped into a fabricated test failure. Cancellation is now rethrown before the catch that reports faults. RetryTestCase's comment on propagating Skipped and NotRun claimed the propagation makes skips visible in reports. It does not: removing it changes neither the TRX outcomes nor the aggregate skipped count, because the platform derives its counts from the message bus rather than from the returned summary. The comment now says what the code actually preserves. Tests - A harness case runs the fault scenario behind --filter-trait and asserts the failure is still published, with a runner option added for trait filtering. Verified red before the fix and green after. - A harness case pins the console diagnostic, which is the only place the original exception survives: the failing case reports the class fixture failure instead, because the fixture is built before the case's own message is read. Three alternatives were tried and measured worse - a fixture-free placeholder class is dropped by namespace filters, and clearing the aggregator discards the case's own message as well, in both variants leaving nothing reported at all. - A run-time skipped [RetryFact] asset pins that a skip is neither retried nor turned into a pass. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryOutcomes/RetryOutcomeTests.cs | 13 ++- .../DiscoveryFaultTests.cs | 41 +++++++++ .../RetryFactOutcomeTests.cs | 8 +- .../TestAssetRunner.cs | 13 ++- .../CustomXunitTestFrameworkDiscoverer.cs | 89 ++++++++++++++++++- .../DiscoveryErrorTestCase.cs | 68 ++++++++++++++ .../RetryTestCase.cs | 5 +- 7 files changed, 227 insertions(+), 10 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs index b5797349a5..6001879b9d 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs @@ -11,7 +11,7 @@ namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryOutcomes { /// /// The full outcome matrix, run without cancellation. - /// Expected: 6 results, 3 failed and 3 passed, with exactly one result per test. + /// Expected: 7 results, 3 failed, 3 passed and 1 skipped, with exactly one result per test. /// public class RetryOutcomeTests { @@ -71,6 +71,17 @@ public void AlwaysPasses_IsReportedPassed() Assert.True(true); } + /// + /// A test that skips itself must be reported as skipped, and must not be retried on the way + /// there: a skip is not a failure, so the attempt loop has to stop at it rather than run the + /// test again and report the last attempt as a pass. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void SkippedAtRunTime_IsReportedSkipped() + { + Assert.Skip("ASSET: skipped at run time"); + } + /// /// A MaxRetries below one would skip the attempt loop and report nothing, and a negative /// delay would make Task.Delay throw. Both are clamped, so the test runs exactly once. diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs index 01677f25c9..572138c599 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using Xunit; @@ -34,6 +35,46 @@ public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheFaultIsRep { [ErrorCaseName] = "Failed", }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failing test case cannot be relied on to carry the cause, because the class fixture it + /// inherits fails to build first and that failure is what the report shows. The console line + /// the discoverer writes is the only place the original exception survives, so it is the part + /// worth pinning: without it a fault would be reported with no way to find out what it was. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheCauseIsWrittenToTheOutput() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault"); + + Assert.Contains("[FixtureArgumentSets] ERROR", run.Output, StringComparison.Ordinal); + Assert.Contains("IndexOutOfRangeException", run.Output, StringComparison.Ordinal); + } + + /// + /// Reporting the fault as a test is only worth anything if the legs that would have run the + /// class still select it. CI legs pick their tests with a positive trait filter, and the + /// variants this class never produced are what would have carried those traits, so a fault + /// case with no traits of its own is filtered out and the leg passes with the class silently + /// missing - which is the failure this whole mechanism exists to prevent, only harder to see. + /// The case therefore declares every argument set value the class asked for. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenTestsAreSelectedByArgumentSetTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs index 0120875a3c..e6910b9a02 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs @@ -18,9 +18,10 @@ public class RetryFactOutcomeTests /// /// Covers the outcome matrix: a non-retriable failure, an exhausted retry, a flaky test - /// that eventually passes, a non-assertion exception, a plain pass, and a clamped - /// configuration. Every one of these must appear exactly once, and the failures must not - /// be swallowed by the retry bookkeeping. + /// that eventually passes, a non-assertion exception, a plain pass, a run-time skip, and a + /// clamped configuration. Every one of these must appear exactly once, and the failures must + /// not be swallowed by the retry bookkeeping. The skip pins that a test which excuses itself + /// is neither retried nor turned into a pass. /// [Fact] public void GivenRetryingTests_WhenTheRunCompletes_ThenEachTestIsReportedExactlyOnce() @@ -36,6 +37,7 @@ public void GivenRetryingTests_WhenTheRunCompletes_ThenEachTestIsReportedExactly [Prefix + "NonAssertionException_IsReportedFailedOnce"] = "Failed", [Prefix + "FlakyThenPasses_IsReportedPassedOnce"] = "Passed", [Prefix + "AlwaysPasses_IsReportedPassed"] = "Passed", + [Prefix + "SkippedAtRunTime_IsReportedSkipped"] = "NotExecuted", [Prefix + "ClampedRetryConfiguration_RunsExactlyOnce"] = "Passed", }); } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index 8f03c9958b..ebefe634bb 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -46,8 +46,13 @@ public static class TestAssetRunner /// supported way to reach the delay-enumerated code path, which a theory also reaches on /// its own when its data cannot be serialized. /// + /// + /// A Name=Value trait filter to apply on top of the namespace filter, in the form the + /// runner takes on the command line. CI legs select their tests this way, so this is how a + /// scenario checks that it is still selected when they do. + /// /// The exit code, output and published results of the run. - public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true) + public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null) { string assetsAssembly = ResolveAssetsAssembly(); string resultsDirectory = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", Guid.NewGuid().ToString("N")); @@ -88,6 +93,12 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool pr arguments.Add("off"); } + if (!string.IsNullOrEmpty(filterTrait)) + { + arguments.Add("--filter-trait"); + arguments.Add(filterTrait); + } + (int exitCode, string output, TimeSpan duration) = Execute(arguments); string trxPath = Path.Combine(resultsDirectory, trxFileName); diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 6935eb3077..7c2d4999f9 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -39,6 +39,12 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl { return await FindTestsForTypeCore(testClass, discoveryOptions, callback); } + catch (OperationCanceledException) + { + // A cancelled run is not a discovery fault. Reporting it as a failing test case would + // turn pressing Ctrl+C, or a runner-imposed timeout, into a test failure. + throw; + } catch (Exception ex) { // An exception thrown out of discovery is reported by xunit.v3 only as an internal @@ -50,6 +56,16 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl Console.WriteLine( $"[FixtureArgumentSets] ERROR: discovery of '{testClass.TestClassName}' failed, so its tests were replaced by a single failing test case. {ex}"); + // The console line above is the authoritative record of the fault. The test case below + // puts the failure in the results and in the exit code, but it cannot be relied on to + // carry the cause: a case travels through the ordinary class runner, which builds the + // class's fixtures first, and a class using fixture argument sets has by definition a + // fixture taking an argument the failed expansion never produced. That fixture failure + // is aggregated ahead of this message and is what the report ends up showing. Both + // ways out of that were tried and are worse - anchoring the case to a class of its own + // takes it out of the namespace and class filters that would have selected the tests + // it replaces, and discarding the aggregated failure discards this message with it, + // leaving the fault unreported again. MethodInfo firstMethod = testClass.Methods.FirstOrDefault(); if (firstMethod == null) { @@ -58,6 +74,14 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl throw; } + // The case stays on the class that failed, rather than on a class of its own, so that + // it is still selected by whatever namespace or class filter would have selected the + // tests it replaces. FixtureArgumentSetClassRunner keeps the class's fixtures from + // pre-empting the message it carries. + // The case stays on the class that failed, rather than on a class of its own, so that + // it is still selected by whatever namespace or class filter would have selected the + // tests it replaces. It is given the argument set traits for the same reason, since a + // leg selecting by those would otherwise pass with the class silently missing. var errorTestMethod = new XunitTestMethod( testClass, firstMethod, @@ -67,19 +91,76 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl // The test case is handed to xunit through the callback, which owns it from then on // and disposes it with the rest of the discovered cases. #pragma warning disable CA2000 - var errorTestCase = new ExecutionErrorTestCase( + var errorTestCase = new DiscoveryErrorTestCase( errorTestMethod, $"{testClass.TestClassName} (fixture argument set discovery)", $"{testClass.UniqueID}-fixture-argument-set-discovery-error", - sourceFilePath: null, - sourceLineNumber: null, - errorMessage: $"Discovering the fixture argument set variants of '{testClass.TestClassName}' failed, so none of its tests ran. {ex}"); + errorMessage: $"Discovering the fixture argument set variants of '{testClass.TestClassName}' failed, so none of its tests ran. {ex}", + traits: BuildFaultTraits(testClass)); #pragma warning restore CA2000 return await callback(errorTestCase); } } + /// + /// Collects every argument set value the class asked for, so that a failure standing in for + /// its tests is selected by the same trait filters they would have been. + /// + /// + /// This runs while handling a discovery failure, and reads the same attribute whose contents + /// may well be what caused it, so it treats any further exception as simply having no traits + /// to offer. Reporting the failure without them is worth more than not reporting it at all. + /// + /// The class whose expansion failed. + /// The traits, keyed by argument enum type name; empty when none could be read. + private static Dictionary> BuildFaultTraits(IXunitTestClass testClass) + { + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + try + { + var attributes = new List(); + + if (testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() is FixtureArgumentSetsAttribute classAttribute) + { + attributes.Add(classAttribute); + } + + foreach (MethodInfo method in testClass.Methods) + { + if (method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() is FixtureArgumentSetsAttribute methodAttribute) + { + attributes.Add(methodAttribute); + } + } + + foreach (SingleFlag flag in attributes.SelectMany(ExpandEnumFlagsFromAttributeData).SelectMany(dimension => dimension ?? Array.Empty())) + { + if (flag.EnumValue == null) + { + continue; + } + + string key = flag.EnumValue.GetType().Name; + if (!traits.TryGetValue(key, out IReadOnlyCollection values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + traits[key] = values; + } + + ((HashSet)values).Add(flag.EnumValue.ToString()); + } + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the argument set traits of '{testClass.TestClassName}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + } + + return traits; + } + private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, ITestFrameworkDiscoveryOptions discoveryOptions, Func> callback) { var attribute = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; diff --git a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs new file mode 100644 index 0000000000..90ab07273d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs @@ -0,0 +1,68 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Reflection; +using EnsureThat; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit +{ + /// + /// A test case standing in for a class whose fixture argument sets could not be expanded, carrying + /// the traits the expansion never got far enough to produce. + /// + /// + /// A fixture argument set class is normally expanded into one variant per argument, and each + /// variant carries a trait naming its argument. Legs that select tests by those traits are the + /// reason this type exists: takes no traits and so would be + /// dropped by a positive trait filter, which puts the run back to passing with the broken class + /// silently missing - the exact outcome reporting the fault as a test was meant to prevent. + /// Declaring every value the class asked for means any filter that would have selected any of its + /// variants also selects the failure. + /// + internal sealed class DiscoveryErrorTestCase : ExecutionErrorTestCase + { + private static readonly FieldInfo TraitsField = typeof(XunitTestCase) + .GetField("traits", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + "XunitTestCase.traits was not found, so a discovery failure cannot be given the traits of the tests it " + + "replaces and would be hidden by a trait filter. This usually means the xunit.v3 version changed."); + + public DiscoveryErrorTestCase( + IXunitTestMethod testMethod, + string testCaseDisplayName, + string uniqueID, + string errorMessage, + IReadOnlyDictionary> traits) + : base(testMethod, testCaseDisplayName, uniqueID, sourceFilePath: null, sourceLineNumber: null, errorMessage: errorMessage) + { + EnsureArg.IsNotNull(traits, nameof(traits)); + + if (TraitsField.GetValue(this) is not Dictionary> existing) + { + return; + } + + foreach (KeyValuePair> trait in traits) + { + if (!existing.TryGetValue(trait.Key, out HashSet values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + existing[trait.Key] = values; + } + + values.UnionWith(trait.Value); + } + } + +#pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes + public DiscoveryErrorTestCase() + { + } +#pragma warning restore CS0618 + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index d256b052bc..b4a01ad8f6 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -208,7 +208,10 @@ public async ValueTask Run( // A test that was skipped or left unrun did not pass, and the counts saying so // live on the attempt's summary. Returning only Total and Failed would report - // it to anything reading this summary as a test that ran and passed. + // it to anything reading this summary as a test that ran and passed. The + // Microsoft Testing Platform runner is not such a reader - it counts the + // messages instead, and reports the skip correctly either way - so this keeps + // the summary contract rather than fixing anything visible in a CI report. runSummary.Skipped = summary.Skipped; runSummary.NotRun = summary.NotRun; return runSummary; From c4d5cd8879f8b166c0752178c0b8f0036b356eaa Mon Sep 17 00:00:00 2001 From: mikaelweave Date: Fri, 21 Aug 2026 09:49:34 -0700 Subject: [PATCH 048/105] Restore parallelism bound and contain discovery faults to one method The custom assembly and collection runners replaced xunit's own dispatch, and in doing so dropped two things it does at those boundaries. Parallelism: xunit's conservative algorithm acquires the parallel semaphore inside the assembly context's RunTestCollection, which is the exact method the override replaced, so --max-threads was not enforced at all and every collection in an assembly ran at once. A custom assembly runner context now owns that semaphore and the override waits on it. Exception aggregation: ExceptionAggregator is a struct over a mutable list, and xunit clones it when passing it from assembly to collection and from collection to class. The overrides passed it straight through, so every concurrently running collection shared one list of exceptions. Discovery faults were reported for a whole class, but cases are published to xunit as the method loop walks, so a fault let out of that loop left the methods already published running and dropped the ones after it, under a message claiming none of the class had run. Each method is now expanded on its own. The failure standing in for a method also carries that method's ordinary traits as well as its argument set values, because CI legs combine the two in filters such as (DataStore=CosmosDb)&(Category=ExportLongRunning) and a fault missing either half is filtered out of the leg that would have run the tests it replaces. A failure inside the caller's callback is no longer reported as a fixture argument set fault, an argument set that expands to no variants now fails the run rather than warning, and DiscoveryErrorTestCase says so when it cannot apply traits instead of returning silently. Adds scenarios covering bounded parallelism, empty argument sets, per-method fault containment, and the SkippableFact and Skip.If compatibility shims, which had no coverage despite around 150 call sites. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../BoundedParallelismFourTests.cs | 23 + .../BoundedParallelismOneTests.cs | 23 + .../BoundedParallelismTestBase.cs | 47 +++ .../BoundedParallelismThreeTests.cs | 23 + .../BoundedParallelismTwoTests.cs | 23 + .../BoundedParallelism/ConcurrencyTracker.cs | 28 ++ .../EmptyArgumentSet/EmptyArgumentSetTests.cs | 32 ++ .../PartialDiscoveryFaultTests.cs | 53 +++ .../Scenarios/SkipShims/SkipShimTests.cs | 87 ++++ .../BoundedParallelismTests.cs | 45 ++ .../DiscoveryFaultTests.cs | 2 +- .../EmptyArgumentSetTests.cs | 56 +++ .../PartialDiscoveryFaultTests.cs | 80 ++++ .../SkipShimTests.cs | 59 +++ .../TestAssetRunner.cs | 12 +- .../CustomXunitTestFrameworkDiscoverer.cs | 399 ++++++++++++------ .../CustomXunitTestFrameworkExecutor.cs | 114 ++++- .../DiscoveryErrorTestCase.cs | 9 + 18 files changed, 976 insertions(+), 139 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismFourTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismOneTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTestBase.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismThreeTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTwoTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/ConcurrencyTracker.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/EmptyArgumentSet/EmptyArgumentSetTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartialDiscoveryFault/PartialDiscoveryFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipShims/SkipShimTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/BoundedParallelismTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipShimTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismFourTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismFourTests.cs new file mode 100644 index 0000000000..cea28659d9 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismFourTests.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The fourth of four classes, and so the fourth of four collections, in this scenario. + /// + public class BoundedParallelismFourTests : BoundedParallelismTestBase + { + /// + /// Runs the shared body that observes how many tests are running at once. + /// + /// A task that completes when the test has finished. + [Fact] + public Task Runs() => RunBody(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismOneTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismOneTests.cs new file mode 100644 index 0000000000..c53b69afc5 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismOneTests.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The first of four classes, and so the first of four collections, in this scenario. + /// + public class BoundedParallelismOneTests : BoundedParallelismTestBase + { + /// + /// Runs the shared body that observes how many tests are running at once. + /// + /// A task that completes when the test has finished. + [Fact] + public Task Runs() => RunBody(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTestBase.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTestBase.cs new file mode 100644 index 0000000000..223f70b137 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTestBase.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The body every class in this scenario runs, so that each class contributes one test that + /// overlaps with the others for long enough to be observed. + /// + /// + /// Each class is its own collection, and collections are the unit of parallelization, so running + /// this scenario with a thread limit of one must serialize them. Asserting on the count taken as + /// a test enters, rather than on a maximum read at the end, keeps the check free of races: if the + /// limit is not applied the classes start together and all but one see a count above the limit. + /// + public abstract class BoundedParallelismTestBase + { + /// + /// Occupies the thread limit for long enough that any other class allowed to start would be + /// seen doing so. + /// + /// A task that completes when the body has finished. + protected static async Task RunBody() + { + int concurrent = ConcurrencyTracker.Enter(); + + try + { + Assert.True( + concurrent <= 1, + $"{concurrent} tests ran at once under a thread limit of one, so collection parallelism is not being bounded."); + + await Task.Delay(TimeSpan.FromMilliseconds(300)); + } + finally + { + ConcurrencyTracker.Exit(); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismThreeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismThreeTests.cs new file mode 100644 index 0000000000..e0b9083bc4 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismThreeTests.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The third of four classes, and so the third of four collections, in this scenario. + /// + public class BoundedParallelismThreeTests : BoundedParallelismTestBase + { + /// + /// Runs the shared body that observes how many tests are running at once. + /// + /// A task that completes when the test has finished. + [Fact] + public Task Runs() => RunBody(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTwoTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTwoTests.cs new file mode 100644 index 0000000000..1b3e210422 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/BoundedParallelismTwoTests.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// The second of four classes, and so the second of four collections, in this scenario. + /// + public class BoundedParallelismTwoTests : BoundedParallelismTestBase + { + /// + /// Runs the shared body that observes how many tests are running at once. + /// + /// A task that completes when the test has finished. + [Fact] + public Task Runs() => RunBody(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/ConcurrencyTracker.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/ConcurrencyTracker.cs new file mode 100644 index 0000000000..4692e7d7a3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/BoundedParallelism/ConcurrencyTracker.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism +{ + /// + /// Counts how many tests of this scenario are inside their body at the same time. + /// + public static class ConcurrencyTracker + { + private static int _current; + + /// + /// Records entry into a test body. + /// + /// The number of tests inside a body once this one has entered. + public static int Enter() => Interlocked.Increment(ref _current); + + /// + /// Records that a test body has been left. + /// + public static void Exit() => Interlocked.Decrement(ref _current); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/EmptyArgumentSet/EmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/EmptyArgumentSet/EmptyArgumentSetTests.cs new file mode 100644 index 0000000000..837aa21069 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/EmptyArgumentSet/EmptyArgumentSetTests.cs @@ -0,0 +1,32 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.EmptyArgumentSet +{ + /// + /// A class declaring a fixture argument set that names no value, so it expands to no variants. + /// + /// + /// The method asks to be expanded over an argument set of zero, which names no flag, so the + /// product of the declared dimensions is empty and the method produces no test cases at all. + /// The class deliberately takes no fixture, so nothing else can fail ahead of the reported case + /// and the message it carries is the one the discoverer wrote. + /// + public class EmptyArgumentSetTests + { + /// + /// Never runs: the argument set it asks for expands to nothing. + /// + [Fact] + [AssetArgumentSets((AssetDataStore)0)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartialDiscoveryFault/PartialDiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartialDiscoveryFault/PartialDiscoveryFaultTests.cs new file mode 100644 index 0000000000..7cf9ec25ca --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartialDiscoveryFault/PartialDiscoveryFaultTests.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartialDiscoveryFault +{ + /// + /// A class where one method's fixture argument set cannot be expanded and the methods around it + /// are declared normally. + /// + /// + /// The failing method is declared between the two healthy ones so that the scenario covers both + /// sides of it: a method discovered before the failure and a method discovered after it. The + /// class takes no fixture, so nothing can fail ahead of the reported case and the message it + /// carries is the discoverer's own. + /// + public class PartialDiscoveryFaultTests + { + /// + /// Runs normally, and is declared before the failing method. + /// + [Fact] + public void RunsBeforeTheFault() + { + Assert.True(true); + } + + /// + /// Never runs: the argument set it asks for names no value, so it expands to nothing. It + /// carries an ordinary trait so that a scenario can check the reported failure keeps it. + /// + [Fact] + [Trait("AssetCategory", "PartialFault")] + [AssetArgumentSets((AssetDataStore)0)] + public void NeverRuns() + { + Assert.True(true); + } + + /// + /// Runs normally, and is declared after the failing method. + /// + [Fact] + public void RunsAfterTheFault() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipShims/SkipShimTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipShims/SkipShimTests.cs new file mode 100644 index 0000000000..42f2f5275c --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipShims/SkipShimTests.cs @@ -0,0 +1,87 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipShims +{ + /// + /// Exercises the source-compatibility shims that keep the repository's legacy + /// SkippableFact and Skip.If call sites working on xunit.v3. + /// + /// + /// The shims stand in for a package that no longer exists on v3, so nothing but these scenarios + /// says whether they still do what the several hundred call sites using them assume. The + /// distinction that matters is skipped against failed: a conditional skip reported as a failure + /// would break every leg that skips tests it cannot run, and a skip that silently passed would + /// let a test that never ran count as one that did. + /// + public class SkipShimTests + { + /// + /// A true condition skips, and is reported as skipped rather than as a failure. + /// + [SkippableFact] + public void SkipIfTrue_IsSkipped() + { + Skip.If(true, "the condition held"); + Assert.Fail("Skip.If(true) should have stopped the test before this point."); + } + + /// + /// A false condition does not skip, so the test runs to completion and passes. + /// + [SkippableFact] + public void SkipIfFalse_Runs() + { + Skip.If(false, "the condition did not hold"); + Assert.True(true); + } + + /// + /// is the inverse, so a false condition skips. + /// + [SkippableFact] + public void SkipIfNotFalse_IsSkipped() + { + Skip.IfNot(false, "the condition did not hold"); + Assert.Fail("Skip.IfNot(false) should have stopped the test before this point."); + } + + /// + /// with a true condition does not skip. + /// + [SkippableFact] + public void SkipIfNotTrue_Runs() + { + Skip.IfNot(true, "the condition held"); + Assert.True(true); + } + + /// + /// The reason given at the call site has to reach the report, otherwise a skipped leg gives + /// no indication of why its tests did not run. + /// + [SkippableFact] + public void SkipWithReason_IsSkipped() + { + Skip.If(true, "a distinctive skip reason"); + Assert.Fail("Skip.If(true) should have stopped the test before this point."); + } + + /// + /// The theory shim skips per data row, so the same method both skips and passes. + /// + /// Whether this row should skip. + [SkippableTheory] + [InlineData(true)] + [InlineData(false)] + public void SkippableTheory_SkipsPerRow(bool skip) + { + Skip.If(skip, "the row asked to be skipped"); + Assert.False(skip); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/BoundedParallelismTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/BoundedParallelismTests.cs new file mode 100644 index 0000000000..fbff189028 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/BoundedParallelismTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that substituting the collection runner does not remove the limit on how many + /// collections run at once. + /// + public class BoundedParallelismTests + { + private const string ScenarioNamespace = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.BoundedParallelism."; + + /// + /// xunit bounds collection parallelism inside its own assembly runner context, in the very + /// dispatch this framework replaces in order to substitute its collection runner. Replacing + /// it without carrying the bound over leaves the thread limit unenforced, so every collection + /// in an assembly starts at once however the run was configured. That does not fail anything + /// by itself, which is what makes it worth pinning: it shows up only as a machine under far + /// more load than asked for, and as the flaky timing-sensitive tests that come with it. + /// + [Fact] + public void GivenAThreadLimitOfOne_WhenCollectionsRun_ThenTheyDoNotOverlap() + { + TestAssetRun run = TestAssetRunner.Run("BoundedParallelism", maxThreads: "1"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioNamespace + "BoundedParallelismOneTests.Runs"] = "Passed", + [ScenarioNamespace + "BoundedParallelismTwoTests.Runs"] = "Passed", + [ScenarioNamespace + "BoundedParallelismThreeTests.Runs"] = "Passed", + [ScenarioNamespace + "BoundedParallelismFourTests.Runs"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs index 572138c599..17a41e5f5f 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests /// public class DiscoveryFaultTests { - private const string ErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests (fixture argument set discovery)"; + private const string ErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery)"; /// /// xUnit reports an exception thrown out of discovery only as an internal diagnostic, which diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs new file mode 100644 index 0000000000..2ba9b1808b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs @@ -0,0 +1,56 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a fixture argument set which names no value fails the run rather than quietly + /// removing the tests that asked for it. + /// + public class EmptyArgumentSetTests + { + private const string ErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.EmptyArgumentSet.EmptyArgumentSetTests.NeverRuns (fixture argument set discovery)"; + + /// + /// An argument set naming no flag collapses the product of the declared dimensions to nothing, + /// so the method produces no test cases. Left alone that is indistinguishable from a class with + /// no tests: the run reports success with the tests simply absent, which is the worst way for a + /// test to stop running because nothing in the report says it ever existed. A declared argument + /// set that expands to nothing is always a misconfiguration, so it is reported as a failure. + /// + [Fact] + public void GivenAnArgumentSetThatNamesNoValue_WhenTheClassIsDiscovered_ThenTheRunFails() + { + TestAssetRun run = TestAssetRunner.Run("EmptyArgumentSet"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The class takes no fixture, so nothing is aggregated ahead of the reported case and the + /// message it carries is the discoverer's own. That is what makes this failure actionable: + /// it names the method and says what is wrong with the argument set it declared. + /// + [Fact] + public void GivenAnArgumentSetThatNamesNoValue_WhenTheClassIsDiscovered_ThenTheFailureNamesTheMethod() + { + TestAssetRun run = TestAssetRunner.Run("EmptyArgumentSet"); + + Assert.Contains("EmptyArgumentSetTests.NeverRuns", run.Output, StringComparison.Ordinal); + Assert.Contains("expanded to no fixture argument sets", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs new file mode 100644 index 0000000000..2e0e079b50 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs @@ -0,0 +1,80 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that one method whose fixture argument sets cannot be expanded costs only that + /// method, and that the failure standing in for it is selected by the same filters it was. + /// + public class PartialDiscoveryFaultTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartialDiscoveryFault.PartialDiscoveryFaultTests"; + private const string ErrorCaseName = ScenarioClass + ".NeverRuns (fixture argument set discovery)"; + + /// + /// Test cases are published to xUnit as the discoverer walks the class's methods, so a + /// failure let out of that walk leaves the methods already published running while the + /// methods after it are silently dropped - and reports that as a single failure claiming + /// the whole class never ran, which is wrong in both directions. Expanding each method + /// separately keeps the blast radius to the method that is actually misdeclared. + /// + [Fact] + public void GivenOneMethodThatCannotBeExpanded_WhenTheClassIsDiscovered_ThenTheOtherMethodsStillRun() + { + TestAssetRun run = TestAssetRunner.Run("PartialDiscoveryFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".RunsBeforeTheFault"] = "Passed", + [ScenarioClass + ".RunsAfterTheFault"] = "Passed", + [ErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failure has to say which method was lost, otherwise a class of many methods reports + /// a fault with no indication of where to look for it. + /// + [Fact] + public void GivenOneMethodThatCannotBeExpanded_WhenTheClassIsDiscovered_ThenTheFailureNamesOnlyThatMethod() + { + TestAssetRun run = TestAssetRunner.Run("PartialDiscoveryFault"); + + Assert.Contains("PartialDiscoveryFaultTests.NeverRuns", run.Output, StringComparison.Ordinal); + Assert.Contains("Other methods of the class were discovered normally", run.Output, StringComparison.Ordinal); + } + + /// + /// CI legs combine ordinary traits with argument set values in filters such as + /// (DataStore=CosmosDb)&(Category=ExportLongRunning). The argument set half of + /// such a filter is covered elsewhere; this covers the ordinary half, which the failing + /// method declares itself. Without it the leg that would have run the method passes with + /// the method silently missing, which is the failure this mechanism exists to prevent. + /// + [Fact] + public void GivenOneMethodThatCannotBeExpanded_WhenTestsAreSelectedByItsOrdinaryTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("PartialDiscoveryFault", filterTrait: "AssetCategory=PartialFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipShimTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipShimTests.cs new file mode 100644 index 0000000000..e9c80f6b58 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipShimTests.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies the source-compatibility shims that keep the repository's legacy + /// SkippableFact and Skip.If call sites working on xunit.v3. + /// + public class SkipShimTests + { + private const string Prefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipShims.SkipShimTests."; + + /// + /// The shims replace a package that has no xunit.v3 release, so their behaviour is defined + /// entirely by this repository and nothing else checks it. The outcome is what matters: a + /// conditional skip reported as a failure would break every leg that skips tests it cannot + /// run, and one reported as a pass would let a test that never ran count as one that did. + /// + [Fact] + public void GivenTestsUsingTheSkipShims_WhenTheyRun_ThenEachIsReportedWithTheOutcomeItsConditionAsksFor() + { + TestAssetRun run = TestAssetRunner.Run("SkipShims"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [Prefix + "SkipIfTrue_IsSkipped"] = "NotExecuted", + [Prefix + "SkipIfFalse_Runs"] = "Passed", + [Prefix + "SkipIfNotFalse_IsSkipped"] = "NotExecuted", + [Prefix + "SkipIfNotTrue_Runs"] = "Passed", + [Prefix + "SkipWithReason_IsSkipped"] = "NotExecuted", + [Prefix + "SkippableTheory_SkipsPerRow(skip: True)"] = "NotExecuted", + [Prefix + "SkippableTheory_SkipsPerRow(skip: False)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + + /// + /// A skipped test is only actionable if the report says why it was skipped, so the reason + /// given at the call site has to survive as far as the output. + /// + [Fact] + public void GivenATestSkippedWithAReason_WhenItRuns_ThenTheReasonReachesTheOutput() + { + TestAssetRun run = TestAssetRunner.Run("SkipShims"); + + Assert.Contains("a distinctive skip reason", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index ebefe634bb..4a8a9f113b 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -51,8 +51,12 @@ public static class TestAssetRunner /// runner takes on the command line. CI legs select their tests this way, so this is how a /// scenario checks that it is still selected when they do. /// + /// + /// The value to pass as the runner's thread limit, in the form it takes on the command line. + /// Scenarios that observe how much runs at once set this so the expected bound is explicit. + /// /// The exit code, output and published results of the run. - public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null) + public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null, string maxThreads = null) { string assetsAssembly = ResolveAssetsAssembly(); string resultsDirectory = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", Guid.NewGuid().ToString("N")); @@ -99,6 +103,12 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool pr arguments.Add(filterTrait); } + if (!string.IsNullOrEmpty(maxThreads)) + { + arguments.Add("--max-threads"); + arguments.Add(maxThreads); + } + (int exitCode, string output, TimeSpan duration) = Execute(arguments); string trxPath = Path.Combine(resultsDirectory, trxFileName); diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 7c2d4999f9..75886d814c 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -35,9 +35,11 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl EnsureArg.IsNotNull(callback, nameof(callback)); EnsureArg.IsNotNull(discoveryOptions, nameof(discoveryOptions)); + var tracker = new CallbackTracker(callback); + try { - return await FindTestsForTypeCore(testClass, discoveryOptions, callback); + return await FindTestsForTypeCore(testClass, discoveryOptions, tracker); } catch (OperationCanceledException) { @@ -45,6 +47,13 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl // turn pressing Ctrl+C, or a runner-imposed timeout, into a test failure. throw; } + catch (Exception ex) when (tracker.IsCallbackFailure(ex)) + { + // The caller's own sink threw. Describing that as a failure to expand this class would + // be wrong, and handing the caller a further test case is unlikely to fare any better, + // so it goes back to xunit to deal with as it would for any other class. + throw; + } catch (Exception ex) { // An exception thrown out of discovery is reported by xunit.v3 only as an internal @@ -53,71 +62,138 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl // success, so a broken expansion would look exactly like a class that has no tests. // Reporting a test case that fails on execution instead puts the fault in the results // and in the exit code, where it cannot be missed. - Console.WriteLine( - $"[FixtureArgumentSets] ERROR: discovery of '{testClass.TestClassName}' failed, so its tests were replaced by a single failing test case. {ex}"); - - // The console line above is the authoritative record of the fault. The test case below - // puts the failure in the results and in the exit code, but it cannot be relied on to - // carry the cause: a case travels through the ordinary class runner, which builds the - // class's fixtures first, and a class using fixture argument sets has by definition a - // fixture taking an argument the failed expansion never produced. That fixture failure - // is aggregated ahead of this message and is what the report ends up showing. Both - // ways out of that were tried and are worse - anchoring the case to a class of its own - // takes it out of the namespace and class filters that would have selected the tests - // it replaces, and discarding the aggregated failure discards this message with it, - // leaving the fault unreported again. - MethodInfo firstMethod = testClass.Methods.FirstOrDefault(); + // + // Reaching here means the failure was outside the per-method loop, which reports its + // own faults, so it belongs to the class as a whole and no method of it was expanded. + // + // testClass.Methods includes every public method, property accessors among them, so + // the anchor is chosen from those xunit would treat as tests: a fault reported + // against a property getter names nothing a reader can act on, and the getter carries + // none of the traits a CI leg selects the class's tests by. + MethodInfo firstMethod = testClass.Methods.FirstOrDefault(IsTestMethod) ?? testClass.Methods.FirstOrDefault(); if (firstMethod == null) { // Nothing to hang a test case off. Rethrowing keeps the original xunit behaviour, - // which is all that is left, and the console line above is the only warning. + // which is all that is left. + Console.WriteLine( + $"[FixtureArgumentSets] ERROR: discovery of '{testClass.TestClassName}' failed and it declares no method to report the failure against. {ex}"); throw; } - // The case stays on the class that failed, rather than on a class of its own, so that - // it is still selected by whatever namespace or class filter would have selected the - // tests it replaces. FixtureArgumentSetClassRunner keeps the class's fixtures from - // pre-empting the message it carries. - // The case stays on the class that failed, rather than on a class of its own, so that - // it is still selected by whatever namespace or class filter would have selected the - // tests it replaces. It is given the argument set traits for the same reason, since a - // leg selecting by those would otherwise pass with the class silently missing. - var errorTestMethod = new XunitTestMethod( + return await ReportDiscoveryFault( testClass, firstMethod, - Array.Empty(), - uniqueID: UniqueIDGenerator.ForTestMethod(testClass.UniqueID, firstMethod.Name)); + tracker, + ex, + $"Discovering the fixture argument set variants of '{testClass.TestClassName}' failed, so none of its tests ran."); + } + } + + /// + /// Determines whether xunit would treat a method as a test. + /// + /// The method to inspect. + /// true when the method carries a fact or theory attribute. + private static bool IsTestMethod(MethodInfo method) => + method.GetCustomAttributes(typeof(FactAttribute), inherit: true).Length > 0; - // The test case is handed to xunit through the callback, which owns it from then on - // and disposes it with the rest of the discovered cases. + /// + /// Reports a discovery failure as a test case that fails when the run reaches it. + /// + /// The class being discovered. + /// The method to report the failure against. + /// The callback discovered test cases are handed to. + /// The failure being reported. + /// A sentence saying what was lost, used to open the failure message. + /// Whether discovery should continue, as the callback reported it. + /// + /// The console line written here is the authoritative record of the fault. The test case puts + /// the failure in the results and in the exit code, but it cannot be relied on to carry the + /// cause: a case travels through the ordinary class runner, which builds the class's fixtures + /// first, and a class using fixture argument sets typically has a fixture taking an argument + /// the failed expansion never produced. That fixture failure is aggregated ahead of this + /// message and is what the report then shows. Both ways out of that were tried and are worse - + /// anchoring the case to a class of its own takes it out of the namespace and class filters + /// that would have selected the tests it replaces, and discarding the aggregated failure + /// discards this message with it, leaving the fault unreported again. + /// + private static async ValueTask ReportDiscoveryFault( + IXunitTestClass testClass, + MethodInfo method, + CallbackTracker tracker, + Exception exception, + string summary) + { + Console.WriteLine($"[FixtureArgumentSets] ERROR: {summary} It was replaced by a failing test case. {exception}"); + + // The case stays on the class that failed, rather than on a class of its own, so that it + // is still selected by whatever namespace or class filter would have selected the tests it + // replaces, and it carries their traits so that a leg selecting by those cannot pass with + // the tests silently missing. + var errorTestMethod = new XunitTestMethod( + testClass, + method, + Array.Empty(), + uniqueID: UniqueIDGenerator.ForTestMethod(testClass.UniqueID, method.Name)); + + // The test case is handed to xunit through the callback, which owns it from then on + // and disposes it with the rest of the discovered cases. #pragma warning disable CA2000 - var errorTestCase = new DiscoveryErrorTestCase( - errorTestMethod, - $"{testClass.TestClassName} (fixture argument set discovery)", - $"{testClass.UniqueID}-fixture-argument-set-discovery-error", - errorMessage: $"Discovering the fixture argument set variants of '{testClass.TestClassName}' failed, so none of its tests ran. {ex}", - traits: BuildFaultTraits(testClass)); + var errorTestCase = new DiscoveryErrorTestCase( + errorTestMethod, + $"{testClass.TestClassName}.{method.Name} (fixture argument set discovery)", + $"{testClass.UniqueID}-{method.Name}-fixture-argument-set-discovery-error", + errorMessage: $"{summary} {exception}", + traits: BuildFaultTraits(testClass, errorTestMethod)); #pragma warning restore CA2000 - return await callback(errorTestCase); - } + return await tracker.Invoke(errorTestCase); } /// - /// Collects every argument set value the class asked for, so that a failure standing in for - /// its tests is selected by the same trait filters they would have been. + /// Collects the traits a failure standing in for a method's tests must carry to be selected by + /// the same filters they would have been. /// /// + /// Two kinds of trait matter. The ordinary ones the method and its class declare come from the + /// anchoring method itself, and CI legs combine them with argument set values in filters such + /// as (DataStore=CosmosDb)&(Category=ExportLongRunning), so leaving either out lets + /// the leg that would have run these tests pass without them. The argument set values are the + /// ones the expansion never got far enough to produce, so they have to be read back from the + /// attributes here. + /// /// This runs while handling a discovery failure, and reads the same attribute whose contents - /// may well be what caused it, so it treats any further exception as simply having no traits - /// to offer. Reporting the failure without them is worth more than not reporting it at all. + /// may well be what caused it, so it treats any further exception as simply having no argument + /// set traits to offer. Reporting the failure with fewer traits is worth more than not + /// reporting it at all. + /// /// /// The class whose expansion failed. - /// The traits, keyed by argument enum type name; empty when none could be read. - private static Dictionary> BuildFaultTraits(IXunitTestClass testClass) + /// The method the failure is reported against. + /// The traits, keyed by trait name; empty when none could be read. + private static Dictionary> BuildFaultTraits(IXunitTestClass testClass, XunitTestMethod anchor) { var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + void Add(string key, string value) + { + if (!traits.TryGetValue(key, out IReadOnlyCollection values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + traits[key] = values; + } + + ((HashSet)values).Add(value); + } + + foreach (KeyValuePair> trait in anchor.Traits) + { + foreach (string value in trait.Value) + { + Add(trait.Key, value); + } + } + try { var attributes = new List(); @@ -142,14 +218,7 @@ private static Dictionary> BuildFaultTraits( continue; } - string key = flag.EnumValue.GetType().Name; - if (!traits.TryGetValue(key, out IReadOnlyCollection values)) - { - values = new HashSet(StringComparer.OrdinalIgnoreCase); - traits[key] = values; - } - - ((HashSet)values).Add(flag.EnumValue.ToString()); + Add(flag.EnumValue.GetType().Name, flag.EnumValue.ToString()); } } catch (Exception ex) @@ -161,7 +230,7 @@ private static Dictionary> BuildFaultTraits( return traits; } - private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, ITestFrameworkDiscoveryOptions discoveryOptions, Func> callback) + private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, ITestFrameworkDiscoveryOptions discoveryOptions, CallbackTracker tracker) { var attribute = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; var methodAttributes = testClass.Methods.ToDictionary( @@ -170,7 +239,7 @@ private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, IT if (attribute == null && methodAttributes.Values.All(value => value == null)) { - return await base.FindTestsForType(testClass, discoveryOptions, callback); + return await base.FindTestsForType(testClass, discoveryOptions, tracker.Invoke); } SingleFlag[][] classLevelOpenParameterSets = Array.Empty(); @@ -187,99 +256,136 @@ private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, IT foreach (var method in testClass.Methods) { - var fixtureParameterAttribute = methodAttributes[method]; - - if (attribute == null && fixtureParameterAttribute == null) + // Each method is expanded on its own so that one method's misdeclared argument set + // costs only that method. Letting the failure out of the loop would leave the methods + // already published running while the methods after it disappeared, under a single + // failure claiming the whole class had not run. + bool continueDiscovery; + try { - var passthroughTestMethod = new XunitTestMethod(testClass, method, Array.Empty(), uniqueID: UniqueIDGenerator.ForTestMethod(testClass.UniqueID, method.Name)); - if (!await FindTestsForMethod(passthroughTestMethod, discoveryOptions, callback)) - { - return false; - } - - continue; + continueDiscovery = await FindTestsForMethodCore(testClass, method, attribute, methodAttributes[method], classLevelOpenParameterSets, classLevelClosedParameterSets, discoveryOptions, tracker); } - - SingleFlag[][] closedSets = classLevelClosedParameterSets; - - if (attribute == null) + catch (OperationCanceledException) { - // Method-level parameter sets with no class-level fallback. - SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); - closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + throw; } - else if (fixtureParameterAttribute != null) + catch (Exception ex) when (!tracker.IsCallbackFailure(ex)) + { + continueDiscovery = await ReportDiscoveryFault( + testClass, + method, + tracker, + ex, + $"Discovering the fixture argument set variants of '{testClass.TestClassName}.{method.Name}' failed, so none of that method's tests ran. Other methods of the class were discovered normally."); + } + + if (!continueDiscovery) { - // get the method-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) - SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); + return false; + } + } + + return true; + } - bool hasOverride = false; - for (int i = 0; i < methodLevelOpenParameterSets.Length; i++) + private async ValueTask FindTestsForMethodCore( + IXunitTestClass testClass, + MethodInfo method, + FixtureArgumentSetsAttribute attribute, + FixtureArgumentSetsAttribute fixtureParameterAttribute, + SingleFlag[][] classLevelOpenParameterSets, + SingleFlag[][] classLevelClosedParameterSets, + ITestFrameworkDiscoveryOptions discoveryOptions, + CallbackTracker tracker) + { + if (attribute == null && fixtureParameterAttribute == null) + { + var passthroughTestMethod = new XunitTestMethod(testClass, method, Array.Empty(), uniqueID: UniqueIDGenerator.ForTestMethod(testClass.UniqueID, method.Name)); + return await FindTestsForMethod(passthroughTestMethod, discoveryOptions, tracker.Invoke); + } + + SingleFlag[][] closedSets = classLevelClosedParameterSets; + + if (attribute == null) + { + // Method-level parameter sets with no class-level fallback. + SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); + closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + } + else if (fixtureParameterAttribute != null) + { + // get the method-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) + SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); + + bool hasOverride = false; + for (int i = 0; i < methodLevelOpenParameterSets.Length; i++) + { + if (methodLevelOpenParameterSets[i]?.Length > 0) { - if (methodLevelOpenParameterSets[i]?.Length > 0) - { - hasOverride = true; - } - else - { - // means take the class-level set - methodLevelOpenParameterSets[i] = classLevelOpenParameterSets[i]; - } + hasOverride = true; } - - if (hasOverride) + else { - // convert to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) - closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + // means take the class-level set + methodLevelOpenParameterSets[i] = classLevelOpenParameterSets[i]; } } - if (closedSets.Length == 0) + if (hasOverride) { - // A dimension that contributes no flags -- an argument set of zero, a value that - // names no single flag, or a null entry -- collapses the cartesian product to - // nothing, and this method then produces no test cases at all. The run stays - // green with the tests simply absent, so say so rather than let them vanish. - Console.WriteLine( - $"[FixtureArgumentSets] WARNING: '{testClass.TestClassName}.{method.Name}' expanded to no fixture argument sets, so none of its tests will run. " + - "Check that every argument set names at least one single-valued flag of a [Flags] enum."); + // convert to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) + closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); } + } - foreach (SingleFlag[] closedVariant in closedSets) + if (closedSets.Length == 0) + { + // Reaching here means an argument set was declared, so a product of nothing is a + // misconfiguration -- an argument set of zero, a value naming no single flag, or a + // null entry -- and every test on this method would otherwise be absent from a run + // that still reported success. Throwing hands it to the handler in FindTestsForType, + // which reports it as a failing test case so it shows up in the results and in the + // exit code. Discovery over every test assembly in this repository produces no such + // case, so this only fires on a class that is genuinely misdeclared. + throw new InvalidOperationException( + $"'{testClass.TestClassName}.{method.Name}' expanded to no fixture argument sets, so none of its tests would run. " + + "Check that every argument set names at least one single-valued flag of a [Flags] enum."); + } + + foreach (SingleFlag[] closedVariant in closedSets) + { + // Every variant stays in the class's original collection. xUnit's unit of + // parallelization is the collection, so giving each variant its own would let + // classes an explicit [Collection] deliberately grouped together -- and the + // variants of a single class -- start running concurrently. Only the identifiers + // below carry the variant, so the collection keeps its original grouping. + var variantKey = BuildVariantKey(testClass.TestCollection, closedVariant); + var classKey = BuildVariantClassKey(variantKey, testClass.Class); + var closedVariantTestClass = _variantClassCache.GetOrAdd( + classKey, + _ => new FixtureArgumentSetTestClass( + testClass.Class, + testClass.TestCollection, + closedVariant, + UniqueIDGenerator.ForTestClass(testClass.TestCollection.UniqueID, classKey))); + + var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); + + closedVariantTestMethod.UpdateArgumentsFromMethod(); + + // xUnit builds the display name from the class and method names only, so every + // variant of a method would otherwise be reported under an identical name and + // a failure could not be attributed to a specific fixture argument set. + var variantArguments = closedVariant; + Func> variantCallback = testCase => { - // Every variant stays in the class's original collection. xUnit's unit of - // parallelization is the collection, so giving each variant its own would let - // classes an explicit [Collection] deliberately grouped together -- and the - // variants of a single class -- start running concurrently. Only the identifiers - // below carry the variant, so the collection keeps its original grouping. - var variantKey = BuildVariantKey(testClass.TestCollection, closedVariant); - var classKey = BuildVariantClassKey(variantKey, testClass.Class); - var closedVariantTestClass = _variantClassCache.GetOrAdd( - classKey, - _ => new FixtureArgumentSetTestClass( - testClass.Class, - testClass.TestCollection, - closedVariant, - UniqueIDGenerator.ForTestClass(testClass.TestCollection.UniqueID, classKey))); - - var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); - - closedVariantTestMethod.UpdateArgumentsFromMethod(); - - // xUnit builds the display name from the class and method names only, so every - // variant of a method would otherwise be reported under an identical name and - // a failure could not be attributed to a specific fixture argument set. - var variantArguments = closedVariant; - Func> variantCallback = testCase => - { - ApplyVariantDisplayName(testCase, variantArguments); - return callback(testCase); - }; + ApplyVariantDisplayName(testCase, variantArguments); + return tracker.Invoke(testCase); + }; - if (!await FindTestsForMethod(closedVariantTestMethod, discoveryOptions, variantCallback)) - { - return false; - } + if (!await FindTestsForMethod(closedVariantTestMethod, discoveryOptions, variantCallback)) + { + return false; } } @@ -412,5 +518,40 @@ public static IEnumerable> CartesianProduct(IEnume emptyProduct, (accumulator, sequence) => accumulator.SelectMany(a => sequence.Select(s => a.Concat(Enumerable.Repeat(s, 1))))); } + + /// + /// Wraps the callback discovered test cases are handed to, so that a failure inside it can be + /// told apart from a failure to expand fixture argument sets. + /// + /// + /// The callback belongs to xunit rather than to this discoverer. Reporting a failure inside it + /// as a fixture argument set fault would name the wrong culprit, and would try to report that + /// through the very callback that has just thrown. + /// + private sealed class CallbackTracker + { + private readonly Func> _callback; + private Exception _failure; + + public CallbackTracker(Func> callback) + { + _callback = callback; + } + + public async ValueTask Invoke(ITestCase testCase) + { + try + { + return await _callback(testCase); + } + catch (Exception ex) + { + _failure = ex; + throw; + } + } + + public bool IsCallbackFailure(Exception exception) => ReferenceEquals(exception, _failure); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index e5205996a9..0c3d6160d8 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -31,25 +31,123 @@ public override async ValueTask RunTestCases(IReadOnlyCollection await runner.Run(TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); } - private sealed class FixtureArgumentSetAssemblyRunner : XunitTestAssemblyRunner + /// + /// The assembly runner context, extended to bound how many collections run at once. + /// + /// + /// xunit applies the conservative parallel algorithm by acquiring a semaphore inside its own + /// context's collection dispatch. Substituting a collection runner means not going through + /// that dispatch, which would leave MaxParallelThreads unenforced and start every + /// collection at once. Its semaphore is private, so this context keeps an equivalent one of + /// its own and dispatches through it. + /// + private sealed class FixtureArgumentSetAssemblyRunnerContext : XunitTestAssemblyRunnerContext + { + private SemaphoreSlim _parallelSemaphore; + + public FixtureArgumentSetAssemblyRunnerContext( + IXunitTestAssembly testAssembly, + IReadOnlyCollection testCases, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions, + CancellationToken cancellationToken) + : base(testAssembly, testCases, executionMessageSink, executionOptions, cancellationToken) + { + } + + public override void SetupParallelism() + { + base.SetupParallelism(); + + // The aggressive algorithm bounds concurrency with a synchronization context, which + // the base call has already installed and which applies to this dispatch too. Only + // the conservative algorithm needs a semaphore here. + if (MaxParallelThreads > 0 && ParallelAlgorithm == ParallelAlgorithm.Conservative) + { + _parallelSemaphore = new SemaphoreSlim(initialCount: MaxParallelThreads); + } + } + + public async ValueTask RunFixtureArgumentSetCollection( + IXunitTestCollection testCollection, + IReadOnlyCollection testCases, + ITestCaseOrderer testCaseOrderer) + { + if (_parallelSemaphore != null) + { + await _parallelSemaphore.WaitAsync(CancellationTokenSource.Token); + } + + try + { + var runner = new FixtureArgumentSetCollectionRunner(); + + // ExceptionAggregator is a struct wrapping a mutable list, so passing this + // context's own would give every collection a shared list to append to and + // clear. Collections run concurrently, so one collection's fixture failure could + // then surface against another, or be cleared before it was ever reported. + return await runner.Run( + testCollection, + testCases, + ExplicitOption, + MessageBus, + testCaseOrderer, + Aggregator.Clone(), + CancellationTokenSource, + AssemblyFixtureMappings); + } + finally + { + _parallelSemaphore?.Release(); + } + } + + public override async ValueTask DisposeAsync() + { + _parallelSemaphore?.Dispose(); + await base.DisposeAsync(); + } + } + + private sealed class FixtureArgumentSetAssemblyRunner : XunitTestAssemblyRunnerBase { - protected override async ValueTask RunTestCollection(XunitTestAssemblyRunnerContext context, IXunitTestCollection testCollection, IReadOnlyCollection testCases) + public async ValueTask Run( + IXunitTestAssembly testAssembly, + IReadOnlyCollection testCases, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions, + CancellationToken cancellationToken) + { + await using var context = new FixtureArgumentSetAssemblyRunnerContext(testAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); + await context.InitializeAsync(); + + return await Run(context); + } + + protected override ValueTask RunTestCollection(FixtureArgumentSetAssemblyRunnerContext context, IXunitTestCollection testCollection, IReadOnlyCollection testCases) { var testCaseOrderer = context.AssemblyTestCaseOrderer ?? DefaultTestCaseOrderer.Instance; - var runner = new FixtureArgumentSetCollectionRunner(); - var summary = await runner.Run(testCollection, testCases, context.ExplicitOption, context.MessageBus, testCaseOrderer, context.Aggregator, context.CancellationTokenSource, context.AssemblyFixtureMappings); - return summary; + return context.RunFixtureArgumentSetCollection(testCollection, testCases, testCaseOrderer); } } private sealed class FixtureArgumentSetCollectionRunner : XunitTestCollectionRunner { - protected override async ValueTask RunTestClass(XunitTestCollectionRunnerContext context, IXunitTestClass testClass, IReadOnlyCollection testCases) + protected override ValueTask RunTestClass(XunitTestCollectionRunnerContext context, IXunitTestClass testClass, IReadOnlyCollection testCases) { + if (testClass == null) + { + // xunit reports cases with no class as failures with an explanatory message. + // Substituting a class runner must not turn that into an argument exception. + return base.RunTestClass(context, testClass, testCases); + } + var testCaseOrderer = context.TestCaseOrderer ?? DefaultTestCaseOrderer.Instance; var classRunner = new FixtureArgumentSetClassRunner(); - var summary = await classRunner.Run(testClass, testCases, context.ExplicitOption, context.MessageBus, testCaseOrderer, context.Aggregator, context.CancellationTokenSource, context.CollectionFixtureMappings); - return summary; + + // Cloned for the same reason as at the collection boundary: classes of a collection + // must not share one mutable list of exceptions. + return classRunner.Run(testClass, testCases, context.ExplicitOption, context.MessageBus, testCaseOrderer, context.Aggregator.Clone(), context.CancellationTokenSource, context.CollectionFixtureMappings); } } diff --git a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs index 90ab07273d..a3b08137d9 100644 --- a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs @@ -44,6 +44,15 @@ public DiscoveryErrorTestCase( if (TraitsField.GetValue(this) is not Dictionary> existing) { + // The field is still there but no longer holds what it did, so the traits cannot be + // applied. Reporting the case without them is still worth more than not reporting it: + // an unfiltered run keeps the failure, and only a run selecting by trait loses it. + // Throwing instead would be worse - xunit reports an exception out of discovery only + // as a suppressed internal diagnostic and drops the class, which is silence again. + Console.WriteLine( + $"[FixtureArgumentSets] ERROR: XunitTestCase.traits holds '{TraitsField.GetValue(this)?.GetType().FullName ?? "null"}' rather than " + + $"Dictionary>, so the failure standing in for '{testCaseDisplayName}' carries no traits and a run " + + $"selecting tests by trait will not see it. This usually means the xunit.v3 version changed."); return; } From 0f29c51bdc23b66045548edc6077aee589e7b0a2 Mon Sep 17 00:00:00 2001 From: mikaelweave Date: Fri, 21 Aug 2026 11:45:01 -0700 Subject: [PATCH 049/105] Report discovery faults per argument set and carry their real cause A discovery fault was reported as one test case declaring every argument set value the class asked for. A trait exclusion drops a case when any of its values match, so that single case was excluded by the SQL leg for holding the Cosmos value and by the Cosmos leg for holding the SQL one: the fault reached neither leg and both stayed green with the class gone. One case is now reported per combination, which both exclusion filters and positive trait filters select correctly. Each fault case is anchored to the variant class its lost tests would have run in, so the executor can supply the fixture argument from the class instead of failing to build the fixture and reporting that failure in place of the discovery error. Also: - keep a method whose argument set attribute cannot be read from costing its whole class its tests, and from emptying the trait pool the fault case is selected by - propagate the five assertion formatting options the overridden executor was dropping, pinned by a test that fails if xunit renames the environment variables - drop private-field reflection in DiscoveryErrorTestCase and FixtureArgumentSetTestMethod where a public member or the base constructor does the same job - stabilise SqlServer unit tests that shared ModelInfoProvider state Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MethodAttributeFaultTests.cs | 61 ++++ ...rtionFormattingEnvironmentVariableTests.cs | 53 +++ .../DiscoveryFaultDispositionTests.cs | 89 +++++ .../DiscoveryFaultTests.cs | 99 +++++- .../MethodAttributeFaultTests.cs | 83 +++++ .../RetryTestCaseNoResultTests.cs | 62 ++++ .../TestAssetRunner.cs | 14 +- .../CustomXunitTestFrameworkDiscoverer.cs | 334 +++++++++++++----- .../CustomXunitTestFrameworkExecutor.cs | 69 +++- .../DiscoveryErrorTestCase.cs | 39 +- .../FixtureArgumentSetTestMethod.cs | 47 ++- .../RetryTestCase.cs | 68 +++- .../Features/Search/CustomQueriesUnitTests.cs | 7 +- .../ScalarTemporalEqualityRewriterTests.cs | 1 + .../CompartmentQueryGeneratorTests.cs | 1 + .../SqlServerFhirDataStoreUnitTests.cs | 1 + .../ModelInfoProviderCollection.cs | 40 +++ 17 files changed, 904 insertions(+), 164 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/ModelInfoProviderCollection.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs new file mode 100644 index 0000000000..7f2e2702d4 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MethodAttributeFault +{ + /// + /// A class where reading one method's fixture argument set attribute is itself what fails, and the + /// methods around it are declared normally. + /// + /// + /// The failing method carries two different fixture argument set attributes, so asking it for the + /// single one it declares throws before any expansion is attempted. This is a different failure + /// from a method whose argument set expands to nothing: it happens while reading the attribute + /// rather than while using it, which is the step that used to run for every method of the class at + /// once and so cost the whole class its tests. The class takes no fixture, so nothing can fail + /// ahead of the reported case and the message it carries is the discoverer's own. One sibling + /// declares an argument set of its own, so the class still has values to pool into the fault + /// case's traits once the failing method's cannot be read. + /// + public class MethodAttributeFaultTests + { + /// + /// Runs normally, and is declared before the failing method. + /// + [Fact] + public void RunsBeforeTheFault() + { + Assert.True(true); + } + + /// + /// Never runs: it declares two fixture argument set attributes, so which one applies cannot be + /// determined. + /// + [Fact] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + + /// + /// Runs normally, and is declared after the failing method. It declares an argument set of its + /// own so that the class has argument set values that can still be read once the failing + /// method's cannot. + /// + [Fact] + [AssetArgumentSets(AssetDataStore.Cosmos)] + public void RunsAfterTheFault() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs new file mode 100644 index 0000000000..11d2002905 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using System.Reflection; +using Xunit; +using Xunit.Sdk; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins the environment variable names the custom executor copies xunit's assertion formatting + /// options into. + /// + /// + /// The executor substitutes its own assembly runner, so it does not go through the base + /// implementation that normally makes this copy and has to make it itself. xunit keeps the names + /// in an internal class that cannot be referenced, so the executor repeats them as literals. These + /// tests read that class back by reflection and compare, so a rename in a future xunit version + /// fails here rather than leaving options such as --print-max-object-depth accepted on the + /// command line and then quietly doing nothing. + /// + public class AssertionFormattingEnvironmentVariableTests + { + /// + /// Checks one option's environment variable name against the one xunit reads it from. + /// + /// The name of the field holding the variable name in xunit. + /// The literal the executor writes the option to. + [Theory] + [InlineData("AssertEquivalentMaxDepth", CustomXunitTestFrameworkExecutor.AssertEquivalentMaxDepthVariable)] + [InlineData("PrintMaxEnumerableLength", CustomXunitTestFrameworkExecutor.PrintMaxEnumerableLengthVariable)] + [InlineData("PrintMaxObjectDepth", CustomXunitTestFrameworkExecutor.PrintMaxObjectDepthVariable)] + [InlineData("PrintMaxObjectMemberCount", CustomXunitTestFrameworkExecutor.PrintMaxObjectMemberCountVariable)] + [InlineData("PrintMaxStringLength", CustomXunitTestFrameworkExecutor.PrintMaxStringLengthVariable)] + public void GivenAnAssertionFormattingOption_WhenItsNameIsComparedToXunits_ThenTheyMatch(string fieldName, string expectedVariableName) + { + Type environmentVariables = typeof(ITestFrameworkExecutionOptions).Assembly + .GetTypes() + .SingleOrDefault(type => type.FullName == "Xunit.Internal.EnvironmentVariables"); + + Assert.NotNull(environmentVariables); + + FieldInfo field = environmentVariables.GetField(fieldName, BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + + Assert.NotNull(field); + Assert.Equal(expectedVariableName, field.GetRawConstantValue()); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs new file mode 100644 index 0000000000..d6d5f1dfa3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs @@ -0,0 +1,89 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers which exceptions out of fixture argument set expansion are reported as failing test + /// cases and which go back to xunit unchanged. + /// + /// + /// Reporting a discovery fault as a test is what stops a broken expansion from leaving a run + /// green with its tests silently absent, and the scenario assets cover that. The exclusions + /// cannot be reached the same way: cancellation would have to arrive at the moment a class is + /// being expanded, which no scenario can arrange reliably. The decision is therefore made by one + /// function so that every case can be pinned directly, because getting an exclusion wrong is not + /// visible in a passing run - it invents a red test naming a class with nothing wrong with it. + /// + public class DiscoveryFaultDispositionTests + { + /// + /// A cancelled run is not a discovery fault. Without this, pressing Ctrl+C or hitting a + /// runner-imposed timeout would produce a failing test case for whichever class happened to + /// be mid-expansion, and a cancelled run would be indistinguishable from a broken one. + /// + [Fact] + public void GivenACancellation_WhenExpansionEnds_ThenItIsRethrownRatherThanReportedAsATest() + { + Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new OperationCanceledException(), + isCallbackFailure: false)); + } + + /// + /// derives from + /// and is what an awaited task throws when the run is cancelled, so it has to be treated the + /// same way. Matching on the exact type instead would let the common case through. + /// + [Fact] + public void GivenATaskCancellation_WhenExpansionEnds_ThenItIsRethrownRatherThanReportedAsATest() + { + Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new TaskCanceledException(), + isCallbackFailure: false)); + } + + /// + /// A failure raised by xunit's own callback is not a fault in expanding argument sets, and + /// the only way to report it would be to hand that same callback another test case. + /// + [Fact] + public void GivenAFailureFromTheCallback_WhenExpansionEnds_ThenItIsRethrownRatherThanReportedAsATest() + { + Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new InvalidOperationException("the sink threw"), + isCallbackFailure: true)); + } + + /// + /// Anything else belongs to the expansion, and is the case the whole mechanism exists for: + /// reported as a failing test case so it reaches the results and the exit code instead of + /// leaving the run green with the class's tests missing. + /// + [Fact] + public void GivenAnExpansionFailure_WhenExpansionEnds_ThenItIsReportedAsATest() + { + Assert.False(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new InvalidOperationException("the argument set was misdeclared"), + isCallbackFailure: false)); + } + + /// + /// Cancellation wins even when it also reached the callback, because there is nothing to + /// report a cancelled run against. + /// + [Fact] + public void GivenACancellationFromTheCallback_WhenExpansionEnds_ThenItIsRethrownRatherThanReportedAsATest() + { + Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new OperationCanceledException(), + isCallbackFailure: true)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs index 17a41e5f5f..49f4ae1bc0 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs @@ -15,7 +15,8 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests /// public class DiscoveryFaultTests { - private const string ErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery)"; + private const string SqlErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string CosmosErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Cosmos)"; /// /// xUnit reports an exception thrown out of discovery only as an internal diagnostic, which @@ -33,17 +34,17 @@ public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheFaultIsRep run, new Dictionary { - [ErrorCaseName] = "Failed", + [SqlErrorCaseName] = "Failed", + [CosmosErrorCaseName] = "Failed", }); Assert.NotEqual(0, run.ExitCode); } /// - /// The failing test case cannot be relied on to carry the cause, because the class fixture it - /// inherits fails to build first and that failure is what the report shows. The console line - /// the discoverer writes is the only place the original exception survives, so it is the part - /// worth pinning: without it a fault would be reported with no way to find out what it was. + /// The console line the discoverer writes is where the original exception survives in full, + /// including the stack that points at the line which threw. Without it a fault would be + /// reported with no way to find out what it was. /// [Fact] public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheCauseIsWrittenToTheOutput() @@ -54,13 +55,35 @@ public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheCauseIsWri Assert.Contains("IndexOutOfRangeException", run.Output, StringComparison.Ordinal); } + /// + /// The failure the runner reports has to carry the cause itself, not just the console. A + /// fault case stands in for the variants of a class that never got built, and those variants + /// are how the executor knows which argument to hand the class fixture; a case that does not + /// say which one it stands for leaves the executor unable to build the fixture, and the + /// report then shows that fixture failure instead of the discovery error behind it. Anyone + /// reading the results would be sent to look at a fixture that is not the problem. This pins + /// the reported message to the real exception so that cannot come back. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheReportedFailureCarriesTheOriginalException() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault"); + + foreach (string caseName in new[] { SqlErrorCaseName, CosmosErrorCaseName }) + { + string reported = ExtractReportedFailure(run.Output, caseName); + + Assert.Contains("IndexOutOfRangeException", reported, StringComparison.Ordinal); + Assert.DoesNotContain("had multiple values", reported, StringComparison.Ordinal); + } + } + /// /// Reporting the fault as a test is only worth anything if the legs that would have run the - /// class still select it. CI legs pick their tests with a positive trait filter, and the + /// class still select it. Some CI legs pick their tests with a positive trait filter, and the /// variants this class never produced are what would have carried those traits, so a fault /// case with no traits of its own is filtered out and the leg passes with the class silently /// missing - which is the failure this whole mechanism exists to prevent, only harder to see. - /// The case therefore declares every argument set value the class asked for. /// [Fact] public void GivenAClassThatCannotBeExpanded_WhenTestsAreSelectedByArgumentSetTrait_ThenTheFaultIsStillReported() @@ -71,10 +94,68 @@ public void GivenAClassThatCannotBeExpanded_WhenTestsAreSelectedByArgumentSetTra run, new Dictionary { - [ErrorCaseName] = "Failed", + [SqlErrorCaseName] = "Failed", }); Assert.NotEqual(0, run.ExitCode); } + + /// + /// The repository's integration legs select their tests by excluding the other data + /// store, and a trait exclusion drops a case when any of its values under that trait matches. + /// A single fault case declaring every value the class asked for would therefore be dropped by + /// the SQL leg for holding the Cosmos value and by the Cosmos leg for holding the SQL one, so + /// the fault would reach neither and both legs would stay green with the class missing. One + /// case per combination is what stops that: excluding one value leaves the others reported. + /// + [Fact] + public void GivenAClassThatCannotBeExpanded_WhenTestsAreSelectedByExcludingAnArgumentSetTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("DiscoveryFault", filterNotTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [CosmosErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// Reads back what the runner reported for one failed case, which is the block it prints + /// after the case's own failed line and before whatever it prints next. + /// + /// + /// The message has to be taken from the console rather than the TRX report, because the TRX + /// writer records no error information for these cases. Searching the whole output instead + /// would prove nothing: the discoverer's own diagnostic line names the same exception, so an + /// assertion over all of it would pass even with the reported failure saying something else + /// entirely. + /// + /// The full console output of the run. + /// The display name of the case whose failure is wanted. + /// The text the runner printed for that failure. + private static string ExtractReportedFailure(string output, string caseName) + { + const string failedMarker = "failed "; + + int start = output.IndexOf(failedMarker + caseName, StringComparison.Ordinal); + Assert.True(start >= 0, $"The run reported no failure for '{caseName}'.{Environment.NewLine}{output}"); + + start += failedMarker.Length + caseName.Length; + + int next = output.IndexOf(Environment.NewLine + failedMarker, start, StringComparison.Ordinal); + int summary = output.IndexOf("Test run summary", start, StringComparison.Ordinal); + + int end = next >= 0 ? next : output.Length; + if (summary >= 0 && summary < end) + { + end = summary; + } + + return output.Substring(start, end - start); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs new file mode 100644 index 0000000000..a2b799b347 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs @@ -0,0 +1,83 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a method whose fixture argument set attribute cannot even be read costs only + /// that method, rather than every method of its class. + /// + public class MethodAttributeFaultTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MethodAttributeFault.MethodAttributeFaultTests"; + private const string ErrorCaseName = ScenarioClass + ".NeverRuns (fixture argument set discovery: Cosmos)"; + + /// + /// Reading the attribute is a separate step from expanding it, and it can fail on its own: a + /// method carrying two different fixture argument set attributes has no single one to expand. + /// Reading every method's attribute up front, before the walk that isolates each method, put + /// that failure outside the isolation and cost the whole class its tests - reported as one + /// failure claiming the class never ran, with the healthy methods gone and no sign of it. + /// Reading each method's attribute inside the walk keeps the loss to the method at fault. + /// + [Fact] + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTheClassIsDiscovered_ThenTheOtherMethodsStillRun() + { + TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".RunsBeforeTheFault"] = "Passed", + [ScenarioClass + ".RunsAfterTheFault (Cosmos)"] = "Passed", + [ErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The values a fault case carries as traits are pooled from every method of the class, and + /// reading them can hit the very declaration that caused the fault. Reading the whole class in + /// one attempt meant that one unreadable method left the pool empty, so the failure went out + /// with no argument set traits at all - and a leg selecting its tests by those traits would + /// then skip the failure and pass with the class missing. Here only the failing method's + /// values are unreadable, so the case must still carry the value its sibling declares. + /// + [Fact] + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTestsAreSelectedByArgumentSetTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault", filterTrait: "AssetDataStore=Cosmos"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".RunsAfterTheFault (Cosmos)"] = "Passed", + [ErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failure has to name the method whose attribute could not be read, otherwise a class of + /// many methods reports a fault with no indication of which declaration to go and fix. + /// + [Fact] + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTheClassIsDiscovered_ThenTheFailureNamesOnlyThatMethod() + { + TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault"); + + Assert.Contains("MethodAttributeFaultTests.NeverRuns", run.Output, StringComparison.Ordinal); + Assert.Contains("Other methods of the class were discovered normally", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs index 26aa777026..034f28ec3f 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using Xunit; using Xunit.Sdk; using Xunit.v3; @@ -114,5 +115,66 @@ public void GivenAnEarlierAttemptIsReplayed_WhenTheSummaryIsBuilt_ThenTheTestCou Assert.Equal(1, summary.Total); Assert.Equal(1, summary.Failed); } + + /// + /// The exception attached to a replayed current-attempt failure has to be the one that + /// attempt captured. Attaching the earlier attempt's exception would report this attempt's + /// failure under the previous attempt's error text. + /// + [Fact] + public void GivenTheCurrentAttemptIsReplayed_WhenSelectingTheException_ThenTheCurrentAttemptsExceptionIsChosen() + { + var current = new InvalidOperationException("current"); + var earlier = new InvalidOperationException("earlier"); + + Exception selected = RetryTestCase.SelectNoResultException(NoResultOutcome.ReplayCurrentAttempt, current, earlier); + + Assert.Same(current, selected); + } + + /// + /// An attempt cut short before it captured anything of its own still has its failure + /// replayed, and nothing is attached rather than reaching back for a stale exception. + /// + [Fact] + public void GivenTheCurrentAttemptIsReplayedWithoutAnException_WhenSelectingTheException_ThenNothingIsChosen() + { + Exception selected = RetryTestCase.SelectNoResultException( + NoResultOutcome.ReplayCurrentAttempt, + currentAttemptException: null, + earlierAttemptException: new InvalidOperationException("earlier")); + + Assert.Null(selected); + } + + /// + /// When the earlier attempt's failure is the one replayed, its exception is the one that + /// describes it. + /// + [Fact] + public void GivenAnEarlierAttemptIsReplayed_WhenSelectingTheException_ThenTheEarlierAttemptsExceptionIsChosen() + { + var current = new InvalidOperationException("current"); + var earlier = new InvalidOperationException("earlier"); + + Exception selected = RetryTestCase.SelectNoResultException(NoResultOutcome.ReplayEarlierAttempt, current, earlier); + + Assert.Same(earlier, selected); + } + + /// + /// Nothing was published for the test, so nothing should be attached to it either. Adding an + /// exception here would turn a test that reported no result into a reported error. + /// + [Fact] + public void GivenNothingIsReplayed_WhenSelectingTheException_ThenNothingIsChosen() + { + Exception selected = RetryTestCase.SelectNoResultException( + NoResultOutcome.ReportNothing, + new InvalidOperationException("current"), + new InvalidOperationException("earlier")); + + Assert.Null(selected); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index 4a8a9f113b..5e8be45707 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -55,8 +55,14 @@ public static class TestAssetRunner /// The value to pass as the runner's thread limit, in the form it takes on the command line. /// Scenarios that observe how much runs at once set this so the expected bound is explicit. /// + /// + /// A Name=Value trait filter to exclude on, in the form the runner takes on the command + /// line. The repository's integration legs select their tests this way - each excludes the + /// other data store - so this is how a scenario checks that a leg excluding one value still + /// sees what it was meant to run. + /// /// The exit code, output and published results of the run. - public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null, string maxThreads = null) + public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null, string maxThreads = null, string filterNotTrait = null) { string assetsAssembly = ResolveAssetsAssembly(); string resultsDirectory = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", Guid.NewGuid().ToString("N")); @@ -103,6 +109,12 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool pr arguments.Add(filterTrait); } + if (!string.IsNullOrEmpty(filterNotTrait)) + { + arguments.Add("--filter-not-trait"); + arguments.Add(filterNotTrait); + } + if (!string.IsNullOrEmpty(maxThreads)) { arguments.Add("--max-threads"); diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 75886d814c..cf413b51d5 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -41,27 +41,20 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl { return await FindTestsForTypeCore(testClass, discoveryOptions, tracker); } - catch (OperationCanceledException) + catch (Exception ex) when (ShouldRethrowRatherThanReport(ex, tracker.IsCallbackFailure(ex))) { - // A cancelled run is not a discovery fault. Reporting it as a failing test case would - // turn pressing Ctrl+C, or a runner-imposed timeout, into a test failure. - throw; - } - catch (Exception ex) when (tracker.IsCallbackFailure(ex)) - { - // The caller's own sink threw. Describing that as a failure to expand this class would - // be wrong, and handing the caller a further test case is unlikely to fare any better, - // so it goes back to xunit to deal with as it would for any other class. + // Either the run was cancelled, or the caller's own sink threw. Neither is a failure + // to expand fixture argument sets, so neither is reported as one. throw; } catch (Exception ex) { - // An exception thrown out of discovery is reported by xunit.v3 only as an internal - // diagnostic message, which is suppressed unless the run was started with - // --xunit-internal-diagnostics on. The class is dropped and the run still reports - // success, so a broken expansion would look exactly like a class that has no tests. - // Reporting a test case that fails on execution instead puts the fault in the results - // and in the exit code, where it cannot be missed. + // An exception thrown out of discovery is reported by xunit.v3 only as a diagnostic + // message, which is suppressed unless the run was started with --xunit-diagnostics. + // The class is dropped and the run still reports success, so a broken expansion would + // look exactly like a class that has no tests. Reporting a test case that fails on + // execution instead puts the fault in the results and in the exit code, where it + // cannot be missed. // // Reaching here means the failure was outside the per-method loop, which reports its // own faults, so it belongs to the class as a whole and no method of it was expanded. @@ -89,6 +82,29 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl } } + /// + /// Decides whether an exception thrown while expanding fixture argument sets goes back to + /// xunit unchanged, or is reported as a failing test case standing in for the tests it lost. + /// + /// + /// Reporting a fault as a test is what stops a broken expansion from leaving a run green with + /// its tests silently absent, but it is only right for a failure that belongs to the expansion. + /// Two do not. A cancelled run is not a fault at all, and turning it into a test case would + /// make pressing Ctrl+C, or a runner-imposed timeout, produce a red test that names a class + /// with nothing wrong with it. A failure raised by xunit's own callback is not ours either, + /// and the only way to report it would be to hand that same callback another test case. + /// + /// This is a separate function so that the decision can be pinned directly. Reached only + /// through a catch clause, the cancellation case in particular needs a run to be cancelled at + /// the moment a class is being expanded, which no scenario can arrange reliably. + /// + /// + /// The exception that ended the expansion. + /// Whether the exception came from xunit's own callback. + /// true to rethrow; false to report the failure as a test case. + internal static bool ShouldRethrowRatherThanReport(Exception exception, bool isCallbackFailure) => + exception is OperationCanceledException || isCallbackFailure; + /// /// Determines whether xunit would treat a method as a test. /// @@ -116,8 +132,26 @@ private static bool IsTestMethod(MethodInfo method) => /// anchoring the case to a class of its own takes it out of the namespace and class filters /// that would have selected the tests it replaces, and discarding the aggregated failure /// discards this message with it, leaving the fault unreported again. + /// + /// One case is reported per combination of argument set values rather than one case carrying + /// them all, because a trait exclusion filter drops a case when any of its values + /// under the named trait matches. A single case declaring both DataStore=CosmosDb and + /// DataStore=SqlServer would be dropped by the SQL leg's + /// --filter-not-trait DataStore=CosmosDb and by the Cosmos leg's + /// --filter-not-trait DataStore=SqlServer alike, so the fault would be reported to + /// nobody while both legs stayed green. + /// + /// + /// A case standing for a combination is anchored to that combination's variant class, the same + /// class the tests it replaces would have run in. That is what lets the failure carry its own + /// message: the executor takes the fixture's argument from the variant class it is running, + /// and only falls back to reading it off the case's traits when there is no variant class to + /// ask - a fallback that cannot tell one combination from another and refuses to guess. With + /// the argument supplied the fixture builds, nothing is aggregated ahead of the case, and the + /// report shows the discovery failure rather than a fixture that could not be constructed. + /// /// - private static async ValueTask ReportDiscoveryFault( + private async ValueTask ReportDiscoveryFault( IXunitTestClass testClass, MethodInfo method, CallbackTracker tracker, @@ -126,28 +160,162 @@ private static async ValueTask ReportDiscoveryFault( { Console.WriteLine($"[FixtureArgumentSets] ERROR: {summary} It was replaced by a failing test case. {exception}"); - // The case stays on the class that failed, rather than on a class of its own, so that it - // is still selected by whatever namespace or class filter would have selected the tests it - // replaces, and it carries their traits so that a leg selecting by those cannot pass with - // the tests silently missing. - var errorTestMethod = new XunitTestMethod( - testClass, - method, - Array.Empty(), - uniqueID: UniqueIDGenerator.ForTestMethod(testClass.UniqueID, method.Name)); - - // The test case is handed to xunit through the callback, which owns it from then on - // and disposes it with the rest of the discovered cases. + foreach (IReadOnlyList combination in BuildFaultArgumentSetCombinations(ReadFaultArgumentSetValues(testClass))) + { + string displaySuffix = combination.Count == 0 + ? string.Empty + : $": {string.Join(", ", combination.Select(flag => flag.EnumValue.ToString()))}"; + string idSuffix = combination.Count == 0 + ? string.Empty + : $"-{string.Join("-", combination.Select(flag => $"{flag.EnumValue.GetType().Name}.{flag.EnumValue}"))}"; + + // The case stays on the class that failed - either the class itself, or the variant of + // it the lost tests would have run in - rather than on a class of its own, so that it + // is still selected by whatever namespace or class filter would have selected them, + // and it carries their traits so that a leg selecting by those cannot pass with the + // tests silently missing. + IXunitTestClass anchorClass = testClass; + if (combination.Count > 0) + { + SingleFlag[] closedVariant = combination.ToArray(); + string classKey = BuildVariantClassKey(BuildVariantKey(testClass.TestCollection, closedVariant), testClass.Class); + anchorClass = _variantClassCache.GetOrAdd( + classKey, + _ => new FixtureArgumentSetTestClass( + testClass.Class, + testClass.TestCollection, + closedVariant, + UniqueIDGenerator.ForTestClass(testClass.TestCollection.UniqueID, classKey))); + } + + var errorTestMethod = new XunitTestMethod( + anchorClass, + method, + Array.Empty(), + uniqueID: UniqueIDGenerator.ForTestMethod(anchorClass.UniqueID, method.Name)); + + // The test case is handed to xunit through the callback, which owns it from then on + // and disposes it with the rest of the discovered cases. #pragma warning disable CA2000 - var errorTestCase = new DiscoveryErrorTestCase( - errorTestMethod, - $"{testClass.TestClassName}.{method.Name} (fixture argument set discovery)", - $"{testClass.UniqueID}-{method.Name}-fixture-argument-set-discovery-error", - errorMessage: $"{summary} {exception}", - traits: BuildFaultTraits(testClass, errorTestMethod)); + var errorTestCase = new DiscoveryErrorTestCase( + errorTestMethod, + $"{testClass.TestClassName}.{method.Name} (fixture argument set discovery{displaySuffix})", + $"{testClass.UniqueID}-{method.Name}-fixture-argument-set-discovery-error{idSuffix}", + errorMessage: $"{summary} {exception}", + traits: BuildFaultTraits(errorTestMethod, combination)); #pragma warning restore CA2000 - return await tracker.Invoke(errorTestCase); + if (!await tracker.Invoke(errorTestCase)) + { + return false; + } + } + + return true; + } + + /// + /// Closes the argument set values a failed class declared over each other, one dimension per + /// enum type, so that each combination can be reported as its own failure. + /// + /// Every argument set value read off the class, in any order. + /// + /// One list per combination. A class declaring no argument set values yields a single empty + /// combination, so a fault is always reported at least once. + /// + /// + /// Values are grouped by enum type because that is what a dimension is: an attribute takes one + /// flags enum per fixture constructor parameter, and the variants a class expands to are the + /// product of the single flags set within each. Reproducing that product here means a leg + /// selecting a variant by trait selects the failure standing in for it, and a leg excluding a + /// variant by trait excludes only that one and still sees the rest. + /// + internal static IReadOnlyList> BuildFaultArgumentSetCombinations(IEnumerable argumentSetValues) + { + EnsureArg.IsNotNull(argumentSetValues, nameof(argumentSetValues)); + + IEnumerable> dimensions = argumentSetValues + .Where(flag => flag.EnumValue != null) + .GroupBy(flag => flag.EnumValue.GetType()) + .OrderBy(dimension => dimension.Key.Name, StringComparer.Ordinal) + .Select(dimension => (IEnumerable)dimension + .GroupBy(flag => flag.EnumValue) + .Select(values => values.First()) + .ToArray()) + .ToArray(); + + return CartesianProduct(dimensions) + .Select(combination => (IReadOnlyList)combination.ToArray()) + .ToArray(); + } + + /// + /// Reads every argument set value a class and its methods declare, for use in reporting a + /// discovery failure. + /// + /// + /// This runs while handling a discovery failure, and reads the same attributes whose contents + /// may well be what caused it, so each attribute is read and expanded on its own and any + /// further exception costs only that one attribute's values. Reporting the failure with fewer + /// traits is worth more than not reporting it at all. + /// + /// Every method's values are pooled, not just the anchor method's, because a class-level fault + /// loses every method. Pooling can only add combinations, and an extra combination reports the + /// fault once more rather than hiding it, whereas a missing one is a leg that passes with the + /// tests absent. + /// + /// + /// The class whose expansion failed. + /// The declared values; empty when none could be read. + private static SingleFlag[] ReadFaultArgumentSetValues(IXunitTestClass testClass) + { + var attributes = new List(); + + // Each attribute is read on its own so that the one whose contents may have caused the + // fault in the first place costs only its own values, rather than emptying the pool and + // leaving the failure with no traits for a leg's filter to select it by. + TryAdd(() => testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute); + + foreach (MethodInfo method in testClass.Methods) + { + MethodInfo closedMethod = method; + TryAdd(() => closedMethod.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute); + } + + return attributes + .SelectMany(SafeExpand) + .Where(flag => flag.EnumValue != null) + .ToArray(); + + void TryAdd(Func read) + { + try + { + if (read() is FixtureArgumentSetsAttribute attribute) + { + attributes.Add(attribute); + } + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: some argument set traits of '{testClass.TestClassName}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + } + } + + IEnumerable SafeExpand(FixtureArgumentSetsAttribute attribute) + { + try + { + return ExpandEnumFlagsFromAttributeData(attribute).SelectMany(dimension => dimension ?? Array.Empty()).ToArray(); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: some argument set traits of '{testClass.TestClassName}' could not be expanded, so a trait filter may not select the failure standing in for its tests. {ex}"); + return Array.Empty(); + } + } } /// @@ -159,19 +327,13 @@ private static async ValueTask ReportDiscoveryFault( /// anchoring method itself, and CI legs combine them with argument set values in filters such /// as (DataStore=CosmosDb)&(Category=ExportLongRunning), so leaving either out lets /// the leg that would have run these tests pass without them. The argument set values are the - /// ones the expansion never got far enough to produce, so they have to be read back from the - /// attributes here. - /// - /// This runs while handling a discovery failure, and reads the same attribute whose contents - /// may well be what caused it, so it treats any further exception as simply having no argument - /// set traits to offer. Reporting the failure with fewer traits is worth more than not - /// reporting it at all. - /// + /// ones the expansion never got far enough to produce, so they are supplied by the caller from + /// the attributes rather than read off any discovered variant. /// - /// The class whose expansion failed. /// The method the failure is reported against. - /// The traits, keyed by trait name; empty when none could be read. - private static Dictionary> BuildFaultTraits(IXunitTestClass testClass, XunitTestMethod anchor) + /// The one combination of argument set values this failure stands for. + /// The traits, keyed by trait name. + private static Dictionary> BuildFaultTraits(XunitTestMethod anchor, IReadOnlyList argumentSetCombination) { var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); @@ -194,50 +356,52 @@ void Add(string key, string value) } } - try + foreach (SingleFlag flag in argumentSetCombination) { - var attributes = new List(); - - if (testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() is FixtureArgumentSetsAttribute classAttribute) - { - attributes.Add(classAttribute); - } + Add(flag.EnumValue.GetType().Name, flag.EnumValue.ToString()); + } - foreach (MethodInfo method in testClass.Methods) - { - if (method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() is FixtureArgumentSetsAttribute methodAttribute) - { - attributes.Add(methodAttribute); - } - } + return traits; + } - foreach (SingleFlag flag in attributes.SelectMany(ExpandEnumFlagsFromAttributeData).SelectMany(dimension => dimension ?? Array.Empty())) - { - if (flag.EnumValue == null) - { - continue; - } + /// + /// Reads the fixture argument set attribute a method declares. + /// + /// The method to inspect. + /// The attribute, or null when the method declares none. + private static FixtureArgumentSetsAttribute GetMethodArgumentSetsAttribute(MethodInfo method) => + method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; - Add(flag.EnumValue.GetType().Name, flag.EnumValue.ToString()); - } + /// + /// Decides whether a class has to be expanded method by method rather than handed to xunit + /// unchanged. + /// + /// The method to inspect. + /// true when the method declares argument sets, or when reading them failed. + /// + /// A method whose attribute cannot even be read answers true so that the class takes the + /// expansion path, where the same read is repeated inside the per-method loop and its failure + /// is reported against that one method. Swallowing the exception here rather than there is what + /// keeps one misdeclared method from costing the whole class its tests; it is not lost, only + /// deferred to the place that can attribute it. + /// + private static bool RequiresPerMethodExpansion(MethodInfo method) + { + try + { + return GetMethodArgumentSetsAttribute(method) != null; } - catch (Exception ex) + catch (Exception) { - Console.WriteLine( - $"[FixtureArgumentSets] WARNING: the argument set traits of '{testClass.TestClassName}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + return true; } - - return traits; } private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, ITestFrameworkDiscoveryOptions discoveryOptions, CallbackTracker tracker) { var attribute = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute; - var methodAttributes = testClass.Methods.ToDictionary( - method => method, - method => method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute); - if (attribute == null && methodAttributes.Values.All(value => value == null)) + if (attribute == null && !testClass.Methods.Any(RequiresPerMethodExpansion)) { return await base.FindTestsForType(testClass, discoveryOptions, tracker.Invoke); } @@ -259,17 +423,15 @@ private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, IT // Each method is expanded on its own so that one method's misdeclared argument set // costs only that method. Letting the failure out of the loop would leave the methods // already published running while the methods after it disappeared, under a single - // failure claiming the whole class had not run. + // failure claiming the whole class had not run. The method's own attribute is read + // inside the try for the same reason: a method carrying two of them, or one whose + // constructor throws, is that method's fault and not the class's. bool continueDiscovery; try { - continueDiscovery = await FindTestsForMethodCore(testClass, method, attribute, methodAttributes[method], classLevelOpenParameterSets, classLevelClosedParameterSets, discoveryOptions, tracker); - } - catch (OperationCanceledException) - { - throw; + continueDiscovery = await FindTestsForMethodCore(testClass, method, attribute, GetMethodArgumentSetsAttribute(method), classLevelOpenParameterSets, classLevelClosedParameterSets, discoveryOptions, tracker); } - catch (Exception ex) when (!tracker.IsCallbackFailure(ex)) + catch (Exception ex) when (!ShouldRethrowRatherThanReport(ex, tracker.IsCallbackFailure(ex))) { continueDiscovery = await ReportDiscoveryFault( testClass, @@ -371,7 +533,7 @@ private async ValueTask FindTestsForMethodCore( var closedVariantTestMethod = new FixtureArgumentSetTestMethod(closedVariantTestClass, method, closedVariant, uniqueId: UniqueIDGenerator.ForTestMethod(closedVariantTestClass.UniqueID, method.Name)); - closedVariantTestMethod.UpdateArgumentsFromMethod(); + closedVariantTestMethod.ApplyArgumentSetTraits(); // xUnit builds the display name from the class and method names only, so every // variant of a method would otherwise be reported under an identical name and diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index 0c3d6160d8..d4d6298f1e 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -1,14 +1,16 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; +using EnsureThat; using Xunit; using Xunit.Sdk; using Xunit.v3; @@ -20,17 +22,71 @@ namespace Microsoft.Health.Extensions.Xunit /// internal sealed class CustomXunitTestFrameworkExecutor : XunitTestFrameworkExecutor { + /// + /// The environment variable Assert reads --assert-equivalent-max-depth from. + /// + /// + /// xunit keeps these names in an internal class, so they are repeated here rather than + /// referenced. AssertionFormattingEnvironmentVariableTests compares them back against + /// that class, so a rename in a future xunit version fails a test rather than turning these + /// options silently back off. + /// + internal const string AssertEquivalentMaxDepthVariable = "XUNIT_ASSERT_EQUIVALENT_MAX_DEPTH"; + + /// The environment variable Assert reads --print-max-enumerable-length from. + internal const string PrintMaxEnumerableLengthVariable = "XUNIT_PRINT_MAX_ENUMERABLE_LENGTH"; + + /// The environment variable Assert reads --print-max-object-depth from. + internal const string PrintMaxObjectDepthVariable = "XUNIT_PRINT_MAX_OBJECT_DEPTH"; + + /// The environment variable Assert reads --print-max-object-member-count from. + internal const string PrintMaxObjectMemberCountVariable = "XUNIT_PRINT_MAX_OBJECT_MEMBER_COUNT"; + + /// The environment variable Assert reads --print-max-string-length from. + internal const string PrintMaxStringLengthVariable = "XUNIT_PRINT_MAX_STRING_LENGTH"; + public CustomXunitTestFrameworkExecutor(Assembly assembly) : base(new XunitTestAssembly(assembly, configFileName: null, assembly.GetName().Version, UniqueIDGenerator.ForAssembly(assembly.Location, null))) { } + /// + /// Runs the discovered test cases through the fixture argument set assembly runner. + /// + /// The test cases to run. + /// The sink execution messages are reported to. + /// The options the run was started with. + /// Cancels the run. + /// A task that completes when the run does. + /// + /// Substituting the assembly runner means not going through the base implementation, which is + /// also where xunit copies the assertion formatting options into the environment that + /// Assert reads them from. Skipping that step would leave options such as + /// --print-max-object-depth accepted on the command line and then silently ignored, so + /// the same copy is made here before the run starts. + /// public override async ValueTask RunTestCases(IReadOnlyCollection testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions, CancellationToken cancellationToken) { + EnsureArg.IsNotNull(executionOptions, nameof(executionOptions)); + + SetEnvironment(AssertEquivalentMaxDepthVariable, executionOptions.AssertEquivalentMaxDepth()); + SetEnvironment(PrintMaxEnumerableLengthVariable, executionOptions.PrintMaxEnumerableLength()); + SetEnvironment(PrintMaxObjectDepthVariable, executionOptions.PrintMaxObjectDepth()); + SetEnvironment(PrintMaxObjectMemberCountVariable, executionOptions.PrintMaxObjectMemberCount()); + SetEnvironment(PrintMaxStringLengthVariable, executionOptions.PrintMaxStringLength()); + var runner = new FixtureArgumentSetAssemblyRunner(); await runner.Run(TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); } + private static void SetEnvironment(string environmentVariableName, int? value) + { + if (value.HasValue) + { + Environment.SetEnvironmentVariable(environmentVariableName, value.Value.ToString(CultureInfo.InvariantCulture)); + } + } + /// /// The assembly runner context, extended to bound how many collections run at once. /// @@ -327,15 +383,8 @@ private static Dictionary> GetTraits(IReadOn foreach (var testCase in testCases) { - if (testCase is XunitTestCase xunitTestCase) - { - MergeTraits(traits, xunitTestCase.Traits.ToDictionary( - kvp => kvp.Key, - kvp => (IReadOnlyCollection)kvp.Value.ToArray(), - StringComparer.OrdinalIgnoreCase)); - continue; - } - + // Read through the metadata interface, which XunitTestCase implements as a + // read-only projection of the same dictionary its own Traits property exposes. MergeTraits(traits, ((ITestCaseMetadata)testCase).Traits); } diff --git a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs index a3b08137d9..e413805a4c 100644 --- a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; -using System.Reflection; using EnsureThat; using Xunit.v3; @@ -21,17 +20,18 @@ namespace Microsoft.Health.Extensions.Xunit /// reason this type exists: takes no traits and so would be /// dropped by a positive trait filter, which puts the run back to passing with the broken class /// silently missing - the exact outcome reporting the fault as a test was meant to prevent. - /// Declaring every value the class asked for means any filter that would have selected any of its - /// variants also selects the failure. + /// + /// One case carries one combination of argument set values, never all of them at once. A trait + /// filter matches a case when any value under the named trait matches, and an exclusion + /// filter is the negation of that, so a single case declaring both DataStore=CosmosDb and + /// DataStore=SqlServer would be excluded by both --filter-not-trait + /// DataStore=SqlServer and --filter-not-trait DataStore=CosmosDb - that is, by every + /// leg this repository runs. The discoverer therefore reports one of these per combination, so + /// each leg selects exactly the case that stands for the tests it lost. + /// /// internal sealed class DiscoveryErrorTestCase : ExecutionErrorTestCase { - private static readonly FieldInfo TraitsField = typeof(XunitTestCase) - .GetField("traits", BindingFlags.Instance | BindingFlags.NonPublic) - ?? throw new InvalidOperationException( - "XunitTestCase.traits was not found, so a discovery failure cannot be given the traits of the tests it " + - "replaces and would be hidden by a trait filter. This usually means the xunit.v3 version changed."); - public DiscoveryErrorTestCase( IXunitTestMethod testMethod, string testCaseDisplayName, @@ -42,26 +42,15 @@ public DiscoveryErrorTestCase( { EnsureArg.IsNotNull(traits, nameof(traits)); - if (TraitsField.GetValue(this) is not Dictionary> existing) - { - // The field is still there but no longer holds what it did, so the traits cannot be - // applied. Reporting the case without them is still worth more than not reporting it: - // an unfiltered run keeps the failure, and only a run selecting by trait loses it. - // Throwing instead would be worse - xunit reports an exception out of discovery only - // as a suppressed internal diagnostic and drops the class, which is silence again. - Console.WriteLine( - $"[FixtureArgumentSets] ERROR: XunitTestCase.traits holds '{TraitsField.GetValue(this)?.GetType().FullName ?? "null"}' rather than " + - $"Dictionary>, so the failure standing in for '{testCaseDisplayName}' carries no traits and a run " + - $"selecting tests by trait will not see it. This usually means the xunit.v3 version changed."); - return; - } - + // XunitTestCase.Traits is the live dictionary the case is filtered on, and the base + // constructor has already created it empty. Adding to it here is what gives the failure + // the traits of the tests it replaces. foreach (KeyValuePair> trait in traits) { - if (!existing.TryGetValue(trait.Key, out HashSet values)) + if (!Traits.TryGetValue(trait.Key, out HashSet values)) { values = new HashSet(StringComparer.OrdinalIgnoreCase); - existing[trait.Key] = values; + Traits[trait.Key] = values; } values.UnionWith(trait.Value); diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index f0ad9aa762..d80667e0ac 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -14,30 +14,18 @@ namespace Microsoft.Health.Extensions.Xunit { internal sealed class FixtureArgumentSetTestMethod : XunitTestMethod { - private static readonly FieldInfo TestMethodArgumentsField = GetRequiredField("testMethodArguments"); private static readonly FieldInfo TraitsField = GetRequiredField("traits"); - private static readonly FieldInfo UniqueIdField = GetRequiredField("uniqueID"); - private static readonly FieldInfo MethodField = GetRequiredField("method"); - private readonly FixtureArgumentSetTestClass _testClass; - private readonly MethodInfo _methodInfo; private readonly IReadOnlyList _fixtureArguments; - private readonly string _uniqueId; public FixtureArgumentSetTestMethod(FixtureArgumentSetTestClass testClass, MethodInfo methodInfo, IReadOnlyList fixtureArguments, string uniqueId) - : base(testClass, methodInfo, testMethodArguments: Array.Empty(), uniqueId) + : base(testClass, methodInfo, testMethodArguments: BuildMethodArguments(fixtureArguments), uniqueId) { EnsureArg.IsNotNull(testClass, nameof(testClass)); EnsureArg.IsNotNull(methodInfo, nameof(methodInfo)); EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - _testClass = testClass; - _methodInfo = methodInfo; _fixtureArguments = fixtureArguments; - _uniqueId = uniqueId; - - UniqueIdField.SetValue(this, _uniqueId); - MethodField.SetValue(this, _methodInfo); } #pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes @@ -66,37 +54,42 @@ private static FieldInfo GetRequiredField(string name) "which would silently change which tests the CI filters select. This usually means the xunit.v3 version changed."); } - private object[] CombineFixtureAndMethodArguments(object[] methodArguments) + /// + /// Builds the arguments the base type is constructed with, which are the fixture's argument + /// set values. + /// + /// + /// These are supplied through the base constructor rather than written back over it + /// afterwards, so that the private field holding them is one less thing this type has to + /// reach into and one less way a change to xunit can go unnoticed. + /// + /// The argument set values this variant stands for. + /// The values as an argument array. + private static object[] BuildMethodArguments(IReadOnlyList fixtureArguments) { - if (_fixtureArguments.Count == 0) - { - return methodArguments; - } + EnsureArg.IsNotNull(fixtureArguments, nameof(fixtureArguments)); - var result = new object[_fixtureArguments.Count + methodArguments.Length]; - for (int i = 0; i < _fixtureArguments.Count; i++) + if (fixtureArguments.Count == 0) { - result[i] = _fixtureArguments[i].EnumValue; + return Array.Empty(); } - if (methodArguments.Length > 0) + var result = new object[fixtureArguments.Count]; + for (int i = 0; i < fixtureArguments.Count; i++) { - Array.Copy(methodArguments, 0, result, _fixtureArguments.Count, methodArguments.Length); + result[i] = fixtureArguments[i].EnumValue; } return result; } - internal void UpdateArgumentsFromMethod() + internal void ApplyArgumentSetTraits() { if (_fixtureArguments.Count == 0) { return; } - var combinedArguments = CombineFixtureAndMethodArguments(Array.Empty()); - TestMethodArgumentsField.SetValue(this, combinedArguments); - var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); #pragma warning disable SA1100 // Do not prefix calls with base unless local implementation exists foreach (var kvp in base.Traits) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index b4a01ad8f6..eaf22e491e 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -183,9 +183,14 @@ public async ValueTask Run( StopRunIfRequested(continueRunning, cancellationTokenSource); - if (lastException != null) + Exception noResultException = SelectNoResultException( + outcome, + currentAttemptException: attemptAggregator.ToException(), + earlierAttemptException: lastException); + + if (noResultException != null) { - aggregator.Add(lastException); + aggregator.Add(noResultException); } RunSummary noResultSummary = CreateNoResultSummary(outcome); @@ -339,9 +344,24 @@ public async ValueTask Run( { Console.WriteLine($"[RetryFact] Unexpected exception while running '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}': {ex.GetType().FullName}: {ex.Message}"); - // Disposing replays anything still deferred, so a crash cannot swallow a - // failure that an earlier attempt already produced. - pendingFailureBus?.Dispose(); + // A crash must not swallow a failure an earlier attempt already produced, but at + // most one attempt's failure may reach the bus: replaying both would publish two + // results for a single test. That is the same supersession rule the no-result + // path applies, so it is decided the same way here. Disposing replays whatever + // is still deferred, and the finally below does that for the current attempt. + NoResultOutcome disposition = DecideNoResultOutcome( + currentAttemptObservedFailure: interceptingBus?.HasDeferredFailure ?? false, + earlierAttemptObservedFailure: pendingFailureBus?.HasDeferredFailure ?? false); + + if (disposition == NoResultOutcome.ReplayEarlierAttempt) + { + pendingFailureBus?.Dispose(); + } + else + { + pendingFailureBus?.DiscardDeferredMessages(); + } + throw; } finally @@ -380,6 +400,12 @@ private static void StopRunIfRequested(bool continueRunning, CancellationTokenSo /// Whether this attempt saw a failure before it was cut short. /// Whether an earlier attempt deferred a failure that has not been superseded. /// Which deferred failure, if any, should reach the results. + /// + /// This rule governs both places an attempt can end without reporting: the empty-summary + /// path, and the crash path where an unexpected exception unwinds the attempt. Both have to + /// let at most one attempt's failure reach the bus, because replaying two would publish two + /// results for a single test, so both decide it here rather than each reimplementing it. + /// internal static NoResultOutcome DecideNoResultOutcome(bool currentAttemptObservedFailure, bool earlierAttemptObservedFailure) { if (currentAttemptObservedFailure) @@ -392,6 +418,38 @@ internal static NoResultOutcome DecideNoResultOutcome(bool currentAttemptObserve : NoResultOutcome.ReportNothing; } + /// + /// Chooses the exception that belongs with the failure being reported for an attempt that + /// published no result of its own. + /// + /// The outcome chosen by . + /// The exception this attempt captured, if any. + /// The exception captured by the most recent attempt that failed, if any. + /// The exception to attach to the reported failure, or null when nothing should be attached. + /// + /// The exception has to describe the same failure that was replayed to the bus, otherwise + /// the run reports one attempt's failure alongside a different attempt's error text. So the + /// choice mirrors the bus exactly: the current attempt's failure carries the current + /// attempt's exception, an earlier attempt's failure carries the earlier one's, and an + /// attempt that reported nothing at all attaches nothing. Attaching the earlier exception to + /// a current-attempt failure is the specific mistake this guards against, because + /// lastException still holds the previous attempt's error at that point. + /// + internal static Exception SelectNoResultException(NoResultOutcome outcome, Exception currentAttemptException, Exception earlierAttemptException) + { + switch (outcome) + { + case NoResultOutcome.ReplayCurrentAttempt: + return currentAttemptException; + + case NoResultOutcome.ReplayEarlierAttempt: + return earlierAttemptException; + + default: + return null; + } + } + /// /// Builds the summary for an attempt that published no result of its own. /// diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs index 41a2590fc3..6819dd96b2 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/CustomQueriesUnitTests.cs @@ -22,9 +22,14 @@ public class CustomQueriesUnitTests [Fact] public async Task GivenCustomQueryClass_WithGivenWaitTime_QueryToDBWillOccurOnlyAfterWaitPeriod() { - // set wait time to 2 seconds; + // set wait time to 1 second CustomQueries.WaitTime = 1; + // CheckQueryHash only reaches the database once the wait period has elapsed since the + // last query, and that timestamp is static state shared with the sibling test. Waiting + // it out here means the first call below always queries, whichever test ran first. + await Task.Delay(1100); + var connection = Substitute.For(); var command = Substitute.For(); var reader = Substitute.For(); diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs index 2f85f02419..adc94404ac 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs @@ -20,6 +20,7 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Search)] + [Collection(ModelInfoProviderCollection.Name)] public class ScalarTemporalEqualityRewriterTests : IClassFixture { private static readonly DateTimeOffset StartOfDay = new DateTimeOffset(2016, 7, 6, 0, 0, 0, TimeSpan.Zero); diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/QueryGenerators/CompartmentQueryGeneratorTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/QueryGenerators/CompartmentQueryGeneratorTests.cs index 2826d89509..3d39b5aa44 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/QueryGenerators/CompartmentQueryGeneratorTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/QueryGenerators/CompartmentQueryGeneratorTests.cs @@ -28,6 +28,7 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions. /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Search)] + [Collection(ModelInfoProviderCollection.Name)] public class CompartmentQueryGeneratorTests : IClassFixture { private readonly ISqlServerFhirModel _model; diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs index b96f069f66..27b38fbcf0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs @@ -46,6 +46,7 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Storage { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.DataSourceValidation)] + [Collection(ModelInfoProviderCollection.Name)] public class SqlServerFhirDataStoreUnitTests { public static IEnumerable RemoveTrailingZerosTestCases() diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/ModelInfoProviderCollection.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/ModelInfoProviderCollection.cs new file mode 100644 index 0000000000..3af5cbb53c --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/ModelInfoProviderCollection.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests +{ + /// + /// Serializes the test classes that read or write the process-global ModelInfoProvider. + /// + /// + /// ModelInfoProvider is a plain static, so a class that replaces it races any class that + /// is relying on it. Concretely, SqlServerFhirDataStoreUnitTests installs a provider that + /// knows only the Group resource type and no compartments, while + /// CompartmentQueryGeneratorTests and ScalarTemporalEqualityRewriterTests need the + /// compartment-aware provider their class fixture installs. When those run concurrently the + /// compartment tests fail with "Expected an expression that evaluates to true", and which tests + /// fail varies from run to run. + /// + /// A class fixture cannot fix this, because it only orders the classes that share it and does + /// nothing about a third class overwriting the same static. Sharing one collection is what + /// actually serializes them: xUnit runs the classes within a collection one after another. + /// DisableParallelization additionally keeps the collection from overlapping other + /// collections, so a future class that touches the provider cannot reintroduce the race without + /// also being added here. + /// + /// The race is latent rather than new, but xUnit v3 schedules these classes concurrently where + /// v2 did not, so the migration is what makes it reproduce. + /// + [CollectionDefinition(Name, DisableParallelization = true)] + public sealed class ModelInfoProviderCollection + { + /// + /// The collection name to put on every test class that reads or writes the global provider. + /// + public const string Name = "ModelInfoProviderTests"; + } +} From 0d6769cfa0856d4c91595cf1f97e13ecf9b420f4 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 12:57:51 -0700 Subject: [PATCH 050/105] Report a discovery fault once per lost test, not once per class A failure standing in for tests that never ran is only useful if the CI leg that would have run them sees it. Merging several lost methods into one failure broke that in both directions: pooling their traits let an exclusion filter drop the failure for methods that never declared the excluded value, and using only one method's traits hid it from a leg selecting by another's. A filter takes or leaves a case whole, so the only safe unit is one failure per lost method per argument set combination it would have run under. Reading the argument sets was merged in the same way, so a method declaring two attributes lost both and inherited a sibling's - reporting a broken SQL test to the Cosmos leg alone. Each attribute is now read and expanded on its own, and only ever from the method the failure stands for. Adds scenarios for a class-level fault, overloads that both fault, and the class/method dimension merge on its success path, which was only ever entered to throw. Also pins which assertion formatting option reaches which environment variable, since all five are ints and a swap is invisible to a test that only compares names. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../ArgumentSetOverrideTests.cs | 86 +++++ .../ArgumentSetOverride/OverrideFixture.cs | 38 ++ .../ClassAttributeFaultTests.cs | 49 +++ .../OverloadedFault/OverloadedFaultTests.cs | 60 ++++ .../ArgumentSetOverrideTests.cs | 70 ++++ ...rtionFormattingEnvironmentVariableTests.cs | 76 ++++ .../ClassAttributeFaultTests.cs | 151 ++++++++ .../DiscoveryFaultTests.cs | 11 +- .../FaultArgumentSetCombinationTests.cs | 138 ++++++++ .../MethodAttributeFaultTests.cs | 45 ++- .../TestAssetRunner.cs | 34 +- .../CustomXunitTestFrameworkDiscoverer.cs | 324 +++++++++++++----- .../CustomXunitTestFrameworkExecutor.cs | 36 +- .../FixtureArgumentSetTestMethod.cs | 5 + 14 files changed, 1009 insertions(+), 114 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/OverrideFixture.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/OverloadedFault/OverloadedFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultArgumentSetCombinationTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs new file mode 100644 index 0000000000..607fe26cea --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs @@ -0,0 +1,86 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ArgumentSetOverride +{ + /// + /// A class whose methods narrow one dimension of the class-level fixture argument set while + /// inheriting the other. A method-level declaration replaces the class-level values dimension by + /// dimension rather than wholesale: a dimension naming at least one flag overrides the class's, + /// and a dimension naming none falls back to it. Both halves of that merge decide which variants + /// of a test exist at all, so a mistake in either silently drops tests from a run that still + /// reports success. + /// + [TwoDimensionArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos, AssetOtherDimension.Some)] + public class ArgumentSetOverrideTests : IClassFixture + { + private readonly OverrideFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public ArgumentSetOverrideTests(OverrideFixture fixture) + { + _fixture = fixture; + } + + /// + /// Declaring nothing of its own leaves this method with the class's values in both + /// dimensions. + /// + [Fact] + public void InheritsBothDimensions() + { + AssertFixtureMatchesDisplayName(); + } + + /// + /// The first dimension names a flag, so it replaces the class's two values with one; the + /// second names none, so it keeps the class's. Reading the method's declaration as a whole + /// rather than dimension by dimension would leave this method with no second dimension and + /// so no variants at all. + /// + [Fact] + [TwoDimensionArgumentSets(AssetDataStore.Cosmos, AssetOtherDimension.None)] + public void OverridesTheFirstDimensionOnly() + { + AssertFixtureMatchesDisplayName(); + } + + /// + /// The mirror image: the first dimension names no flag and falls back to the class's two + /// values, while the second names one of its own. The fallback has to work at any position, + /// not only the last. + /// + [Fact] + [TwoDimensionArgumentSets((AssetDataStore)0, AssetOtherDimension.Some)] + public void OverridesTheSecondDimensionOnly() + { + AssertFixtureMatchesDisplayName(); + } + + /// + /// Ties the values the fixture was built from to the name the variant reports under. + /// Asserting only that the fixture holds some valid value would hold for every variant + /// alike, so a merge that produced the right number of variants from the wrong values - + /// or built every fixture from the same one - would pass unnoticed. + /// + private void AssertFixtureMatchesDisplayName() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.EndsWith( + $" ({_fixture.DataStore}, {_fixture.OtherDimension})", + displayName, + StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/OverrideFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/OverrideFixture.cs new file mode 100644 index 0000000000..3bca4fff1f --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/OverrideFixture.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ArgumentSetOverride +{ + /// + /// A class fixture built from both fixture argument set dimensions, so a test can tell which + /// value of each dimension it actually received rather than only which name it ran under. + /// + public class OverrideFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The value of the first dimension this variant was constructed for. + /// The value of the second dimension this variant was constructed for. + public OverrideFixture(AssetDataStore dataStore, AssetOtherDimension otherDimension) + { + DataStore = dataStore; + OtherDimension = otherDimension; + } + + /// + /// Gets the value of the first dimension this variant was constructed for. + /// + public AssetDataStore DataStore { get; } + + /// + /// Gets the value of the second dimension this variant was constructed for. + /// + public AssetOtherDimension OtherDimension { get; } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs new file mode 100644 index 0000000000..3e00443a86 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ClassAttributeFault +{ + /// + /// A class whose own fixture argument set declaration cannot be read, so the fault belongs to the + /// class rather than to any one of its methods. + /// + /// + /// The class carries two different fixture argument set attributes, so asking it for the single + /// one it declares throws before the walk over its methods begins. That is what makes this a + /// class-level fault: every method is lost at once, and the one failure reported has to stand in + /// for all of them. The methods declare different traits so that a failure carrying only the + /// first method's traits can be told apart from one carrying every method's. + /// + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public class ClassAttributeFaultTests + { + /// + /// Never runs, and is declared first, so it is the method the failure is anchored to. + /// + [Fact] + [Trait("Category", "DeclaredFirst")] + public void FirstNeverRuns() + { + Assert.True(true); + } + + /// + /// Never runs, and declares a trait no earlier method does, together with an argument set of + /// its own, so a leg selecting on both at once only sees the failure if it carries both. + /// + [Fact] + [Trait("Category", "DeclaredLast")] + [AssetArgumentSets(AssetDataStore.Cosmos)] + public void SecondNeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/OverloadedFault/OverloadedFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/OverloadedFault/OverloadedFaultTests.cs new file mode 100644 index 0000000000..3226110ed6 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/OverloadedFault/OverloadedFaultTests.cs @@ -0,0 +1,60 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.OverloadedFault +{ + /// + /// A class where two overloads of the same test method both fail discovery, so two failures have + /// to be reported against methods that share a name. + /// + /// + /// Each overload carries two different fixture argument set attributes, so asking either for the + /// single one it declares throws and each is reported on its own. Overloads are what make this + /// scenario worth having: anything identifying the reported failure by method name alone gives + /// both failures the same identity, and one of them is then dropped - the method it stood for + /// silently absent from a run that still reports the other. + /// + /// xunit's own analyzer rejects overloaded test method names, which is why no such class exists + /// in this repository and why this is defence rather than a live bug. The analyzer is suppressed + /// here rather than the scenario dropped, because the rule is a warning a consuming project can + /// turn off, and a fault case that loses a method is not something the reader of a green run can + /// see. + /// + /// +#pragma warning disable xUnit1024 + public class OverloadedFaultTests + { + /// + /// Never runs: it declares two fixture argument set attributes. + /// + /// Unused; present only to give this overload a distinct signature. + [Theory] + [InlineData(1)] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns(int value) + { + Assert.True(value > 0); + } + + /// + /// Never runs either, and shares its name with the overload above. + /// + /// Unused; present only to give this overload a distinct signature. + [Theory] + [InlineData("x")] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns(string value) + { + Assert.NotNull(value); + } + } +#pragma warning restore xUnit1024 +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs new file mode 100644 index 0000000000..635817fdd2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs @@ -0,0 +1,70 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers the merge of a method-level fixture argument set declaration with its class's. The + /// merge runs dimension by dimension - a dimension naming at least one flag overrides the + /// class's, one naming none inherits it - and it decides which variants of a test exist at all. + /// Getting it wrong drops tests from a run that still reports success, which is the one failure + /// mode this whole mechanism exists to prevent. + /// + public class ArgumentSetOverrideTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ArgumentSetOverride.ArgumentSetOverrideTests"; + + /// + /// The whole merge in one run: a method declaring nothing keeps both of the class's + /// dimensions, and each method narrowing one dimension keeps the class's values in the other. + /// Asserting the exact set is what makes a variant that quietly stopped being discovered + /// visible - a count alone would still pass if one variant were replaced by another. + /// + [Fact] + public void GivenMethodsNarrowingOneDimension_WhenTheyAreDiscovered_ThenEachInheritsTheClassValuesForTheOther() + { + TestAssetRun run = TestAssetRunner.Run("ArgumentSetOverride"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".InheritsBothDimensions (Sql, Some)"] = "Passed", + [ScenarioClass + ".InheritsBothDimensions (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheFirstDimensionOnly (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Sql, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Cosmos, Some)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + + /// + /// Each variant asserts that the fixture it was handed matches the name it runs under, so a + /// passing run is also evidence the merged values reached the fixture rather than only the + /// display name. Selecting by one dimension's value shows the pairing from the other side: + /// only the variants built for that value may answer, including the ones that inherited it + /// rather than declaring it. + /// + [Fact] + public void GivenMergedArgumentSets_WhenTestsAreSelectedByOneDimension_ThenOnlyTheVariantsBuiltForItRun() + { + TestAssetRun run = TestAssetRunner.Run("ArgumentSetOverride", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".InheritsBothDimensions (Sql, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Sql, Some)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs index 11d2002905..9b2417e5c4 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Linq; using System.Reflection; using Xunit; @@ -49,5 +50,80 @@ public void GivenAnAssertionFormattingOption_WhenItsNameIsComparedToXunits_ThenT Assert.NotNull(field); Assert.Equal(expectedVariableName, field.GetRawConstantValue()); } + + /// + /// Pins which option goes into which variable, not just that the names are right. + /// + /// + /// Every one of these options is an , so two of them swapped compiles, sets + /// every variable, and leaves all five names correct - the only symptom is assertion failure + /// messages truncating at the wrong place, which the name comparison above cannot see. Giving + /// each option a distinct value is what makes a swap fail here. + /// + [Fact] + public void GivenEveryAssertionFormattingOption_WhenTheyAreCopiedToTheEnvironment_ThenEachReachesItsOwnVariable() + { + var options = new RecordingExecutionOptions(); + options.SetAssertEquivalentMaxDepth(11); + options.SetPrintMaxEnumerableLength(12); + options.SetPrintMaxObjectDepth(13); + options.SetPrintMaxObjectMemberCount(14); + options.SetPrintMaxStringLength(15); + + Dictionary environment = CustomXunitTestFrameworkExecutor + .BuildAssertionFormattingEnvironment(options) + .ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + + Assert.Equal( + new Dictionary(StringComparer.Ordinal) + { + [CustomXunitTestFrameworkExecutor.AssertEquivalentMaxDepthVariable] = 11, + [CustomXunitTestFrameworkExecutor.PrintMaxEnumerableLengthVariable] = 12, + [CustomXunitTestFrameworkExecutor.PrintMaxObjectDepthVariable] = 13, + [CustomXunitTestFrameworkExecutor.PrintMaxObjectMemberCountVariable] = 14, + [CustomXunitTestFrameworkExecutor.PrintMaxStringLengthVariable] = 15, + }, + environment); + } + + /// + /// An option nobody passed has to stay unset rather than being written as some default. The + /// variables are process-wide and outlive the run that set them, so writing a value xunit + /// would otherwise have chosen for itself would silently override whatever the environment + /// already said. + /// + [Fact] + public void GivenNoAssertionFormattingOptions_WhenTheyAreCopiedToTheEnvironment_ThenNoVariableIsGivenAValue() + { + IReadOnlyList> environment = + CustomXunitTestFrameworkExecutor.BuildAssertionFormattingEnvironment(new RecordingExecutionOptions()); + + Assert.All(environment, pair => Assert.Null(pair.Value)); + } + + /// + /// The smallest thing that can stand in for the runner's options: xunit's own extension + /// methods supply the option names, so the test states only the pairing and never repeats a + /// name that could drift from the one the runner really uses. + /// + private sealed class RecordingExecutionOptions : ITestFrameworkExecutionOptions + { + private readonly Dictionary _values = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public TValue GetValue(string name) + { + return _values.TryGetValue(name, out object value) ? (TValue)value : default; + } + + public void SetValue(string name, TValue value) + { + _values[name] = value; + } + + public string ToJson() + { + return "{}"; + } + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs new file mode 100644 index 0000000000..51c612c3bb --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs @@ -0,0 +1,151 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that a fault belonging to a whole class is reported so that every leg which would + /// have run any of its tests still sees a failure, and no leg sees one it should not. + /// + /// + /// A class-level fault loses every method at once, which is where a merged failure does the most + /// damage. One failure carrying every method's traits is dropped by a leg excluding any one of + /// them; one carrying only the first method's is invisible to a leg selecting by another's. + /// Either way a leg passes green with tests missing, and nothing in its output says so. The + /// failures are therefore reported one per lost method, per combination that method would have + /// run under. + /// + public class ClassAttributeFaultTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ClassAttributeFault.ClassAttributeFaultTests"; + private const string FirstSqlCase = ScenarioClass + ".FirstNeverRuns (fixture argument set discovery: Sql)"; + private const string FirstSqlSomeCase = ScenarioClass + ".FirstNeverRuns (fixture argument set discovery: Sql, Some)"; + private const string SecondSqlCase = ScenarioClass + ".SecondNeverRuns (fixture argument set discovery: Sql)"; + private const string SecondSqlSomeCase = ScenarioClass + ".SecondNeverRuns (fixture argument set discovery: Sql, Some)"; + private const string SecondCosmosCase = ScenarioClass + ".SecondNeverRuns (fixture argument set discovery: Cosmos)"; + + /// + /// A fault in the class's own declaration loses every method, so every method gets a failure + /// standing in for it, under each combination it would have run. + /// + /// + /// Reporting one failure for the class was the obvious thing and is wrong: a class of many + /// methods that disagree about their traits cannot be represented by one case, because a + /// filter selects or drops that case whole. + /// + [Fact] + public void GivenAClassWhoseOwnAttributeCannotBeRead_WhenItIsDiscovered_ThenEveryLostMethodGetsAFailure() + { + TestAssetRun run = TestAssetRunner.Run("ClassAttributeFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstSqlCase] = "Failed", + [FirstSqlSomeCase] = "Failed", + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + [SecondCosmosCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The class's declaration cannot be read as the single attribute the expansion would have + /// used, but each attribute on its own can be. Answering the failure with no values at all + /// would leave every failure carrying no argument set trait, and a leg selecting by one would + /// pass with the whole class missing. + /// + [Fact] + public void GivenAClassDeclaringTwoArgumentSetAttributes_WhenItIsDiscovered_ThenTheFailuresCarryBothDeclarations() + { + TestAssetRun run = TestAssetRunner.Run("ClassAttributeFault", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstSqlCase] = "Failed", + [FirstSqlSomeCase] = "Failed", + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// This is the shape the repository's own E2E and export legs use: a query selecting on an + /// argument set value and an ordinary trait at once. A failure missing either half matches + /// nothing, so this is the only filter that pins both halves reaching the same case. + /// + [Fact] + public void GivenAClassLevelFault_WhenTestsAreSelectedByACompoundQuery_ThenTheMatchingFailureIsReported() + { + TestAssetRun run = TestAssetRunner.Run( + "ClassAttributeFault", + filterQueryTraits: "(AssetDataStore=Cosmos)&(Category=DeclaredLast)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SecondCosmosCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The other half of the compound query: only the second method declares + /// AssetDataStore.Cosmos, and only the first declares Category=DeclaredFirst, so + /// no test would ever have run under both. A failure matching this query would mean the + /// failures are carrying traits pooled across methods - which is what makes a leg excluding + /// one method's trait drop the failures standing in for all the others. + /// + [Fact] + public void GivenAClassLevelFault_WhenAQuerySelectsACombinationNoMethodDeclared_ThenNothingMatches() + { + TestAssetRun run = TestAssetRunner.Run( + "ClassAttributeFault", + filterQueryTraits: "(AssetDataStore=Cosmos)&(Category=DeclaredFirst)"); + + Assert.Empty(run.Results); + } + + /// + /// Two overloads that both fail discovery are two lost methods, so both have to be reported. + /// Identifying a failure by its method name alone gave them one identity between them, and + /// xunit kept a single case - leaving one method's loss invisible. + /// + /// + /// The failures share a display name, because that is built from the method name too, so this + /// counts results rather than naming them. Each overload is reported under both combinations + /// its two declarations expand to, so four failures stand for the two lost methods. + /// + [Fact] + public void GivenTwoOverloadsThatBothFailDiscovery_WhenTheyAreDiscovered_ThenBothAreReported() + { + TestAssetRun run = TestAssetRunner.Run("OverloadedFault"); + + Assert.Equal(0, run.ErrorCount); + Assert.Equal( + 4, + run.Results.Count(result => + string.Equals(result.Outcome, "Failed", StringComparison.Ordinal) && + result.Name != null && + result.Name.Contains("OverloadedFaultTests.NeverRuns", StringComparison.Ordinal))); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs index 49f4ae1bc0..7cf25f5fc0 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs @@ -19,11 +19,12 @@ public class DiscoveryFaultTests private const string CosmosErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Cosmos)"; /// - /// xUnit reports an exception thrown out of discovery only as an internal diagnostic, which - /// is suppressed by default, and carries on without the class. A run containing any other - /// healthy class therefore still reports success, so a broken expansion is indistinguishable - /// from a class that has no tests. The discoverer turns the fault into a failing test case - /// instead, which puts it in the results and in the exit code. + /// xUnit reports an exception thrown out of discovery only as a diagnostic message, which is + /// suppressed unless the run was started with --xunit-diagnostics, and carries on + /// without the class. A run containing any other healthy class therefore still reports + /// success, so a broken expansion is indistinguishable from a class that has no tests. The + /// discoverer turns the fault into a failing test case instead, which puts it in the results + /// and in the exit code. /// [Fact] public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheFaultIsReportedAsAFailedTest() diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultArgumentSetCombinationTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultArgumentSetCombinationTests.cs new file mode 100644 index 0000000000..6ecd484be3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultArgumentSetCombinationTests.cs @@ -0,0 +1,138 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers which failures a class whose expansion faulted is reported as, given the argument set + /// attributes it declared. + /// + /// + /// The scenario assets exercise this end to end, but only over the one shape this repository + /// happens to use: a single enum dimension whose values every method agrees on. The shapes that + /// are not covered there are the ones where getting this wrong is invisible - a combination that + /// should exist but does not is a CI leg passing green with the class's tests missing, which is + /// precisely what reporting the fault as a test exists to prevent. They are pinned directly here + /// rather than by adding assets no product code resembles. + /// + public class FaultArgumentSetCombinationTests + { + [Flags] + private enum Store + { + Sql = 1, + Cosmos = 2, + } + + [Flags] + private enum Format + { + Json = 1, + Ndjson = 2, + } + + /// + /// A class declaring nothing still has to be reported, or a fault in a class that takes no + /// argument sets would produce no failing test at all. + /// + [Fact] + public void GivenNoArgumentSets_WhenCombinationsAreBuilt_ThenASingleEmptyCombinationIsProduced() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations(Array.Empty()); + + Assert.Empty(Assert.Single(combinations)); + } + + /// + /// Every variant an attribute would have expanded to needs its own failure, so that a leg + /// selecting one variant by trait still sees a failure standing in for it. + /// + [Fact] + public void GivenTwoDimensionsOnOneAttribute_WhenCombinationsAreBuilt_ThenTheirProductIsProduced() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations( + new[] { new[] { Flags(Store.Sql, Store.Cosmos), Flags(Format.Json, Format.Ndjson) } }); + + Assert.Equal( + new[] { "Sql+Json", "Sql+Ndjson", "Cosmos+Json", "Cosmos+Ndjson" }.OrderBy(x => x, StringComparer.Ordinal), + combinations.Select(Describe).OrderBy(x => x, StringComparer.Ordinal)); + } + + /// + /// This is the case that makes each attribute's values its own product rather than one pooled + /// product. Were the two pooled, the only combinations produced would carry a format, and a + /// leg running --filter-not-trait Format=Json alongside one running + /// --filter-not-trait Format=Ndjson would between them exclude every failure the class + /// produced - both legs green, the whole class silently absent. + /// + [Fact] + public void GivenTwoAttributesUsingDifferentDimensions_WhenCombinationsAreBuilt_ThenNeitherBorrowsTheOthersValues() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations( + new[] + { + new[] { Flags(Store.Sql, Store.Cosmos) }, + new[] { Flags(Format.Json) }, + }); + + Assert.Equal( + new[] { "Cosmos", "Json", "Sql" }, + combinations.Select(Describe).OrderBy(x => x, StringComparer.Ordinal)); + } + + /// + /// Methods usually declare the same argument sets as each other, and two failures standing for + /// the same combination would collide on unique ID as well as reporting the fault twice. + /// + [Fact] + public void GivenTwoAttributesDeclaringTheSameValues_WhenCombinationsAreBuilt_ThenTheCombinationIsProducedOnce() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations( + new[] + { + new[] { Flags(Store.Sql, Store.Cosmos) }, + new[] { Flags(Store.Cosmos, Store.Sql) }, + }); + + Assert.Equal( + new[] { "Cosmos", "Sql" }, + combinations.Select(Describe).OrderBy(x => x, StringComparer.Ordinal)); + } + + /// + /// An attribute whose values could not be expanded arrives here as no dimensions at all. It + /// must not take the readable attributes' combinations down with it. + /// + [Fact] + public void GivenAnAttributeWithNoUsableDimensions_WhenCombinationsAreBuilt_ThenTheOtherAttributesAreStillClosed() + { + var combinations = CustomXunitTestFrameworkDiscoverer.BuildFaultArgumentSetCombinations( + new[] + { + new[] { Array.Empty() }, + new[] { Flags(Store.Sql, Store.Cosmos) }, + }); + + Assert.Equal( + new[] { string.Empty, "Cosmos", "Sql" }, + combinations.Select(Describe).OrderBy(x => x, StringComparer.Ordinal)); + } + + private static SingleFlag[] Flags(params Enum[] values) + { + return values.Select(value => new SingleFlag(value)).ToArray(); + } + + private static string Describe(IReadOnlyList combination) + { + return string.Join("+", combination.Select(flag => flag.EnumValue.ToString())); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs index a2b799b347..086a9c9d3a 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MethodAttributeFaultTests.cs @@ -16,7 +16,8 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests public class MethodAttributeFaultTests { private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MethodAttributeFault.MethodAttributeFaultTests"; - private const string ErrorCaseName = ScenarioClass + ".NeverRuns (fixture argument set discovery: Cosmos)"; + private const string SqlErrorCaseName = ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"; + private const string SqlSomeErrorCaseName = ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql, Some)"; /// /// Reading the attribute is a separate step from expanding it, and it can fail on its own: a @@ -37,22 +38,45 @@ public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTheClassIsDiscovered_Th { [ScenarioClass + ".RunsBeforeTheFault"] = "Passed", [ScenarioClass + ".RunsAfterTheFault (Cosmos)"] = "Passed", - [ErrorCaseName] = "Failed", + [SqlErrorCaseName] = "Failed", + [SqlSomeErrorCaseName] = "Failed", }); Assert.NotEqual(0, run.ExitCode); } /// - /// The values a fault case carries as traits are pooled from every method of the class, and - /// reading them can hit the very declaration that caused the fault. Reading the whole class in - /// one attempt meant that one unreadable method left the pool empty, so the failure went out - /// with no argument set traits at all - and a leg selecting its tests by those traits would - /// then skip the failure and pass with the class missing. Here only the failing method's - /// values are unreadable, so the case must still carry the value its sibling declares. + /// The failure has to carry the values the failing method itself declared, so that the leg + /// which would have run it is the leg that sees it. Asking the method for the single attribute + /// it declares is what threw, and answering that with nothing left the failure carrying only + /// values borrowed from a sibling - reported to the Cosmos leg, which would never have run + /// this method, and hidden from the SQL leg, which would. Reading each of the method's + /// attributes on its own keeps the failure where its tests were. /// [Fact] - public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTestsAreSelectedByArgumentSetTrait_ThenTheFaultIsStillReported() + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTestsAreSelectedByItsOwnArgumentSetTrait_ThenTheFaultIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SqlErrorCaseName] = "Failed", + [SqlSomeErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The other side of the same guarantee: the sibling's value must not leak onto the failure. + /// A failure tagged with a value its method never declared is a red test on a leg that would + /// never have run it, and - worse - is dropped by that leg's exclusion filter in the runs + /// where the fault should have been reported elsewhere. + /// + [Fact] + public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTestsAreSelectedByASiblingsArgumentSetTrait_ThenOnlyTheSiblingRuns() { TestAssetRun run = TestAssetRunner.Run("MethodAttributeFault", filterTrait: "AssetDataStore=Cosmos"); @@ -61,10 +85,9 @@ public void GivenOneMethodWhoseAttributeCannotBeRead_WhenTestsAreSelectedByArgum new Dictionary { [ScenarioClass + ".RunsAfterTheFault (Cosmos)"] = "Passed", - [ErrorCaseName] = "Failed", }); - Assert.NotEqual(0, run.ExitCode); + Assert.Equal(0, run.ExitCode); } /// diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index 5e8be45707..dec541116f 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -61,9 +61,25 @@ public static class TestAssetRunner /// other data store - so this is how a scenario checks that a leg excluding one value still /// sees what it was meant to run. /// + /// + /// The trait predicate of a query filter, in the form the runner takes inside the brackets of a + /// query - such as (DataStore=CosmosDb)&(Category=ExportLongRunning). This is the + /// compound form the repository's E2E and export legs select with, where a case runs only if it + /// carries every named trait, so a failure missing any one of them is invisible to the leg that + /// would have run the tests it stands for. The runner scopes the query to the scenario itself, + /// because a query filter cannot be combined with the plain namespace filter used otherwise. + /// /// The exit code, output and published results of the run. - public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null, string maxThreads = null, string filterNotTrait = null) + public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null, string maxThreads = null, string filterNotTrait = null, string filterQueryTraits = null) { + // The runner rejects a query filter outright once any plain filter has been added, so the + // two forms cannot be mixed. Saying that here keeps a scenario that tries from failing as an + // unhandled exception inside the asset, which reads as the asset being broken. + if (!string.IsNullOrEmpty(filterQueryTraits) && (!string.IsNullOrEmpty(filterTrait) || !string.IsNullOrEmpty(filterNotTrait))) + { + throw new ArgumentException("A query filter cannot be combined with a trait filter; express the whole selection as a query.", nameof(filterQueryTraits)); + } + string assetsAssembly = ResolveAssetsAssembly(); string resultsDirectory = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", Guid.NewGuid().ToString("N")); Directory.CreateDirectory(resultsDirectory); @@ -80,8 +96,6 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool pr { "exec", assetsAssembly, - "--filter-namespace", - ScenarioNamespacePrefix + scenario, "--parallel", "collections", "--report-trx", @@ -91,6 +105,20 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool pr resultsDirectory, }; + if (string.IsNullOrEmpty(filterQueryTraits)) + { + arguments.Add("--filter-namespace"); + arguments.Add(ScenarioNamespacePrefix + scenario); + } + else + { + // A query names each level of the test's identity in turn - assembly, namespace, + // class, method - before the traits, so scoping it to the scenario means matching + // any assembly and any class and method within that one namespace. + arguments.Add("--filter-query"); + arguments.Add(FormattableString.Invariant($"/*/{ScenarioNamespacePrefix}{scenario}/*/*/[{filterQueryTraits}]")); + } + if (stopOnFail) { arguments.Add("--stop-on-fail"); diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index cf413b51d5..b3dcae7f51 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -60,11 +60,22 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl // own faults, so it belongs to the class as a whole and no method of it was expanded. // // testClass.Methods includes every public method, property accessors among them, so - // the anchor is chosen from those xunit would treat as tests: a fault reported - // against a property getter names nothing a reader can act on, and the getter carries - // none of the traits a CI leg selects the class's tests by. - MethodInfo firstMethod = testClass.Methods.FirstOrDefault(IsTestMethod) ?? testClass.Methods.FirstOrDefault(); - if (firstMethod == null) + // the failures are reported against those xunit would treat as tests: a fault + // reported against a property getter names nothing a reader can act on, and the + // getter carries none of the traits a CI leg selects the class's tests by. + // + // One failure is reported per lost method rather than one for the class, because a + // single failure can only carry one set of traits, and the methods of a class need + // not agree on theirs. A failure carrying every method's traits is dropped by a leg + // excluding any one of them; a failure carrying only the first method's is invisible + // to a leg selecting by another's. Either way a leg passes green with tests missing. + MethodInfo[] lostMethods = testClass.Methods.Where(IsTestMethod).ToArray(); + if (lostMethods.Length == 0) + { + lostMethods = testClass.Methods.Take(1).ToArray(); + } + + if (lostMethods.Length == 0) { // Nothing to hang a test case off. Rethrowing keeps the original xunit behaviour, // which is all that is left. @@ -75,7 +86,7 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl return await ReportDiscoveryFault( testClass, - firstMethod, + lostMethods, tracker, ex, $"Discovering the fixture argument set variants of '{testClass.TestClassName}' failed, so none of its tests ran."); @@ -108,16 +119,34 @@ internal static bool ShouldRethrowRatherThanReport(Exception exception, bool isC /// /// Determines whether xunit would treat a method as a test. /// + /// + /// This is asked while handling a discovery failure, over the very metadata that may have + /// caused it, so a method whose attributes cannot be read is treated as not a test rather than + /// allowed to throw. The caller then falls back to any method at all, which reports the failure + /// against a less helpful name - still far better than letting the class vanish from a green + /// run, which is what an escaping exception here would cause. + /// /// The method to inspect. /// true when the method carries a fact or theory attribute. - private static bool IsTestMethod(MethodInfo method) => - method.GetCustomAttributes(typeof(FactAttribute), inherit: true).Length > 0; + private static bool IsTestMethod(MethodInfo method) + { + try + { + return method.GetCustomAttributes(typeof(FactAttribute), inherit: true).Length > 0; + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the attributes of '{method.Name}' could not be read while choosing a method to report a discovery failure against. {ex}"); + return false; + } + } /// /// Reports a discovery failure as a test case that fails when the run reaches it. /// /// The class being discovered. - /// The method to report the failure against. + /// The methods the fault lost, each of which gets its own failures. /// The callback discovered test cases are handed to. /// The failure being reported. /// A sentence saying what was lost, used to open the failure message. @@ -133,13 +162,17 @@ private static bool IsTestMethod(MethodInfo method) => /// that would have selected the tests it replaces, and discarding the aggregated failure /// discards this message with it, leaving the fault unreported again. /// - /// One case is reported per combination of argument set values rather than one case carrying - /// them all, because a trait exclusion filter drops a case when any of its values - /// under the named trait matches. A single case declaring both DataStore=CosmosDb and - /// DataStore=SqlServer would be dropped by the SQL leg's - /// --filter-not-trait DataStore=CosmosDb and by the Cosmos leg's - /// --filter-not-trait DataStore=SqlServer alike, so the fault would be reported to - /// nobody while both legs stayed green. + /// Every failure reported here stands for exactly one thing that would have run: one method, + /// under one combination of argument set values. Nothing is merged, because a case carrying + /// more than one method's traits, or more than one combination's values, is a case a filter can + /// drop for a reason that applies to only part of what it stands for. A trait filter drops a + /// case when any of its values under the named trait matches, so a single case + /// declaring both DataStore=CosmosDb and DataStore=SqlServer is dropped by the + /// SQL leg's --filter-not-trait DataStore=CosmosDb and by the Cosmos leg's + /// --filter-not-trait DataStore=SqlServer alike - reported to nobody while both legs + /// stay green. The same holds of ordinary traits: pooling one method's + /// Category=ExportLongRunning into the failures standing for methods that do not + /// declare it would let the normal leg's Category!=ExportLongRunning drop all of them. /// /// /// A case standing for a combination is anchored to that combination's variant class, the same @@ -153,14 +186,36 @@ private static bool IsTestMethod(MethodInfo method) => /// private async ValueTask ReportDiscoveryFault( IXunitTestClass testClass, - MethodInfo method, + IReadOnlyList lostMethods, CallbackTracker tracker, Exception exception, string summary) { Console.WriteLine($"[FixtureArgumentSets] ERROR: {summary} It was replaced by a failing test case. {exception}"); - foreach (IReadOnlyList combination in BuildFaultArgumentSetCombinations(ReadFaultArgumentSetValues(testClass))) + foreach (MethodInfo method in lostMethods) + { + if (!await ReportDiscoveryFaultForMethod(testClass, method, tracker, exception, summary)) + { + return false; + } + } + + return true; + } + + private async ValueTask ReportDiscoveryFaultForMethod( + IXunitTestClass testClass, + MethodInfo method, + CallbackTracker tracker, + Exception exception, + string summary) + { + // Overloads share a name, so a fault reported against each of two of them would otherwise + // give both cases the same unique ID and xunit would keep only one. + string methodKey = BuildFaultMethodKey(method); + + foreach (IReadOnlyList combination in BuildFaultArgumentSetCombinations(ReadFaultArgumentSetDimensions(testClass, method))) { string displaySuffix = combination.Count == 0 ? string.Empty @@ -192,7 +247,7 @@ private async ValueTask ReportDiscoveryFault( anchorClass, method, Array.Empty(), - uniqueID: UniqueIDGenerator.ForTestMethod(anchorClass.UniqueID, method.Name)); + uniqueID: UniqueIDGenerator.ForTestMethod(anchorClass.UniqueID, methodKey)); // The test case is handed to xunit through the callback, which owns it from then on // and disposes it with the rest of the discovered cases. @@ -200,7 +255,7 @@ private async ValueTask ReportDiscoveryFault( var errorTestCase = new DiscoveryErrorTestCase( errorTestMethod, $"{testClass.TestClassName}.{method.Name} (fixture argument set discovery{displaySuffix})", - $"{testClass.UniqueID}-{method.Name}-fixture-argument-set-discovery-error{idSuffix}", + $"{testClass.UniqueID}-{methodKey}-fixture-argument-set-discovery-error{idSuffix}", errorMessage: $"{summary} {exception}", traits: BuildFaultTraits(errorTestMethod, combination)); #pragma warning restore CA2000 @@ -215,43 +270,82 @@ private async ValueTask ReportDiscoveryFault( } /// - /// Closes the argument set values a failed class declared over each other, one dimension per - /// enum type, so that each combination can be reported as its own failure. + /// Closes the argument set values a failed class declared over each other, so that each + /// combination its tests would have run as can be reported as its own failure. /// - /// Every argument set value read off the class, in any order. + /// + /// The dimensions each declared argument set attribute expands to, one entry per attribute, + /// each holding that attribute's dimensions in the order it declares them. + /// /// - /// One list per combination. A class declaring no argument set values yields a single empty - /// combination, so a fault is always reported at least once. + /// One list per combination, deduplicated. A class declaring no argument set values yields a + /// single empty combination, so a fault is always reported at least once. /// /// - /// Values are grouped by enum type because that is what a dimension is: an attribute takes one - /// flags enum per fixture constructor parameter, and the variants a class expands to are the - /// product of the single flags set within each. Reproducing that product here means a leg - /// selecting a variant by trait selects the failure standing in for it, and a leg excluding a - /// variant by trait excludes only that one and still sees the rest. + /// A dimension is one flags enum, taken by one of the fixture's constructor parameters, and + /// the variants an attribute expands to are the product of the single flags set within each of + /// its dimensions. Reproducing that product means a leg selecting a variant by trait selects + /// the failure standing in for it, and a leg excluding a variant by trait excludes only that + /// one and still sees the rest. + /// + /// Each attribute is closed over separately rather than every value being pooled into one + /// product, because attributes need not agree on which dimensions they use. Pooling would give + /// the failures standing in for a method that declares only a data store a format value it + /// never asked for, and a leg excluding that format would then exclude every failure the class + /// produced - passing green with those tests missing, which is the one outcome this whole + /// mechanism exists to prevent. Closing each attribute separately keeps every combination one + /// that some method really would have run as. + /// /// - internal static IReadOnlyList> BuildFaultArgumentSetCombinations(IEnumerable argumentSetValues) + internal static IReadOnlyList> BuildFaultArgumentSetCombinations(IEnumerable argumentSetDimensions) { - EnsureArg.IsNotNull(argumentSetValues, nameof(argumentSetValues)); - - IEnumerable> dimensions = argumentSetValues - .Where(flag => flag.EnumValue != null) - .GroupBy(flag => flag.EnumValue.GetType()) - .OrderBy(dimension => dimension.Key.Name, StringComparer.Ordinal) - .Select(dimension => (IEnumerable)dimension - .GroupBy(flag => flag.EnumValue) - .Select(values => values.First()) - .ToArray()) - .ToArray(); + EnsureArg.IsNotNull(argumentSetDimensions, nameof(argumentSetDimensions)); - return CartesianProduct(dimensions) - .Select(combination => (IReadOnlyList)combination.ToArray()) - .ToArray(); + var combinations = new List>(); + + // Two attributes declaring the same values would otherwise be reported twice, and the two + // failures would share a unique ID. + var seen = new HashSet(StringComparer.Ordinal); + + foreach (SingleFlag[][] attributeDimensions in argumentSetDimensions) + { + IEnumerable> dimensions = (attributeDimensions ?? Array.Empty()) + .Where(dimension => dimension != null) + .Select(dimension => (IEnumerable)dimension + .Where(flag => flag.EnumValue != null) + .GroupBy(flag => flag.EnumValue) + .Select(values => values.First()) + .ToArray()) + .Where(dimension => dimension.Any()) + .ToArray(); + + foreach (IEnumerable combination in CartesianProduct(dimensions)) + { + SingleFlag[] closed = combination.ToArray(); + string key = string.Join( + "|", + closed + .Select(flag => $"{flag.EnumValue.GetType().FullName}.{flag.EnumValue}") + .OrderBy(text => text, StringComparer.Ordinal)); + + if (seen.Add(key)) + { + combinations.Add(closed); + } + } + } + + if (combinations.Count == 0) + { + combinations.Add(Array.Empty()); + } + + return combinations; } /// - /// Reads every argument set value a class and its methods declare, for use in reporting a - /// discovery failure. + /// Reads the argument set dimensions that apply to one lost method, for use in reporting the + /// discovery failure that stands in for it. /// /// /// This runs while handling a discovery failure, and reads the same attributes whose contents @@ -259,61 +353,59 @@ internal static IReadOnlyList> BuildFaultArgumentSetCo /// further exception costs only that one attribute's values. Reporting the failure with fewer /// traits is worth more than not reporting it at all. /// - /// Every method's values are pooled, not just the anchor method's, because a class-level fault - /// loses every method. Pooling can only add combinations, and an extra combination reports the - /// fault once more rather than hiding it, whereas a missing one is a leg that passes with the - /// tests absent. + /// Every attribute the class and the method declare is read, not the single one the expansion + /// would have used, because declaring two is one of the ways the expansion fails: asking for + /// the single attribute throws, and answering that with no values at all leaves the failure + /// carrying none of the traits its own declaration named. A method declaring + /// DataStore=SqlServer twice over would then be reported to the Cosmos leg and not to + /// the SQL one - the leg that would have run it passing green without it. + /// + /// + /// Only this method's own attributes are read, never a sibling's. The failure stands for this + /// method alone, and a value borrowed from a sibling is a value no test of this method would + /// have carried, which an exclusion filter can then use to drop the failure entirely. /// /// /// The class whose expansion failed. - /// The declared values; empty when none could be read. - private static SingleFlag[] ReadFaultArgumentSetValues(IXunitTestClass testClass) + /// The lost method the failure stands in for. + /// The dimensions of each declared attribute; empty when none could be read. + private static List ReadFaultArgumentSetDimensions(IXunitTestClass testClass, MethodInfo method) { - var attributes = new List(); - - // Each attribute is read on its own so that the one whose contents may have caused the - // fault in the first place costs only its own values, rather than emptying the pool and - // leaving the failure with no traits for a leg's filter to select it by. - TryAdd(() => testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute); + var dimensions = new List(); - foreach (MethodInfo method in testClass.Methods) - { - MethodInfo closedMethod = method; - TryAdd(() => closedMethod.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false).SingleOrDefault() as FixtureArgumentSetsAttribute); - } + AddAll(() => testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false), testClass.TestClassName); + AddAll(() => method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false), $"{testClass.TestClassName}.{method.Name}"); - return attributes - .SelectMany(SafeExpand) - .Where(flag => flag.EnumValue != null) - .ToArray(); + return dimensions; - void TryAdd(Func read) + void AddAll(Func read, string owner) { + object[] attributes; try { - if (read() is FixtureArgumentSetsAttribute attribute) - { - attributes.Add(attribute); - } + attributes = read(); } catch (Exception ex) { Console.WriteLine( - $"[FixtureArgumentSets] WARNING: some argument set traits of '{testClass.TestClassName}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + $"[FixtureArgumentSets] WARNING: the argument set attributes of '{owner}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + return; } - } - IEnumerable SafeExpand(FixtureArgumentSetsAttribute attribute) - { - try - { - return ExpandEnumFlagsFromAttributeData(attribute).SelectMany(dimension => dimension ?? Array.Empty()).ToArray(); - } - catch (Exception ex) + // Each attribute is expanded on its own so that the one whose contents may have caused + // the fault in the first place costs only its own values, rather than leaving the + // failure with no traits for a leg's filter to select it by. + foreach (FixtureArgumentSetsAttribute attribute in attributes.OfType()) { - Console.WriteLine( - $"[FixtureArgumentSets] WARNING: some argument set traits of '{testClass.TestClassName}' could not be expanded, so a trait filter may not select the failure standing in for its tests. {ex}"); - return Array.Empty(); + try + { + dimensions.Add(ExpandEnumFlagsFromAttributeData(attribute)); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: an argument set attribute of '{owner}' could not be expanded, so a trait filter may not select the failure standing in for its tests. {ex}"); + } } } } @@ -323,17 +415,28 @@ IEnumerable SafeExpand(FixtureArgumentSetsAttribute attribute) /// the same filters they would have been. /// /// + /// /// Two kinds of trait matter. The ordinary ones the method and its class declare come from the /// anchoring method itself, and CI legs combine them with argument set values in filters such /// as (DataStore=CosmosDb)&(Category=ExportLongRunning), so leaving either out lets /// the leg that would have run these tests pass without them. The argument set values are the /// ones the expansion never got far enough to produce, so they are supplied by the caller from /// the attributes rather than read off any discovered variant. + /// + /// + /// Reading the anchor's own traits runs xunit's trait discovery, which evaluates user code and + /// the method's and class's attributes, and so can fail for the same reason the expansion did. + /// That is caught here: a failure reported with only its argument set traits is still reported, + /// whereas letting the read escape would take the whole mechanism down with it and leave the + /// class silently absent from a green run. + /// /// /// The method the failure is reported against. /// The one combination of argument set values this failure stands for. /// The traits, keyed by trait name. - private static Dictionary> BuildFaultTraits(XunitTestMethod anchor, IReadOnlyList argumentSetCombination) + private static Dictionary> BuildFaultTraits( + XunitTestMethod anchor, + IReadOnlyList argumentSetCombination) { var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); @@ -348,13 +451,21 @@ void Add(string key, string value) ((HashSet)values).Add(value); } - foreach (KeyValuePair> trait in anchor.Traits) + try { - foreach (string value in trait.Value) + foreach (KeyValuePair> trait in anchor.Traits) { - Add(trait.Key, value); + foreach (string value in trait.Value) + { + Add(trait.Key, value); + } } } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the ordinary traits of '{anchor.TestClass.TestClassName}.{anchor.MethodName}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + } foreach (SingleFlag flag in argumentSetCombination) { @@ -364,6 +475,39 @@ void Add(string key, string value) return traits; } + /// + /// Builds the part of a discovery failure's unique ID that identifies the method it is + /// reported against. + /// + /// + /// The method's name alone is not unique: a class may overload a test method, and two failures + /// sharing a unique ID leave xunit reporting only one of them - the other silently absent, + /// which is the outcome reporting these failures exists to prevent. The parameter types are + /// included to tell overloads apart, and are read defensively because this runs while handling + /// a failure that may itself be a type that cannot be loaded. + /// + /// The method the failure is reported against. + /// A key identifying the method, falling back to its name alone. + private static string BuildFaultMethodKey(MethodInfo method) + { + try + { + ParameterInfo[] parameters = method.GetParameters(); + if (parameters.Length == 0) + { + return method.Name; + } + + return $"{method.Name}({string.Join(",", parameters.Select(parameter => parameter.ParameterType.FullName ?? parameter.ParameterType.Name))})"; + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the signature of '{method.Name}' could not be read, so a failure reported against an overload of it may be dropped as a duplicate. {ex}"); + return method.Name; + } + } + /// /// Reads the fixture argument set attribute a method declares. /// @@ -435,7 +579,7 @@ private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, IT { continueDiscovery = await ReportDiscoveryFault( testClass, - method, + new[] { method }, tracker, ex, $"Discovering the fixture argument set variants of '{testClass.TestClassName}.{method.Name}' failed, so none of that method's tests ran. Other methods of the class were discovered normally."); diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs index d4d6298f1e..d6656d1984 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkExecutor.cs @@ -69,16 +69,42 @@ public override async ValueTask RunTestCases(IReadOnlyCollection { EnsureArg.IsNotNull(executionOptions, nameof(executionOptions)); - SetEnvironment(AssertEquivalentMaxDepthVariable, executionOptions.AssertEquivalentMaxDepth()); - SetEnvironment(PrintMaxEnumerableLengthVariable, executionOptions.PrintMaxEnumerableLength()); - SetEnvironment(PrintMaxObjectDepthVariable, executionOptions.PrintMaxObjectDepth()); - SetEnvironment(PrintMaxObjectMemberCountVariable, executionOptions.PrintMaxObjectMemberCount()); - SetEnvironment(PrintMaxStringLengthVariable, executionOptions.PrintMaxStringLength()); + foreach (KeyValuePair variable in BuildAssertionFormattingEnvironment(executionOptions)) + { + SetEnvironment(variable.Key, variable.Value); + } var runner = new FixtureArgumentSetAssemblyRunner(); await runner.Run(TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); } + /// + /// Pairs each assertion formatting option with the environment variable Assert reads it + /// from. + /// + /// + /// The pairing is the part that can go wrong silently. Every one of these options is an + /// , so two of them swapped still compiles, still sets both variables and + /// still leaves the names correct - it only makes assertion failure messages truncate at the + /// wrong place, which no test asserting on names would notice. Returning the pairs rather than + /// setting them directly is what lets a test state them. + /// + /// The options the run was started with. + /// Each variable name with the value to give it, or null to leave it alone. + internal static IReadOnlyList> BuildAssertionFormattingEnvironment(ITestFrameworkExecutionOptions executionOptions) + { + EnsureArg.IsNotNull(executionOptions, nameof(executionOptions)); + + return new[] + { + new KeyValuePair(AssertEquivalentMaxDepthVariable, executionOptions.AssertEquivalentMaxDepth()), + new KeyValuePair(PrintMaxEnumerableLengthVariable, executionOptions.PrintMaxEnumerableLength()), + new KeyValuePair(PrintMaxObjectDepthVariable, executionOptions.PrintMaxObjectDepth()), + new KeyValuePair(PrintMaxObjectMemberCountVariable, executionOptions.PrintMaxObjectMemberCount()), + new KeyValuePair(PrintMaxStringLengthVariable, executionOptions.PrintMaxStringLength()), + }; + } + private static void SetEnvironment(string environmentVariableName, int? value) { if (value.HasValue) diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index d80667e0ac..dbb2b1e900 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -31,6 +31,11 @@ public FixtureArgumentSetTestMethod(FixtureArgumentSetTestClass testClass, Metho #pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes public FixtureArgumentSetTestMethod() { + // The de-serializer does not run either of the field's assignments, and the argument set + // values are recovered from the base type's method arguments rather than from this field. + // Leaving it null would turn any later read into a NullReferenceException reported as a + // test failure with no bearing on the test, so it starts empty as it does on the class. + _fixtureArguments = Array.Empty(); } #pragma warning restore CS0618 From a3857d640e1d96a6922cdda909cc08fa4cc079d7 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 14:00:25 -0700 Subject: [PATCH 051/105] Inherit the dimensions a method's attribute does not declare A method whose fixture argument set attribute takes fewer dimensions than its class's had the trailing ones dropped rather than inherited. Its tests still ran, but without those values, and so without the traits a CI leg selects them by: the leg passed green with the tests silently absent. Also recognise test methods by IFactAttribute, which is what xunit itself discovers by, so a method marked by an attribute that implements the interface without deriving from FactAttribute still gets a failure standing in for it; and tell generic overloads differing only in arity apart, which a key built from name and parameter types alone could not do. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../ArgumentSetOverrideTests.cs | 13 ++ .../ClassAttributeFaultTests.cs | 9 +- .../MethodAttributeFaultTests.cs | 4 +- .../ArgumentSetOverrideTests.cs | 28 ++++ .../ClassAttributeFaultTests.cs | 78 ++++++++++ .../FaultMethodSelectionTests.cs | 146 ++++++++++++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 138 ++++++++++++++--- .../DiscoveryErrorTestCase.cs | 4 +- .../FixtureArgumentSetTestMethod.cs | 8 +- 9 files changed, 394 insertions(+), 34 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs index 607fe26cea..7339ee2e5d 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ArgumentSetOverride/ArgumentSetOverrideTests.cs @@ -67,6 +67,19 @@ public void OverridesTheSecondDimensionOnly() AssertFixtureMatchesDisplayName(); } + /// + /// Declares fewer dimensions than the class, by carrying an attribute type that takes fewer + /// of them. The dimensions the method says nothing about are the class's, exactly as if it + /// had named them and left them empty - dropping them instead would leave these tests + /// running without the values, and so without the traits, that a CI leg selects them by. + /// + [Fact] + [AssetArgumentSets(AssetDataStore.Cosmos)] + public void DeclaresFewerDimensionsThanTheClass() + { + AssertFixtureMatchesDisplayName(); + } + /// /// Ties the values the fixture was built from to the name the variant reports under. /// Asserting only that the fixture holds some valid value would hold for every variant diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs index 3e00443a86..cc75bd49d5 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ClassAttributeFault/ClassAttributeFaultTests.cs @@ -16,16 +16,17 @@ namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ClassAttributeF /// /// The class carries two different fixture argument set attributes, so asking it for the single /// one it declares throws before the walk over its methods begins. That is what makes this a - /// class-level fault: every method is lost at once, and the one failure reported has to stand in - /// for all of them. The methods declare different traits so that a failure carrying only the - /// first method's traits can be told apart from one carrying every method's. + /// class-level fault: every method is lost at once, so each one gets its own failure standing in + /// for it. The methods declare different traits so that failures carrying only their own + /// declarations can be told apart from failures carrying every method's pooled together. /// [AssetArgumentSets(AssetDataStore.Sql)] [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] public class ClassAttributeFaultTests { /// - /// Never runs, and is declared first, so it is the method the failure is anchored to. + /// Never runs. Being declared first, it is the method whose ordinary traits a pooled failure + /// would most plausibly have borrowed, which is what the assertions look for. /// [Fact] [Trait("Category", "DeclaredFirst")] diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs index 7f2e2702d4..91ac7c67f7 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MethodAttributeFault/MethodAttributeFaultTests.cs @@ -20,8 +20,8 @@ namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MethodAttribute /// rather than while using it, which is the step that used to run for every method of the class at /// once and so cost the whole class its tests. The class takes no fixture, so nothing can fail /// ahead of the reported case and the message it carries is the discoverer's own. One sibling - /// declares an argument set of its own, so the class still has values to pool into the fault - /// case's traits once the failing method's cannot be read. + /// declares an argument set of its own, and a different one, so a fault case that borrowed a + /// sibling's values instead of using the failing method's can be told apart from one that did not. /// public class MethodAttributeFaultTests { diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs index 635817fdd2..bdd826aae3 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ArgumentSetOverrideTests.cs @@ -39,6 +39,7 @@ public void GivenMethodsNarrowingOneDimension_WhenTheyAreDiscovered_ThenEachInhe [ScenarioClass + ".OverridesTheFirstDimensionOnly (Cosmos, Some)"] = "Passed", [ScenarioClass + ".OverridesTheSecondDimensionOnly (Sql, Some)"] = "Passed", [ScenarioClass + ".OverridesTheSecondDimensionOnly (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".DeclaresFewerDimensionsThanTheClass (Cosmos, Some)"] = "Passed", }); Assert.Equal(0, run.ExitCode); @@ -66,5 +67,32 @@ public void GivenMergedArgumentSets_WhenTestsAreSelectedByOneDimension_ThenOnlyT Assert.Equal(0, run.ExitCode); } + + /// + /// A method whose attribute takes fewer dimensions than its class's says nothing about the + /// rest, and so inherits them. Selecting by an inherited dimension's trait is the only thing + /// that proves it: a dimension that was dropped instead of inherited takes its trait with + /// it, so the test would still run and pass under an unfiltered run while being invisible to + /// the CI leg that selects on that trait - a green leg with the test silently absent. + /// + [Fact] + public void GivenAMethodDeclaringFewerDimensions_WhenTestsAreSelectedByAnInheritedDimension_ThenItStillRuns() + { + TestAssetRun run = TestAssetRunner.Run("ArgumentSetOverride", filterTrait: "AssetOtherDimension=Some"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".InheritsBothDimensions (Sql, Some)"] = "Passed", + [ScenarioClass + ".InheritsBothDimensions (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheFirstDimensionOnly (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Sql, Some)"] = "Passed", + [ScenarioClass + ".OverridesTheSecondDimensionOnly (Cosmos, Some)"] = "Passed", + [ScenarioClass + ".DeclaresFewerDimensionsThanTheClass (Cosmos, Some)"] = "Passed", + }); + + Assert.Equal(0, run.ExitCode); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs index 51c612c3bb..03d50d6d20 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs @@ -122,6 +122,84 @@ public void GivenAClassLevelFault_WhenAQuerySelectsACombinationNoMethodDeclared_ Assert.Empty(run.Results); } + /// + /// The form the repository's SQL and Cosmos integration legs use: everything except one data + /// store's tests. Excluding a value one method declares must not drop the failures standing in + /// for the methods that never declared it. + /// + /// + /// This is the exact hole a single per-class failure left. That failure would have carried + /// every method's values, Cosmos among them, so this filter would have dropped it and + /// the leg would have passed green with the entire class missing - the second method's Sql + /// variants included, which that leg was the only one to run. + /// + [Fact] + public void GivenAClassLevelFault_WhenALegExcludesOneDataStore_ThenTheOtherMethodsFailuresSurvive() + { + TestAssetRun run = TestAssetRunner.Run("ClassAttributeFault", filterNotTrait: "AssetDataStore=Cosmos"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstSqlCase] = "Failed", + [FirstSqlSomeCase] = "Failed", + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The same exclusion written as a query, which is how the E2E legs spell it. A case carrying + /// no value for the trait at all satisfies != as readily as one carrying a different + /// value, so this also pins that the failures carry their values rather than omitting them. + /// + [Fact] + public void GivenAClassLevelFault_WhenAQueryExcludesOneDataStore_ThenOnlyTheOtherFailuresAreReported() + { + TestAssetRun run = TestAssetRunner.Run( + "ClassAttributeFault", + filterQueryTraits: "(AssetDataStore!=Cosmos)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [FirstSqlCase] = "Failed", + [FirstSqlSomeCase] = "Failed", + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The shape the repository's E2E main leg uses: one argument set value selected positively + /// and a category excluded at the same time. It is the combination that matters - a failure + /// carrying the right data store but the excluded category is dropped, and one carrying + /// neither never matches - so this pins that each half is decided against the same case. + /// + [Fact] + public void GivenAClassLevelFault_WhenALegSelectsOneDataStoreAndExcludesACategory_ThenOnlyTheOtherMethodsFailuresAreReported() + { + TestAssetRun run = TestAssetRunner.Run( + "ClassAttributeFault", + filterQueryTraits: "(AssetDataStore=Sql)&(Category!=DeclaredFirst)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SecondSqlCase] = "Failed", + [SecondSqlSomeCase] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + /// /// Two overloads that both fail discovery are two lost methods, so both have to be reported. /// Identifying a failure by its method name alone gave them one identity between them, and diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs new file mode 100644 index 0000000000..fa82508cc0 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs @@ -0,0 +1,146 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Reflection; +using Xunit; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers how a class-level discovery failure chooses the methods it is reported against, and how + /// it tells two of them apart. + /// + /// + /// Both decisions have the same failure mode. A test method that is not recognised as one gets no + /// failure standing in for it, and two methods given the same identity get one failure between + /// them, xunit keeping whichever arrived first. Either way the run reports success with tests + /// missing and nothing in its output saying so, which is what this whole mechanism exists to + /// prevent. Both are asserted directly rather than through a scenario because arranging them in a + /// real test assembly needs an attribute xunit has no discoverer for. + /// + public class FaultMethodSelectionTests + { + /// + /// xunit discovers tests by the interface, not by the + /// class, so an attribute implementing the interface without + /// deriving from the class still marks a test. Recognising only the class would leave such a + /// method out of the methods a class-level failure is reported against. + /// + [Fact] + public void GivenAMethodMarkedByAnAttributeImplementingTheFactInterface_WhenItIsInspected_ThenItCountsAsATest() + { + MethodInfo method = typeof(Subject).GetMethod(nameof(Subject.MarkedByInterfaceOnly)); + + Assert.False(typeof(FactAttribute).IsAssignableFrom(typeof(InterfaceOnlyFactAttribute))); + Assert.True(CustomXunitTestFrameworkDiscoverer.IsTestMethod(method)); + } + + /// + /// A method carrying no fact attribute at all is not a test, so it is not one of the methods a + /// failure is reported against - a failure named after a helper carries none of the traits a + /// CI leg selects the class's tests by. + /// + [Fact] + public void GivenAMethodCarryingNoFactAttribute_WhenItIsInspected_ThenItDoesNotCountAsATest() + { + MethodInfo method = typeof(Subject).GetMethod(nameof(Subject.NotATest)); + + Assert.False(CustomXunitTestFrameworkDiscoverer.IsTestMethod(method)); + } + + /// + /// Generic overloads differing only in arity share a name and both take no parameters, so a key + /// built from the name and parameter types alone is the same for both. The two failures would + /// then share a unique ID and xunit would keep one, losing the other method silently. + /// + [Fact] + public void GivenGenericOverloadsDifferingOnlyInArity_WhenTheirFaultKeysAreBuilt_ThenTheKeysDiffer() + { + MethodInfo one = typeof(Subject).GetMethod(nameof(Subject.GenericOverload), 1, Type.EmptyTypes); + MethodInfo two = typeof(Subject).GetMethod(nameof(Subject.GenericOverload), 2, Type.EmptyTypes); + + Assert.NotEqual( + CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(one), + CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(two)); + } + + /// + /// Ordinary overloads differing only in parameter types must stay distinguishable too, so that + /// widening the key for generic arity did not narrow it anywhere else. + /// + [Fact] + public void GivenOverloadsDifferingOnlyInParameterTypes_WhenTheirFaultKeysAreBuilt_ThenTheKeysDiffer() + { + MethodInfo one = typeof(Subject).GetMethod(nameof(Subject.Overload), new[] { typeof(int) }); + MethodInfo two = typeof(Subject).GetMethod(nameof(Subject.Overload), new[] { typeof(string) }); + + Assert.NotEqual( + CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(one), + CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(two)); + } + + /// + /// An attribute that marks a test the way xunit recognises one - by implementing the interface - + /// without deriving from . + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + private sealed class InterfaceOnlyFactAttribute : Attribute, IFactAttribute + { + public string DisplayName => null; + + public bool Explicit => false; + + public string Skip => null; + + public Type[] SkipExceptions => null; + + public Type SkipType => null; + + public string SkipUnless => null; + + public string SkipWhen => null; + + public string SourceFilePath => null; + + public int? SourceLineNumber => null; + + public int Timeout => 0; + } + + /// + /// The methods the assertions above reflect over. None of them is discovered as a test: they + /// are inspected directly. + /// + private sealed class Subject + { + [InterfaceOnlyFact] + public void MarkedByInterfaceOnly() + { + } + + public void NotATest() + { + } + + public void GenericOverload() + { + } + + public void GenericOverload() + { + } + + public void Overload(int value) + { + } + + public void Overload(string value) + { + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index b3dcae7f51..14dcc2f1bb 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -69,11 +69,7 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl // not agree on theirs. A failure carrying every method's traits is dropped by a leg // excluding any one of them; a failure carrying only the first method's is invisible // to a leg selecting by another's. Either way a leg passes green with tests missing. - MethodInfo[] lostMethods = testClass.Methods.Where(IsTestMethod).ToArray(); - if (lostMethods.Length == 0) - { - lostMethods = testClass.Methods.Take(1).ToArray(); - } + MethodInfo[] lostMethods = TryReadMethodsToReportAgainst(testClass); if (lostMethods.Length == 0) { @@ -93,6 +89,41 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl } } + /// + /// Reads the methods a class's discovery failure should be reported against. + /// + /// + /// This runs inside the handler for a failure whose cause may be the class itself - a type + /// that cannot be loaded, or whose members cannot be reflected over. Reading its methods can + /// therefore throw the same way the discovery did, and an exception escaping a catch clause + /// replaces the original with one raised while handling it, losing the only description of + /// what actually went wrong. Returning nothing instead leaves the caller to rethrow the + /// original, which is the outcome when there is no method to report against anyway. + /// + /// Property accessors and the like are preferred over nothing, but only after the methods + /// xunit would treat as tests: a failure reported against a getter names nothing a reader can + /// act on, and carries none of the traits a CI leg selects the class's tests by. + /// + /// + /// The class whose discovery failed. + /// The methods to report against, empty if they could not be read. + private static MethodInfo[] TryReadMethodsToReportAgainst(IXunitTestClass testClass) + { + try + { + MethodInfo[] testMethods = testClass.Methods.Where(IsTestMethod).ToArray(); + + return testMethods.Length > 0 ? testMethods : testClass.Methods.Take(1).ToArray(); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] ERROR: the methods of '{testClass.TestClassName}' could not be read while reporting a discovery failure against them. {ex}"); + + return Array.Empty(); + } + } + /// /// Decides whether an exception thrown while expanding fixture argument sets goes back to /// xunit unchanged, or is reported as a failing test case standing in for the tests it lost. @@ -120,19 +151,27 @@ internal static bool ShouldRethrowRatherThanReport(Exception exception, bool isC /// Determines whether xunit would treat a method as a test. /// /// + /// The test is for , which is what xunit itself discovers by, and + /// not for . An attribute may implement the interface without + /// deriving from that class, and such a method is every bit as much a test: matching only the + /// class would leave it out of the methods a failure is reported against, so the tests it + /// stands for would be missing from a run that still reported success - the one outcome this + /// mechanism exists to prevent. Matching the interface can only widen the set. + /// /// This is asked while handling a discovery failure, over the very metadata that may have /// caused it, so a method whose attributes cannot be read is treated as not a test rather than /// allowed to throw. The caller then falls back to any method at all, which reports the failure /// against a less helpful name - still far better than letting the class vanish from a green /// run, which is what an escaping exception here would cause. + /// /// /// The method to inspect. /// true when the method carries a fact or theory attribute. - private static bool IsTestMethod(MethodInfo method) + internal static bool IsTestMethod(MethodInfo method) { try { - return method.GetCustomAttributes(typeof(FactAttribute), inherit: true).Length > 0; + return method.GetCustomAttributes(inherit: true).Any(attribute => attribute is IFactAttribute); } catch (Exception ex) { @@ -191,7 +230,7 @@ private async ValueTask ReportDiscoveryFault( Exception exception, string summary) { - Console.WriteLine($"[FixtureArgumentSets] ERROR: {summary} It was replaced by a failing test case. {exception}"); + Console.WriteLine($"[FixtureArgumentSets] ERROR: {summary} Each lost test was replaced by a failing test case. {exception}"); foreach (MethodInfo method in lostMethods) { @@ -215,7 +254,9 @@ private async ValueTask ReportDiscoveryFaultForMethod( // give both cases the same unique ID and xunit would keep only one. string methodKey = BuildFaultMethodKey(method); - foreach (IReadOnlyList combination in BuildFaultArgumentSetCombinations(ReadFaultArgumentSetDimensions(testClass, method))) + IReadOnlyList> combinations = BuildFaultArgumentSetCombinations(ReadFaultArgumentSetDimensions(testClass, method)); + + foreach (IReadOnlyList combination in combinations) { string displaySuffix = combination.Count == 0 ? string.Empty @@ -365,6 +406,24 @@ internal static IReadOnlyList> BuildFaultArgumentSetCo /// method alone, and a value borrowed from a sibling is a value no test of this method would /// have carried, which an exclusion filter can then use to drop the failure entirely. /// + /// + /// A method's own attributes are read alongside its class's rather than merged over them the + /// way the successful path merges them. The merge cannot be trusted here - it is one of the + /// things that may have just failed - and the two directions of error are not equal. Reading + /// both over-reports: a method narrowing a dimension is also reported under the class's wider + /// values, so a leg sees a failure for a variant that would not have existed. Merging and + /// getting it wrong under-reports, and a leg then passes green with tests missing. The + /// over-reported failure is loud and traceable to this same declaration; the under-reported + /// one is silent, so the union is deliberate. + /// + /// + /// What this cannot do is invent values it could not read. When an attribute itself cannot be + /// read or expanded, the failure carries no argument set trait, and a leg selecting positively + /// on one - as the E2E and export legs do, with (DataStore=SqlServer)&(...) - will + /// not select it. Such a leg sees no failure. The console output above still names the class, + /// and the legs that filter by exclusion still report it, but a positive filter cannot match a + /// trait that no longer exists and no arrangement of the remaining traits changes that. + /// /// /// The class whose expansion failed. /// The lost method the failure stands in for. @@ -482,29 +541,49 @@ void Add(string key, string value) /// /// The method's name alone is not unique: a class may overload a test method, and two failures /// sharing a unique ID leave xunit reporting only one of them - the other silently absent, - /// which is the outcome reporting these failures exists to prevent. The parameter types are - /// included to tell overloads apart, and are read defensively because this runs while handling - /// a failure that may itself be a type that cannot be loaded. + /// which is the outcome reporting these failures exists to prevent. The parameter types and + /// the generic arity are included to tell overloads apart - overloads may differ by arity + /// alone, as Run<T>() and Run<T, U>() do, and both take no parameters + /// - and are read defensively because this runs while handling a failure that may itself be a + /// type that cannot be loaded. /// /// The method the failure is reported against. /// A key identifying the method, falling back to its name alone. - private static string BuildFaultMethodKey(MethodInfo method) + internal static string BuildFaultMethodKey(MethodInfo method) { try { + string name = method.IsGenericMethodDefinition + ? $"{method.Name}`{method.GetGenericArguments().Length}" + : method.Name; + ParameterInfo[] parameters = method.GetParameters(); if (parameters.Length == 0) { - return method.Name; + return name; } - return $"{method.Name}({string.Join(",", parameters.Select(parameter => parameter.ParameterType.FullName ?? parameter.ParameterType.Name))})"; + return $"{name}({string.Join(",", parameters.Select(parameter => parameter.ParameterType.FullName ?? parameter.ParameterType.Name))})"; } catch (Exception ex) { Console.WriteLine( - $"[FixtureArgumentSets] WARNING: the signature of '{method.Name}' could not be read, so a failure reported against an overload of it may be dropped as a duplicate. {ex}"); - return method.Name; + $"[FixtureArgumentSets] WARNING: the signature of '{method.Name}' could not be read, so its failure is identified by metadata token instead. {ex}"); + + try + { + // The token distinguishes overloads within an assembly without reading any type + // the failure may have been caused by being unable to load. Falling back to the + // name alone would let two overloads share a unique ID, and xunit reports only + // one of a colliding pair - losing the very failure this is reporting. + return $"{method.Name}#{method.MetadataToken}"; + } + catch (Exception tokenException) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the metadata token of '{method.Name}' could not be read either, so a failure reported against an overload of it may be dropped as a duplicate. {tokenException}"); + return method.Name; + } } } @@ -623,24 +702,39 @@ private async ValueTask FindTestsForMethodCore( // get the method-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB) SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttribute); + // The merge runs over every dimension either side declares, not just the ones the + // method does. A method carrying an attribute with fewer dimensions than its class - + // which happens when the two carry different attribute types - would otherwise have + // the class's trailing dimensions dropped rather than inherited, and its tests would + // then run without the values, and so without the traits, that a CI leg selects them + // by: the leg would pass with those tests silently absent. + int dimensionCount = Math.Max(methodLevelOpenParameterSets.Length, classLevelOpenParameterSets.Length); + var mergedOpenParameterSets = new SingleFlag[dimensionCount][]; + bool hasOverride = false; - for (int i = 0; i < methodLevelOpenParameterSets.Length; i++) + for (int i = 0; i < dimensionCount; i++) { - if (methodLevelOpenParameterSets[i]?.Length > 0) + SingleFlag[] methodLevelDimension = i < methodLevelOpenParameterSets.Length ? methodLevelOpenParameterSets[i] : null; + + if (methodLevelDimension?.Length > 0) { hasOverride = true; + mergedOpenParameterSets[i] = methodLevelDimension; } else { - // means take the class-level set - methodLevelOpenParameterSets[i] = classLevelOpenParameterSets[i]; + // means take the class-level set. A method declaring a dimension the class + // does not have is a misconfiguration, and reading past the end here is what + // surfaces it: the throw is caught by the handler in FindTestsForType and + // reported as a failing test case rather than losing the method quietly. + mergedOpenParameterSets[i] = classLevelOpenParameterSets[i]; } } if (hasOverride) { // convert to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB) - closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray(); + closedSets = CartesianProduct(mergedOpenParameterSets).Select(e => e.ToArray()).ToArray(); } } diff --git a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs index e413805a4c..9e28278913 100644 --- a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs @@ -11,8 +11,8 @@ namespace Microsoft.Health.Extensions.Xunit { /// - /// A test case standing in for a class whose fixture argument sets could not be expanded, carrying - /// the traits the expansion never got far enough to produce. + /// A test case standing in for one test that a failure to expand fixture argument sets lost, + /// carrying the traits the expansion never got far enough to produce. /// /// /// A fixture argument set class is normally expanded into one variant per argument, and each diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index dbb2b1e900..37e94ba597 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -31,10 +31,10 @@ public FixtureArgumentSetTestMethod(FixtureArgumentSetTestClass testClass, Metho #pragma warning disable CS0618 // Called by the de-serializer; should only be called by deriving classes for de-serialization purposes public FixtureArgumentSetTestMethod() { - // The de-serializer does not run either of the field's assignments, and the argument set - // values are recovered from the base type's method arguments rather than from this field. - // Leaving it null would turn any later read into a NullReferenceException reported as a - // test failure with no bearing on the test, so it starts empty as it does on the class. + // The de-serializer bypasses the constructor that assigns this field, and the argument set + // values are recovered from the base type's method arguments rather than from it. Leaving + // it null would turn any later read into a NullReferenceException reported as a test + // failure with no bearing on the test, so it starts empty as it does on the class. _fixtureArguments = Array.Empty(); } #pragma warning restore CS0618 From a83b64466766f965195f7d455a605dbf46cf1e89 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 15:13:48 -0700 Subject: [PATCH 052/105] Keep a method whose test-ness cannot be read, and refuse a mismatched dimension A class-level discovery failure is reported against the methods it lost, so a method left out of that set has no failure standing in for it. IsTestMethod answered "not a test" for a method whose attributes threw while being read - very often the attributes that caused the failure being reported - so a real test could be dropped, and a readable sibling kept the fallback from firing. It now answers null for unknown, and the selection keeps tests and unknowns alike. The merge pairs class and method dimensions by position while the executor binds fixture arguments by enum type, and only convention keeps the two agreeing. A method declaring a different enum in a position is not narrowing the class's dimension but replacing it, and its variants would carry no value - and so no trait - for the dimension the class declared: a leg selecting on it would run none of that method's tests and still pass. That is now refused at discovery. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MismatchedDimensionTests.cs | 62 +++++++++++ ...therDimensionFirstArgumentSetsAttribute.cs | 30 ++++++ .../FaultMethodSelectionTests.cs | 65 ++++++++++++ .../MismatchedDimensionTests.cs | 87 +++++++++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 100 +++++++++++++----- 5 files changed, 320 insertions(+), 24 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/MismatchedDimensionTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/OtherDimensionFirstArgumentSetsAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/MismatchedDimensionTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/MismatchedDimensionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/MismatchedDimensionTests.cs new file mode 100644 index 0000000000..4d198048fa --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/MismatchedDimensionTests.cs @@ -0,0 +1,62 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MismatchedDimension +{ + /// + /// A class one of whose methods declares a different enum than the class does in the same + /// fixture argument set dimension. + /// + /// + /// The merge pairs dimensions by position while the executor binds fixture arguments by type, and + /// only convention keeps the two agreeing. Where they disagree the method's variants carry no + /// value for the dimension the class declared, and so none of the traits a CI leg selects by: the + /// leg would run none of this method's tests and still report success. The discoverer refuses the + /// declaration instead, so the mistake is a red test rather than a silent absence. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class MismatchedDimensionTests : IClassFixture + { + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public MismatchedDimensionTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// Declares where the class declares + /// , so discovery of this method fails and a failing test case + /// stands in for it. It never runs, which is why it can assert nothing. + /// + [Fact] + [OtherDimensionFirstArgumentSets(AssetOtherDimension.Some)] + public void NeverRuns() + { + throw new InvalidOperationException("This method must never be executed: its discovery is expected to fail."); + } + + /// + /// Declares nothing of its own, so it expands normally. A fault on one method must not cost + /// its siblings their variants. + /// + [Fact] + public void SiblingStillRuns() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.Contains($"({_fixture.DataStore})", displayName, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/OtherDimensionFirstArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/OtherDimensionFirstArgumentSetsAttribute.cs new file mode 100644 index 0000000000..b85e623027 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/MismatchedDimension/OtherDimensionFirstArgumentSetsAttribute.cs @@ -0,0 +1,30 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MismatchedDimension +{ + /// + /// Declares one fixture argument set dimension, taking a different enum than + /// takes in the same position. + /// + /// + /// A method carrying this under a class carrying that one is the declaration the discoverer has + /// to refuse: legal C#, reading as though it narrows the class's dimension, while in fact + /// replacing it with an unrelated one. + /// + public sealed class OtherDimensionFirstArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The first, and only, dimension. + public OtherDimensionFirstArgumentSetsAttribute(AssetOtherDimension first) + : base(first) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs index fa82508cc0..31d7ee1641 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/FaultMethodSelectionTests.cs @@ -83,6 +83,52 @@ public void GivenOverloadsDifferingOnlyInParameterTypes_WhenTheirFaultKeysAreBui CustomXunitTestFrameworkDiscoverer.BuildFaultMethodKey(two)); } + /// + /// A method whose attributes throw while being read is not known to be a test, and must not + /// be reported as known not to be one: the two answers lead to opposite decisions below. + /// + [Fact] + public void GivenAMethodWhoseAttributesCannotBeRead_WhenItIsInspected_ThenItsTestNessIsUnknown() + { + MethodInfo method = typeof(Subject).GetMethod(nameof(Subject.AttributesCannotBeRead)); + + Assert.Null(CustomXunitTestFrameworkDiscoverer.IsTestMethod(method)); + } + + /// + /// The attributes that cannot be read are very often the ones that caused the failure being + /// reported, so a method whose test-ness is unknown is kept. Dropping it would leave a real + /// test with no failure standing in for it, and a leg selecting by a trait only that method + /// carried would pass green with it absent - the presence of a readable sibling is exactly + /// what would hide it, because the fallback never fires. + /// + [Fact] + public void GivenAMethodWhoseAttributesCannotBeReadAlongsideAReadableTest_WhenTheMethodsAreSelected_ThenItIsKept() + { + MethodInfo unreadable = typeof(Subject).GetMethod(nameof(Subject.AttributesCannotBeRead)); + MethodInfo readable = typeof(Subject).GetMethod(nameof(Subject.MarkedByInterfaceOnly)); + + MethodInfo[] selected = CustomXunitTestFrameworkDiscoverer.SelectMethodsToReportAgainst(new[] { readable, unreadable }); + + Assert.Equal(new[] { readable, unreadable }, selected); + } + + /// + /// With nothing that is, or may be, a test there is still a failure to report, so the first + /// method the class declares stands in for it. Reporting nowhere would put the class back to + /// vanishing from a green run. + /// + [Fact] + public void GivenNoMethodIsATest_WhenTheMethodsAreSelected_ThenTheFirstMethodIsUsed() + { + MethodInfo first = typeof(Subject).GetMethod(nameof(Subject.NotATest)); + MethodInfo second = typeof(Subject).GetMethod(nameof(Subject.Overload), new[] { typeof(int) }); + + MethodInfo[] selected = CustomXunitTestFrameworkDiscoverer.SelectMethodsToReportAgainst(new[] { first, second }); + + Assert.Equal(new[] { first }, selected); + } + /// /// An attribute that marks a test the way xunit recognises one - by implementing the interface - /// without deriving from . @@ -111,6 +157,19 @@ private sealed class InterfaceOnlyFactAttribute : Attribute, IFactAttribute public int Timeout => 0; } + /// + /// An attribute whose construction throws, standing in for one whose type cannot be loaded - + /// reading the attributes of a method carrying it throws rather than answering. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + private sealed class UnreadableAttribute : Attribute + { + public UnreadableAttribute() + { + throw new InvalidOperationException("This attribute cannot be constructed."); + } + } + /// /// The methods the assertions above reflect over. None of them is discovered as a test: they /// are inspected directly. @@ -126,6 +185,12 @@ public void NotATest() { } + [InterfaceOnlyFact] + [Unreadable] + public void AttributesCannotBeRead() + { + } + public void GenericOverload() { } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/MismatchedDimensionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MismatchedDimensionTests.cs new file mode 100644 index 0000000000..0ab947d300 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/MismatchedDimensionTests.cs @@ -0,0 +1,87 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a method whose argument set attribute names a different enum than its class's does in + /// the same dimension. + /// + /// + /// The merge pairs dimensions by position while the executor binds fixture arguments by enum type, + /// and nothing but convention keeps the two agreeing. Where they disagree the method's variants + /// carry no value for the dimension the class declared, and so none of the traits the SQL and + /// Cosmos legs select by: both legs would run none of this method's tests and both would still + /// report success. The discoverer therefore refuses the declaration and reports a failure, which + /// is loud, in place of an expansion that would have been silent. + /// + public class MismatchedDimensionTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.MismatchedDimension.MismatchedDimensionTests"; + + /// + /// The refusal is reported as a failure standing in for the offending method, under each of + /// the class's combinations, while the method that declared nothing unusual runs normally. + /// + [Fact] + public void GivenAMethodDeclaringADifferentDimensionThanItsClass_WhenItIsDiscovered_ThenItIsReportedAndItsSiblingStillRuns() + { + TestAssetRun run = TestAssetRunner.Run("MismatchedDimension"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Cosmos)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Some)"] = "Failed", + [ScenarioClass + ".SiblingStillRuns (Sql)"] = "Passed", + [ScenarioClass + ".SiblingStillRuns (Cosmos)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failure names both enums and the position they disagree in, because the declaration + /// that caused it is legal C# and reads as though it should work. + /// + [Fact] + public void GivenAMethodDeclaringADifferentDimensionThanItsClass_WhenItIsDiscovered_ThenTheFailureNamesBothDimensions() + { + TestAssetRun run = TestAssetRunner.Run("MismatchedDimension"); + + Assert.Contains("AssetOtherDimension", run.Output, StringComparison.Ordinal); + Assert.Contains("AssetDataStore", run.Output, StringComparison.Ordinal); + Assert.Contains("argument set dimension 0", run.Output, StringComparison.Ordinal); + } + + /// + /// The SQL leg's own filter shape. The refused method must still be reported to the leg that + /// would have run it - a failure the leg cannot see is no better than the silent expansion the + /// refusal replaced. + /// + [Fact] + public void GivenAMethodDeclaringADifferentDimensionThanItsClass_WhenALegExcludesTheOtherDataStore_ThenTheFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("MismatchedDimension", filterNotTrait: "AssetDataStore=Cosmos"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Some)"] = "Failed", + [ScenarioClass + ".SiblingStillRuns (Sql)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 14dcc2f1bb..2d25cf9f06 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -99,11 +99,6 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl /// replaces the original with one raised while handling it, losing the only description of /// what actually went wrong. Returning nothing instead leaves the caller to rethrow the /// original, which is the outcome when there is no method to report against anyway. - /// - /// Property accessors and the like are preferred over nothing, but only after the methods - /// xunit would treat as tests: a failure reported against a getter names nothing a reader can - /// act on, and carries none of the traits a CI leg selects the class's tests by. - /// /// /// The class whose discovery failed. /// The methods to report against, empty if they could not be read. @@ -111,9 +106,7 @@ private static MethodInfo[] TryReadMethodsToReportAgainst(IXunitTestClass testCl { try { - MethodInfo[] testMethods = testClass.Methods.Where(IsTestMethod).ToArray(); - - return testMethods.Length > 0 ? testMethods : testClass.Methods.Take(1).ToArray(); + return SelectMethodsToReportAgainst(testClass.Methods); } catch (Exception ex) { @@ -159,15 +152,19 @@ internal static bool ShouldRethrowRatherThanReport(Exception exception, bool isC /// mechanism exists to prevent. Matching the interface can only widen the set. /// /// This is asked while handling a discovery failure, over the very metadata that may have - /// caused it, so a method whose attributes cannot be read is treated as not a test rather than - /// allowed to throw. The caller then falls back to any method at all, which reports the failure - /// against a less helpful name - still far better than letting the class vanish from a green - /// run, which is what an escaping exception here would cause. + /// caused it, so a method whose attributes cannot be read answers that its test-ness is + /// unknown rather than throwing, and rather than claiming it is not a test. Those are not the + /// same answer: the attributes that cannot be read are very often the ones that caused the + /// failure being reported, and answering "not a test" there would drop the method from the + /// set a failure stands in for. /// /// /// The method to inspect. - /// true when the method carries a fact or theory attribute. - internal static bool IsTestMethod(MethodInfo method) + /// + /// true when the method carries a fact or theory attribute, false when it + /// certainly does not, and null when its attributes could not be read. + /// + internal static bool? IsTestMethod(MethodInfo method) { try { @@ -176,11 +173,38 @@ internal static bool IsTestMethod(MethodInfo method) catch (Exception ex) { Console.WriteLine( - $"[FixtureArgumentSets] WARNING: the attributes of '{method.Name}' could not be read while choosing a method to report a discovery failure against. {ex}"); - return false; + $"[FixtureArgumentSets] WARNING: the attributes of '{method.Name}' could not be read while choosing the methods to report a discovery failure against, so it is treated as a test. {ex}"); + return null; } } + /// + /// Chooses the methods a class's discovery failure is reported against. + /// + /// + /// A method whose test-ness could not be determined is kept rather than dropped. The two + /// mistakes are not equal: reporting a failure against a helper is noise a reader can see and + /// dismiss, while dropping a method that really was a test means no failure stands in for it, + /// and a leg selecting by a trait only that method carried passes green with it absent. + /// + /// Only when no method is a test, or may be one, does this fall back to whatever the class + /// declares first, so that a failure is still reported somewhere rather than not at all. A + /// failure reported against a property accessor names nothing a reader can act on, which is + /// why it is the last resort rather than part of the set. + /// + /// + /// The methods the class declares. + /// The methods to report against. + internal static MethodInfo[] SelectMethodsToReportAgainst(IEnumerable methods) + { + EnsureArg.IsNotNull(methods, nameof(methods)); + + MethodInfo[] all = methods as MethodInfo[] ?? methods.ToArray(); + MethodInfo[] testMethods = all.Where(method => IsTestMethod(method) != false).ToArray(); + + return testMethods.Length > 0 ? testMethods : all.Take(1).ToArray(); + } + /// /// Reports a discovery failure as a test case that fails when the run reaches it. /// @@ -718,15 +742,42 @@ private async ValueTask FindTestsForMethodCore( if (methodLevelDimension?.Length > 0) { + SingleFlag[] classLevelDimension = i < classLevelOpenParameterSets.Length ? classLevelOpenParameterSets[i] : null; + if (classLevelDimension?.Length > 0) + { + Type methodLevelType = methodLevelDimension[0].EnumValue?.GetType(); + Type classLevelType = classLevelDimension[0].EnumValue?.GetType(); + + if (methodLevelType != classLevelType) + { + // Dimensions are matched by position, but the fixture's constructor + // arguments are matched by enum type. A method whose attribute takes a + // different enum in this position is therefore not overriding the + // class's dimension but replacing it with an unrelated one, and the + // variants would carry no value - and so no trait - for the dimension + // the class declared. A leg selecting on it would then run none of + // this method's tests and still pass. Only the convention that a class + // and its methods use argument set attributes taking the same enums in + // the same order keeps that from happening, so it is checked rather + // than assumed. + throw new InvalidOperationException( + $"'{testClass.TestClassName}.{method.Name}' declares '{methodLevelType}' where its class declares '{classLevelType}' in argument set dimension {i}. " + + "A method-level fixture argument set attribute must take the same enums, in the same order, as the class-level one it overrides."); + } + } + hasOverride = true; mergedOpenParameterSets[i] = methodLevelDimension; } else { - // means take the class-level set. A method declaring a dimension the class - // does not have is a misconfiguration, and reading past the end here is what - // surfaces it: the throw is caught by the handler in FindTestsForType and - // reported as a failing test case rather than losing the method quietly. + // Take the class-level set: this position is either one the method left empty + // or one it says nothing about at all. Reading past the end of the class's + // dimensions throws, which happens for a method that declares more dimensions + // than its class and leaves the extra one empty - it is asking to inherit a + // dimension that does not exist. That throw is caught by the per-method + // handler in FindTestsForTypeCore and reported as a failing test case + // standing in for this method, rather than losing the method quietly. mergedOpenParameterSets[i] = classLevelOpenParameterSets[i]; } } @@ -743,10 +794,11 @@ private async ValueTask FindTestsForMethodCore( // Reaching here means an argument set was declared, so a product of nothing is a // misconfiguration -- an argument set of zero, a value naming no single flag, or a // null entry -- and every test on this method would otherwise be absent from a run - // that still reported success. Throwing hands it to the handler in FindTestsForType, - // which reports it as a failing test case so it shows up in the results and in the - // exit code. Discovery over every test assembly in this repository produces no such - // case, so this only fires on a class that is genuinely misdeclared. + // that still reported success. Throwing hands it to the per-method handler in + // FindTestsForTypeCore, which reports it as a failing test case so it shows up in + // the results and in the exit code. Discovery over every test assembly in this + // repository produces no such case, so this only fires on a class that is genuinely + // misdeclared. throw new InvalidOperationException( $"'{testClass.TestClassName}.{method.Name}' expanded to no fixture argument sets, so none of its tests would run. " + "Check that every argument set names at least one single-valued flag of a [Flags] enum."); From 85119c8c929a3e2c4fe6bfee8d363f35d39c4a97 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 15:26:15 -0700 Subject: [PATCH 053/105] Put the variant's traits on the error case xunit builds for a broken test A test xunit cannot run - a fact declaring parameters, a theory with no data - is reported as an ExecutionErrorTestCase, which is constructed with no traits at all. Every CI leg here selects by trait, and a positive filter cannot match a trait that is absent, so the export and E2E legs saw none of those failures: the malformed test was simply missing and the leg reported success. Discovery now merges the variant method's traits onto every case it hands back, which leaves a healthy case unchanged and makes a broken one visible to the leg that owns it. Verified against the scenario: selecting AssetDataStore=Sql over a class with a malformed fact exited 0 with the failure absent before the change, and exits 2 with it present after. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../VariantTraits/VariantTraitsTests.cs | 66 ++++++++++++++ .../VariantTraitsTests.cs | 85 +++++++++++++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 68 +++++++++++++-- .../DiscoveryErrorTestCase.cs | 9 +- 4 files changed, 216 insertions(+), 12 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs new file mode 100644 index 0000000000..01c8ab9139 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs @@ -0,0 +1,66 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.VariantTraits +{ + /// + /// A class whose variants must carry the argument set's trait onto every test case discovered + /// from them, including the one xunit builds for a test it cannot run. + /// + /// + /// Every CI leg in this repository selects by trait, so a test case reaching the runner without + /// the data store trait is one no leg selecting a data store can see. That is harmless for a + /// passing test and silent for a broken one: the leg runs neither, and reports success either + /// way. Both a malformed test and a healthy one carrying a trait of its own are declared here, + /// because the two reach the runner by different routes. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class VariantTraitsTests : IClassFixture + { + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public VariantTraitsTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// Declaring a parameter on a fact is a test xunit cannot run, so it reports the method as an + /// error test case rather than discovering it normally. That case is built without traits, + /// which is what makes it worth declaring here: the analyser rule against writing it is + /// suppressed because the malformed declaration is the point. + /// + /// Never supplied: the declaration is deliberately malformed. +#pragma warning disable xUnit1001 // Fact methods cannot have parameters + [Fact] + public void MalformedFactIsStillReportedToAFilteringLeg(int value) +#pragma warning restore xUnit1001 + { + throw new InvalidOperationException("This method can never run: xunit reports it as an error instead."); + } + + /// + /// Carries a trait of its own alongside the injected one, in the shape the export and E2E legs + /// select by - a data store and a category together. A variant that kept only the injected + /// trait would drop this one and vanish from those legs. + /// + [Trait("Category", "ExportLongRunning")] + [Fact] + public void HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.Contains($"({_fixture.DataStore})", displayName, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs new file mode 100644 index 0000000000..8620260a22 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs @@ -0,0 +1,85 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers the traits a fixture argument set variant puts on the test cases discovered from it. + /// + /// + /// Every CI leg selects by trait, so these decide what a leg runs. A test case that reaches the + /// runner without the injected data store trait cannot be selected by a leg naming one, and a + /// variant that kept only the injected trait would lose the category the export and E2E legs + /// name alongside it. Either way the leg reports success with tests it was meant to run absent, + /// and nothing in its output says so. + /// + public class VariantTraitsTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.VariantTraits.VariantTraitsTests"; + + /// + /// Unfiltered, both tests are present in both variants and the malformed one fails. + /// + [Fact] + public void GivenAClassWithAMalformedTest_WhenItIsRunUnfiltered_ThenEveryVariantOfBothTestsIsReported() + { + TestAssetRun run = TestAssetRunner.Run("VariantTraits"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Cosmos)"] = "Failed", + [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", + [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Cosmos)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// xunit builds an error test case for a test it cannot run, and builds it with no traits at + /// all. A leg naming a data store would then select neither variant of the malformed test and + /// report success, so the traits of the variant it was discovered from have to be put back. + /// + [Fact] + public void GivenAMalformedTest_WhenALegSelectsOneDataStore_ThenItsFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("VariantTraits", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The compound shape the export and E2E legs select by. A test runs only if it carries every + /// named trait, so injecting the data store trait must add to the test's own rather than + /// replace them. + /// + [Fact] + public void GivenATestCarryingItsOwnTrait_WhenALegSelectsOnBothTraits_ThenItIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("VariantTraits", filterQueryTraits: "(AssetDataStore=Sql)&(Category=ExportLongRunning)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 2d25cf9f06..66c21db18d 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -215,15 +215,16 @@ internal static MethodInfo[] SelectMethodsToReportAgainst(IEnumerableA sentence saying what was lost, used to open the failure message. /// Whether discovery should continue, as the callback reported it. /// - /// The console line written here is the authoritative record of the fault. The test case puts - /// the failure in the results and in the exit code, but it cannot be relied on to carry the - /// cause: a case travels through the ordinary class runner, which builds the class's fixtures - /// first, and a class using fixture argument sets typically has a fixture taking an argument - /// the failed expansion never produced. That fixture failure is aggregated ahead of this - /// message and is what the report then shows. Both ways out of that were tried and are worse - - /// anchoring the case to a class of its own takes it out of the namespace and class filters - /// that would have selected the tests it replaces, and discarding the aggregated failure - /// discards this message with it, leaving the fault unreported again. + /// The console line written here is the record of the fault that does not depend on anything + /// downstream. The test case is what puts the failure in the results and in the exit code, and + /// it does normally carry the cause - see the last paragraph - but only because it is anchored + /// to the variant class whose fixture can still be built. A case travels through the ordinary + /// class runner, which builds the class's fixtures first, so anchoring it anywhere the + /// fixture's argument is unavailable would aggregate a fixture failure ahead of this message + /// and report that instead. Both ways out of that were tried and are worse - anchoring the case + /// to a class of its own takes it out of the namespace and class filters that would have + /// selected the tests it replaces, and discarding the aggregated failure discards this message + /// with it, leaving the fault unreported again. /// /// Every failure reported here stands for exactly one thing that would have run: one method, /// under one combination of argument set values. Nothing is merged, because a case carrying @@ -829,9 +830,11 @@ private async ValueTask FindTestsForMethodCore( // variant of a method would otherwise be reported under an identical name and // a failure could not be attributed to a specific fixture argument set. var variantArguments = closedVariant; + var variantMethod = closedVariantTestMethod; Func> variantCallback = testCase => { ApplyVariantDisplayName(testCase, variantArguments); + ApplyVariantTraits(testCase, variantMethod); return tracker.Invoke(testCase); }; @@ -887,6 +890,53 @@ private static void ApplyVariantDisplayName(ITestCase testCase, SingleFlag[] clo TestCaseDisplayNameField.SetValue(xunitTestCase, displayName + suffix); } + /// + /// Copies the variant's traits onto a discovered test case that did not get them. + /// + /// + /// A test case built from a method normally carries that method's traits, so for a healthy + /// test this adds nothing. The case that matters is the one xunit builds for a test it cannot + /// run - a fact declaring parameters, a theory with no data - which it reports as an + /// ExecutionErrorTestCase constructed with no traits at all. Without them that failure + /// is invisible to any leg selecting on a trait: this repository's E2E and export legs each + /// require a positive DataStore, and a filter cannot match a trait that is absent. The + /// malformed test would then be missing from those legs and they would report success, which + /// is precisely the outcome the rest of this class exists to prevent. + /// + /// Traits are merged rather than replaced so that anything the discovery already attached - + /// a theory row's own traits, say - survives. + /// + /// + /// The discovered test case. + /// The variant method the test case was discovered from. + private static void ApplyVariantTraits(ITestCase testCase, FixtureArgumentSetTestMethod variantMethod) + { + if (testCase is not XunitTestCase xunitTestCase) + { + // ApplyVariantDisplayName has already reported the type, so this only adds what else + // is lost by it: nothing here can reach a case type this class does not know how to + // write to, and a case with no traits is one a filtering leg cannot see. + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the fixture argument set traits of '{testCase.TestCaseDisplayName}' could NOT be applied, " + + "so a CI leg selecting on them will not run it."); + return; + } + + foreach (KeyValuePair> trait in variantMethod.Traits) + { + if (!xunitTestCase.Traits.TryGetValue(trait.Key, out HashSet values)) + { + values = new HashSet(StringComparer.OrdinalIgnoreCase); + xunitTestCase.Traits[trait.Key] = values; + } + + foreach (string value in trait.Value) + { + values.Add(value); + } + } + } + /// /// Builds a key that distinguishes one fixture argument set from another within a collection. /// diff --git a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs index 9e28278913..ed137c30fc 100644 --- a/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/DiscoveryErrorTestCase.cs @@ -25,9 +25,12 @@ namespace Microsoft.Health.Extensions.Xunit /// filter matches a case when any value under the named trait matches, and an exclusion /// filter is the negation of that, so a single case declaring both DataStore=CosmosDb and /// DataStore=SqlServer would be excluded by both --filter-not-trait - /// DataStore=SqlServer and --filter-not-trait DataStore=CosmosDb - that is, by every - /// leg this repository runs. The discoverer therefore reports one of these per combination, so - /// each leg selects exactly the case that stands for the tests it lost. + /// DataStore=SqlServer and --filter-not-trait DataStore=CosmosDb - that is, by both of + /// the integration legs this repository runs, which select by exclusion alone. The discoverer + /// therefore reports one of these per combination, so each leg selects exactly the case that + /// stands for the tests it lost. The export and E2E legs instead require a positive + /// DataStore, and a case can only reach them by carrying that leg's value, which is the + /// same reason the combinations are reported separately rather than pooled. /// /// internal sealed class DiscoveryErrorTestCase : ExecutionErrorTestCase From b753090332dcc28915b10e2927abd66488a5d9d6 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 16:11:06 -0700 Subject: [PATCH 054/105] Say what a misread assertion type actually costs a retrying test A failure type whose name merely contains "Xunit" or "Assert" is treated as an assertion failure, and with RetryOnAssertionFailure at its default the test is not retried at all. The comment claimed it cost a single attempt. Also pin the fault case's own trait under a compound positive filter, and collapse a nested condition in the flag enumeration. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Scenarios/VariantTraits/VariantTraitsTests.cs | 1 + .../VariantTraitsTests.cs | 8 ++++++-- .../CustomXunitTestFrameworkDiscoverer.cs | 7 ++----- src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs | 6 +++++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs index 01c8ab9139..b8b71edd54 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs @@ -41,6 +41,7 @@ public VariantTraitsTests(AssetFixture fixture) /// suppressed because the malformed declaration is the point. /// /// Never supplied: the declaration is deliberately malformed. + [Trait("Category", "ExportLongRunning")] #pragma warning disable xUnit1001 // Fact methods cannot have parameters [Fact] public void MalformedFactIsStillReportedToAFilteringLeg(int value) diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs index 8620260a22..4d51f79e9a 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs @@ -67,10 +67,11 @@ public void GivenAMalformedTest_WhenALegSelectsOneDataStore_ThenItsFailureIsStil /// /// The compound shape the export and E2E legs select by. A test runs only if it carries every /// named trait, so injecting the data store trait must add to the test's own rather than - /// replace them. + /// replace them - for the error case xunit built as much as for the healthy one, since the + /// error case gets every trait it has this way and would otherwise carry only the injected one. /// [Fact] - public void GivenATestCarryingItsOwnTrait_WhenALegSelectsOnBothTraits_ThenItIsStillSelected() + public void GivenTestsCarryingTheirOwnTrait_WhenALegSelectsOnBothTraits_ThenBothAreStillSelected() { TestAssetRun run = TestAssetRunner.Run("VariantTraits", filterQueryTraits: "(AssetDataStore=Sql)&(Category=ExportLongRunning)"); @@ -78,8 +79,11 @@ public void GivenATestCarryingItsOwnTrait_WhenALegSelectsOnBothTraits_ThenItIsSt run, new Dictionary { + [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", }); + + Assert.NotEqual(0, run.ExitCode); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 66c21db18d..16997941ab 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -989,12 +989,9 @@ IEnumerable GetSingleValuedFlags(Enum e) foreach (Enum value in Enum.GetValues(e.GetType())) { var flagAsLong = Convert.ToInt64(value); - if (IsPowerOfTwo(flagAsLong)) + if (IsPowerOfTwo(flagAsLong) && (enumAsLong & flagAsLong) != 0) { - if ((enumAsLong & flagAsLong) != 0) - { - yield return new SingleFlag(value); - } + yield return new SingleFlag(value); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index eaf22e491e..8291910bfe 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -584,7 +584,11 @@ public bool QueueMessage(IMessageSinkMessage message) // check: xUnit's own assertion exceptions live under Xunit.Sdk, but assertion // libraries used alongside it do not, and matching "Assert" anywhere in the type // name catches those too. The trade is that an unrelated type whose name happens to - // contain either word is classified the same way, which only costs an attempt. + // contain either word is classified the same way, and with RetryOnAssertionFailure + // left at its default of false that costs the test its remaining attempts rather + // than spending one: an assertion failure is taken to be deterministic, so it is + // not retried. The failure is still reported red either way, so the cost is a + // flaky test going unretried, never a failure going unseen. IsAssertionFailure = failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 && (failed.ExceptionTypes[0].Contains("Xunit", StringComparison.Ordinal) || From 34b355a45a518563c123b087c70a6773c6d25328 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 16:23:44 -0700 Subject: [PATCH 055/105] Name the attribute that is wrong instead of an array index A method declaring more argument set dimensions than its class and naming no value in the extra one was reported with "Index was outside the bounds of the array", which says nothing about the attribute the developer has to fix. Also cover the other malformed shape that reaches xunit's error case. A theory with no data comes from a different discoverer than a fact with parameters, and only the latter was pinned as still reaching a leg that filters on traits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../DiscoveryFault/DiscoveryFaultTests.cs | 6 ++--- .../VariantTraits/VariantTraitsTests.cs | 17 ++++++++++++++ .../DiscoveryFaultTests.cs | 4 ++-- .../VariantTraitsTests.cs | 4 ++++ .../CustomXunitTestFrameworkDiscoverer.cs | 23 ++++++++++++++----- 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs index 2b334de698..81bd51274a 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DiscoveryFault/DiscoveryFaultTests.cs @@ -14,9 +14,9 @@ namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault /// /// The class declares one dimension and the method below declares two, the second of which /// names no flag. A dimension that names no flag means "use the class-level dimension in this - /// position", and the class has no second position, so the expansion reads past the end of the - /// class-level dimensions. The misconfiguration itself is not the point: it is simply the - /// cheapest way to make discovery of a real class throw, which is what this scenario is for. + /// position", and the class has no second position, so the expansion has nothing to inherit and + /// refuses. The misconfiguration itself is not the point: it is simply the cheapest way to make + /// discovery of a real class throw, which is what this scenario is for. /// [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] public class DiscoveryFaultTests : IClassFixture diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs index b8b71edd54..6906a295de 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/VariantTraits/VariantTraitsTests.cs @@ -50,6 +50,23 @@ public void MalformedFactIsStillReportedToAFilteringLeg(int value) throw new InvalidOperationException("This method can never run: xunit reports it as an error instead."); } + /// + /// A theory with no data rows is the other way to reach xunit's error case, and it reaches it + /// from a different discoverer than the malformed fact above. Both are declared because a fix + /// that covered only one would leave the other invisible to a filtering leg. + /// + /// Never supplied: no data row is declared. + [Trait("Category", "ExportLongRunning")] +#pragma warning disable xUnit1003 // Theory methods must have test data +#pragma warning disable xUnit1026 // Theory method does not use parameter + [Theory] + public void MalformedTheoryIsStillReportedToAFilteringLeg(int value) +#pragma warning restore xUnit1026 +#pragma warning restore xUnit1003 + { + throw new InvalidOperationException("This method can never run: xunit reports it as an error instead."); + } + /// /// Carries a trait of its own alongside the injected one, in the shape the export and E2E legs /// select by - a data store and a category together. A variant that kept only the injected diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs index 7cf25f5fc0..e15949ed1e 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs @@ -53,7 +53,7 @@ public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheCauseIsWri TestAssetRun run = TestAssetRunner.Run("DiscoveryFault"); Assert.Contains("[FixtureArgumentSets] ERROR", run.Output, StringComparison.Ordinal); - Assert.Contains("IndexOutOfRangeException", run.Output, StringComparison.Ordinal); + Assert.Contains("fixture argument set dimensions where its class declares", run.Output, StringComparison.Ordinal); } /// @@ -74,7 +74,7 @@ public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheReportedFa { string reported = ExtractReportedFailure(run.Output, caseName); - Assert.Contains("IndexOutOfRangeException", reported, StringComparison.Ordinal); + Assert.Contains("fixture argument set dimensions where its class declares", reported, StringComparison.Ordinal); Assert.DoesNotContain("had multiple values", reported, StringComparison.Ordinal); } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs index 4d51f79e9a..be8bb1b32b 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/VariantTraitsTests.cs @@ -35,7 +35,9 @@ public void GivenAClassWithAMalformedTest_WhenItIsRunUnfiltered_ThenEveryVariant new Dictionary { [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".MalformedTheoryIsStillReportedToAFilteringLeg (Sql)"] = "Failed", [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Cosmos)"] = "Failed", + [ScenarioClass + ".MalformedTheoryIsStillReportedToAFilteringLeg (Cosmos)"] = "Failed", [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Cosmos)"] = "Passed", }); @@ -58,6 +60,7 @@ public void GivenAMalformedTest_WhenALegSelectsOneDataStore_ThenItsFailureIsStil new Dictionary { [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".MalformedTheoryIsStillReportedToAFilteringLeg (Sql)"] = "Failed", [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", }); @@ -80,6 +83,7 @@ public void GivenTestsCarryingTheirOwnTrait_WhenALegSelectsOnBothTraits_ThenBoth new Dictionary { [ScenarioClass + ".MalformedFactIsStillReportedToAFilteringLeg (Sql)"] = "Failed", + [ScenarioClass + ".MalformedTheoryIsStillReportedToAFilteringLeg (Sql)"] = "Failed", [ScenarioClass + ".HealthyTestKeepsBothItsOwnTraitAndTheInjectedOne (Sql)"] = "Passed", }); diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 16997941ab..2a256b90a0 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -773,12 +773,23 @@ private async ValueTask FindTestsForMethodCore( else { // Take the class-level set: this position is either one the method left empty - // or one it says nothing about at all. Reading past the end of the class's - // dimensions throws, which happens for a method that declares more dimensions - // than its class and leaves the extra one empty - it is asking to inherit a - // dimension that does not exist. That throw is caught by the per-method - // handler in FindTestsForTypeCore and reported as a failing test case - // standing in for this method, rather than losing the method quietly. + // or one it says nothing about at all. A method that declares more dimensions + // than its class and leaves the extra one empty is asking to inherit a + // dimension that does not exist, so there is nothing to take. Reading past the + // end would throw too, but with a message naming an array index rather than the + // attribute that is wrong, and this exception is what the developer is shown as + // the body of the failing test standing in for the method. + if (i >= classLevelOpenParameterSets.Length) + { + throw new InvalidOperationException( + $"'{testClass.TestClassName}.{method.Name}' declares {methodLevelOpenParameterSets.Length} fixture argument set dimensions where its class declares {classLevelOpenParameterSets.Length}, and names no value in dimension {i}. " + + "A dimension naming no value inherits the class-level one in that position, and the class has no dimension there. " + + "Either name the values that dimension should take, or drop it so the method declares no more dimensions than its class."); + } + + // That throw is caught by the per-method handler in FindTestsForTypeCore and + // reported as a failing test case standing in for this method, rather than + // losing the method quietly. mergedOpenParameterSets[i] = classLevelOpenParameterSets[i]; } } From 5d4ebd9d3b6977806bca2e4232208cf78899bb9f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 16:32:52 -0700 Subject: [PATCH 056/105] Stop a collection definition from hiding the tests of classes that join it xunit v3 puts a collection's traits on the tests of every class in the collection, which v2 did not. Two classes each defined the IndexAndReindex collection, and the one that won carried Category=ReindexOperation, so all twenty-four CustomSearchParamTests cases inherited it - a category the Stu3 and R4 Cosmos E2E legs exclude. Those legs selected none of its twelve Cosmos cases and reported success, with nothing saying the tests had been dropped. The collection now has one definition of its own, carrying no traits, and both classes join it. Categories stay on the classes that mean them. Measured on the built R4 E2E assembly: the Cosmos E2E leg's own filter selected 0 of these cases before and selects 12 after, and ReindexTests stays excluded. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../JoinsTheCollectionTests.cs | 25 ++++++++ .../StaysOutOfTheCollectionTests.cs | 25 ++++++++ .../TraitCarryingCollectionDefinition.cs | 24 +++++++ .../CollectionTraitsTests.cs | 64 +++++++++++++++++++ .../Rest/IndexAndReindexCollection.cs | 27 ++++++++ .../Rest/Reindex/ReindexTests.cs | 2 +- .../Rest/Search/CustomSearchParamTests.cs | 1 - 7 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/JoinsTheCollectionTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/StaysOutOfTheCollectionTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/TraitCarryingCollectionDefinition.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs create mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/JoinsTheCollectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/JoinsTheCollectionTests.cs new file mode 100644 index 0000000000..00036f2b21 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/JoinsTheCollectionTests.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits +{ + /// + /// Joins a trait-carrying collection without declaring that trait itself. + /// + [Collection("TraitCarryingCollection")] + public class JoinsTheCollectionTests + { + /// + /// Present or absent under a filter depending on whether the collection's trait reached it. + /// + [Fact] + public void InheritsWhateverTheCollectionCarries() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/StaysOutOfTheCollectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/StaysOutOfTheCollectionTests.cs new file mode 100644 index 0000000000..a9600736c6 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/StaysOutOfTheCollectionTests.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits +{ + /// + /// Stays out of the collection, so a filter that removes the collection's members can be told + /// apart from one that selected nothing at all. + /// + public class StaysOutOfTheCollectionTests + { + /// + /// Selected by any filter that does not name a trait this test carries. + /// + [Fact] + public void CarriesNoCollectionTrait() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/TraitCarryingCollectionDefinition.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/TraitCarryingCollectionDefinition.cs new file mode 100644 index 0000000000..055f4b6364 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/TraitCarryingCollectionDefinition.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits +{ + /// + /// Defines a collection that carries a trait of its own. + /// + /// + /// Whether a collection's traits land on the tests of every class in it decides what several CI + /// legs run. Those legs exclude a category, and a class sharing a collection with an excluded + /// one would then be excluded too - silently, because an exclusion filter reports success for + /// the tests it did not select. This scenario is what says which of the two xunit does. + /// + [CollectionDefinition("TraitCarryingCollection")] + [Trait("Category", "CollectionOwned")] + public class TraitCarryingCollectionDefinition + { + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs new file mode 100644 index 0000000000..22b31cbf8d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs @@ -0,0 +1,64 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers whether a collection's traits reach the tests of the classes in it. + /// + /// + /// Several CI legs in this repository select by excluding a category, so a trait arriving on a + /// test from somewhere its own class never declared decides whether those legs run it. A class + /// that joins a collection to be serialised against another one would, if collection traits + /// propagate, also inherit that collection's categories and disappear from every leg excluding + /// them - and an exclusion filter reports success for what it did not select, so nothing would + /// say so. This pins the behaviour the test projects are written against. + /// + public class CollectionTraitsTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits.JoinsTheCollectionTests"; + private const string OutsiderClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits.StaysOutOfTheCollectionTests"; + + /// + /// Unfiltered, the test in the collection runs. + /// + [Fact] + public void GivenAClassInATraitCarryingCollection_WhenItIsRunUnfiltered_ThenItIsReported() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraits"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".InheritsWhateverTheCollectionCarries"] = "Passed", + [OutsiderClass + ".CarriesNoCollectionTrait"] = "Passed", + }); + } + + /// + /// The shape that matters: a leg excluding the collection's category. xunit v3 puts a + /// collection's traits on the tests of every class in it, so the member disappears even + /// though its own class never declared that category, while the class outside the collection + /// stays. Nothing in the leg's output would say the member was dropped, which is why any + /// class joining a collection has to be read as carrying that collection's categories too. + /// + [Fact] + public void GivenAClassInATraitCarryingCollection_WhenALegExcludesThatCategory_ThenOnlyTheCollectionsMemberIsDropped() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraits", filterNotTrait: "Category=CollectionOwned"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [OutsiderClass + ".CarriesNoCollectionTrait"] = "Passed", + }); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs new file mode 100644 index 0000000000..539328fada --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Tests.E2E.Rest +{ + /// + /// Serialises the test classes that add, reindex, or delete search parameters, which share the + /// server's search parameter registry and cannot run alongside each other. + /// + /// + /// This class deliberately carries no traits. xunit v3 puts a collection's traits on the tests of + /// every class in the collection, so a category declared here would be inherited by every member + /// - including members that only joined to be serialised. CI legs that exclude a category would + /// then skip those members as well, and an exclusion filter reports success for what it did not + /// select, so nothing in the leg's output would say the tests were dropped. Categories therefore + /// belong on the individual test classes, never here. + /// + [CollectionDefinition(Categories.IndexAndReindex, DisableParallelization = true)] + public class IndexAndReindexCollection + { + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index b888d9cc10..eca946d53b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -34,7 +34,7 @@ namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Reindex { - [CollectionDefinition(Categories.IndexAndReindex, DisableParallelization = true)] + [Collection(Categories.IndexAndReindex)] [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.IndexAndReindex)] [Trait(Traits.Category, Categories.ReindexOperation)] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs index e3296c866c..f149749322 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/CustomSearchParamTests.cs @@ -24,7 +24,6 @@ namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Search { - [CollectionDefinition(Categories.IndexAndReindex, DisableParallelization = true)] [Collection(Categories.IndexAndReindex)] [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Search)] From 6b7d5552fcb9fdc2f210505b9079137c7cba9552 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 17:14:02 -0700 Subject: [PATCH 057/105] Pin a variant of a class that also joins a trait-carrying collection This is the shape the shipping suite uses wherever a class both selects a data store and joins a collection, and the two behaviours were only covered apart: one scenario had a collection but no variants, the other variants but no collection. A change letting either trait replace the other would have kept both passing while hiding every variant from the legs that name both. Also stop a fixture docstring citing ResourceIdProvider as an AsyncLocal holder. It holds a plain field, and this branch already deletes that claim where it was load-bearing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../VariantsInTheCollectionTests.cs | 48 +++++++++++++++++++ .../AsyncLocalWritingFixture.cs | 4 +- .../CollectionTraitsTests.cs | 23 +++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/VariantsInTheCollectionTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/VariantsInTheCollectionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/VariantsInTheCollectionTests.cs new file mode 100644 index 0000000000..56ed088b13 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraits/VariantsInTheCollectionTests.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits +{ + /// + /// Joins the trait-carrying collection and is also expanded into one variant per data store, the + /// shape the shipping suite uses wherever a class both selects a data store and joins a + /// collection to be serialised. + /// + /// + /// Each variant has to end up carrying both the trait the collection owns and the data store + /// trait the expansion injects. Losing either one hides the variant from a leg naming it, and + /// the legs that name both are the ones that select positively, so nothing would report the loss. + /// + [Collection("TraitCarryingCollection")] + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class VariantsInTheCollectionTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public VariantsInTheCollectionTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Selected only by a leg naming both the collection's trait and this variant's data store. + /// + [Fact] + public void CarriesBothTheCollectionTraitAndItsDataStore() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs index dbe1d2baf8..7318d18874 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ExecutionContextFlow/AsyncLocalWritingFixture.cs @@ -7,8 +7,8 @@ namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ExecutionContex { /// /// A class fixture that writes to an from its - /// constructor, the way FhirStorageTestsFixture builds its ResourceIdProvider - /// there rather than in InitializeAsync. + /// constructor, the way FhirRequestContextAccessor holds its context, so that a fixture + /// built before the test runs decides what the test can read back. /// public sealed class AsyncLocalWritingFixture { diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs index 22b31cbf8d..f972f43e8d 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitsTests.cs @@ -23,6 +23,7 @@ public class CollectionTraitsTests { private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits.JoinsTheCollectionTests"; private const string OutsiderClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits.StaysOutOfTheCollectionTests"; + private const string VariantClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraits.VariantsInTheCollectionTests"; /// /// Unfiltered, the test in the collection runs. @@ -38,6 +39,8 @@ public void GivenAClassInATraitCarryingCollection_WhenItIsRunUnfiltered_ThenItIs { [ScenarioClass + ".InheritsWhateverTheCollectionCarries"] = "Passed", [OutsiderClass + ".CarriesNoCollectionTrait"] = "Passed", + [VariantClass + ".CarriesBothTheCollectionTraitAndItsDataStore (Sql)"] = "Passed", + [VariantClass + ".CarriesBothTheCollectionTraitAndItsDataStore (Cosmos)"] = "Passed", }); } @@ -60,5 +63,25 @@ public void GivenAClassInATraitCarryingCollection_WhenALegExcludesThatCategory_T [OutsiderClass + ".CarriesNoCollectionTrait"] = "Passed", }); } + + /// + /// The shape the shipping suite actually uses: a class that both joins a trait-carrying + /// collection and is expanded into one variant per data store. A leg selecting positively on + /// the collection's category and one data store must still find that variant, which it can + /// only do if the injected data store trait is added to the collection's rather than either + /// one replacing the other. + /// + [Fact] + public void GivenVariantsInATraitCarryingCollection_WhenALegSelectsOnBothTraits_ThenTheMatchingVariantIsFound() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraits", filterQueryTraits: "(AssetDataStore=Sql)&(Category=CollectionOwned)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [VariantClass + ".CarriesBothTheCollectionTraitAndItsDataStore (Sql)"] = "Passed", + }); + } } } From 810332a94e555064b1f8e2f2731f364b526acadf Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 17:55:49 -0700 Subject: [PATCH 058/105] Actually compile the collection definition the E2E classes join The shared E2E project lists every file it compiles, so a new file that is not listed is silently not built. The definition added with the trait fix was in that state: both classes had dropped their own [CollectionDefinition], so with nothing declaring it the collection existed only implicitly and the DisableParallelization both classes asked for was quietly lost. Rebuilt and re-measured on the R4 E2E assembly with the file genuinely compiled: CustomSearchParamTests is 24 cases, 0 of them ReindexOperation, and the Cosmos E2E leg selects its 12 while ReindexTests stays excluded. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- UnitTest1.cs | 1 + .../Microsoft.Health.Fhir.Shared.Tests.E2E.projitems | 1 + .../Rest/IndexAndReindexCollection.cs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 UnitTest1.cs diff --git a/UnitTest1.cs b/UnitTest1.cs new file mode 100644 index 0000000000..8dd87f71b0 --- /dev/null +++ b/UnitTest1.cs @@ -0,0 +1 @@ +using Xunit; namespace TestProj { public class UnitTest1 { [Fact] [Trait("DataStore", "CosmosDb")] public void Test1() {} } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems index ecc4406467..4d13e103dc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems @@ -82,6 +82,7 @@ + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs index 539328fada..c1cd350123 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/IndexAndReindexCollection.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using Microsoft.Health.Test.Utilities; +using Microsoft.Health.Fhir.Tests.Common; using Xunit; namespace Microsoft.Health.Fhir.Tests.E2E.Rest From 64a92361b64b3ed9b340736b37865c12aacc54b5 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 17:55:59 -0700 Subject: [PATCH 059/105] Remove a probe file a review agent left in the tree Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- UnitTest1.cs | 1 - 1 file changed, 1 deletion(-) delete mode 100644 UnitTest1.cs diff --git a/UnitTest1.cs b/UnitTest1.cs deleted file mode 100644 index 8dd87f71b0..0000000000 --- a/UnitTest1.cs +++ /dev/null @@ -1 +0,0 @@ -using Xunit; namespace TestProj { public class UnitTest1 { [Fact] [Trait("DataStore", "CosmosDb")] public void Test1() {} } } From 079705d5b84dfe85b2c219a61848a8526c4d751a Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 18:51:16 -0700 Subject: [PATCH 060/105] Read argument set values from metadata when the attribute will not construct An attribute whose constructor throws is one of the ways a class fails to expand, and reading it again to find out what it declared runs the same constructor. The failure standing in for the lost tests was therefore left bare, and the export and E2E legs select positively on a data store, so they matched nothing and reported success with a whole class missing. The values are in the assembly metadata, which can be read without running anything. The fault path now falls back to that. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../ThrowingArgumentSetsAttribute.cs | 33 ++++ .../ThrowingArgumentSetsTests.cs | 45 +++++ .../ThrowingArgumentSetsTests.cs | 78 +++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 157 +++++++++++++++--- 4 files changed, 287 insertions(+), 26 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingArgumentSetsTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsAttribute.cs new file mode 100644 index 0000000000..274ddc5a06 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsAttribute.cs @@ -0,0 +1,33 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingArgumentSets +{ + /// + /// An argument set attribute whose constructor throws. + /// + /// + /// Reading an attribute runs its constructor, so a class carrying this one cannot be expanded and + /// cannot have its declared values read back the ordinary way either. The values are still in the + /// assembly's metadata, and the failure standing in for the lost tests has to be given them, or a + /// leg selecting positively on a data store would match nothing and report success. + /// + public sealed class ThrowingArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class, which + /// never completes. + /// + /// The data stores the class would have been expanded over. + public ThrowingArgumentSetsAttribute(AssetDataStore dataStore) + : base(dataStore) + { + throw new InvalidOperationException("This argument set attribute cannot be constructed."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsTests.cs new file mode 100644 index 0000000000..898e088d80 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingArgumentSets/ThrowingArgumentSetsTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingArgumentSets +{ + /// + /// A class whose argument set attribute cannot be constructed, so none of its tests can be built. + /// + /// + /// The failure standing in for the lost tests still has to carry the data store trait the tests + /// would have carried, which can only come from metadata here. A leg selecting positively on a + /// data store, as the export and E2E legs do, would otherwise see nothing at all and pass. + /// + [ThrowingArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class ThrowingArgumentSetsTests : IClassFixture + { + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public ThrowingArgumentSetsTests(AssetFixture fixture) + { + Fixture = fixture; + } + + /// + /// Gets the variant fixture this class was constructed with. + /// + protected AssetFixture Fixture { get; } + + /// + /// Never runs: the class carries an argument set attribute that cannot be constructed. + /// + [Fact] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingArgumentSetsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingArgumentSetsTests.cs new file mode 100644 index 0000000000..04c2de3080 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingArgumentSetsTests.cs @@ -0,0 +1,78 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a class whose argument set attribute cannot be constructed. + /// + /// + /// Reading an attribute runs its constructor, so this is a class the discoverer cannot expand and + /// cannot interrogate the ordinary way either. The failure standing in for its lost tests still + /// has to carry the data store trait those tests would have had: the export and E2E legs select + /// positively on one, and a filter cannot match a trait that is not there, so those legs would + /// run nothing and report success with a whole class missing. + /// + public class ThrowingArgumentSetsTests + { + private const string SqlErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingArgumentSets.ThrowingArgumentSetsTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string CosmosErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingArgumentSets.ThrowingArgumentSetsTests.NeverRuns (fixture argument set discovery: Cosmos)"; + + /// + /// Unfiltered, one failure stands in for the lost tests of each declared data store. + /// + [Fact] + public void GivenAnAttributeThatCannotBeConstructed_WhenItIsDiscovered_ThenTheFaultIsReportedForEachDataStore() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingArgumentSets"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SqlErrorCaseName] = "Failed", + [CosmosErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The shape that matters. A leg naming one data store has to see the failure for that store, + /// which it can only do if the values were taken from metadata rather than from the attribute + /// the discoverer was unable to construct. + /// + [Fact] + public void GivenAnAttributeThatCannotBeConstructed_WhenALegSelectsOneDataStore_ThenThatLegStillSeesTheFailure() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingArgumentSets", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [SqlErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The cause has to survive into the reported failure, or whoever reads the results is told a + /// test failed without being told that the class never expanded. + /// + [Fact] + public void GivenAnAttributeThatCannotBeConstructed_WhenItIsDiscovered_ThenTheCauseIsWrittenToTheOutput() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingArgumentSets"); + + Assert.Contains("This argument set attribute cannot be constructed.", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 2a256b90a0..c9a39b09a4 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -442,12 +442,19 @@ internal static IReadOnlyList> BuildFaultArgumentSetCo /// one is silent, so the union is deliberate. /// /// - /// What this cannot do is invent values it could not read. When an attribute itself cannot be - /// read or expanded, the failure carries no argument set trait, and a leg selecting positively - /// on one - as the E2E and export legs do, with (DataStore=SqlServer)&(...) - will - /// not select it. Such a leg sees no failure. The console output above still names the class, - /// and the legs that filter by exclusion still report it, but a positive filter cannot match a - /// trait that no longer exists and no arrangement of the remaining traits changes that. + /// Reading an attribute runs its constructor, so an attribute whose constructor throws is both + /// a cause of a fault and, ordinarily, unreadable afterwards. The values it was given are still + /// in the assembly's metadata, which can be read without running anything, so that is the + /// fallback: the failure still carries the traits the lost tests would have had. This matters + /// because the E2E and export legs select positively on an argument set, with + /// (DataStore=SqlServer)&(...), and a positive filter cannot match a trait that is + /// not there - such a leg would match nothing and report success with a class missing. + /// + /// + /// What remains beyond reach is a declaration whose metadata cannot be read either, or whose + /// argument sets are not enum constants in the metadata at all. The failure then carries no + /// argument set trait and only the legs that filter by exclusion report it; the console output + /// above still names the class either way. /// /// /// The class whose expansion failed. @@ -457,12 +464,18 @@ private static List ReadFaultArgumentSetDimensions(IXunitTestCla { var dimensions = new List(); - AddAll(() => testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false), testClass.TestClassName); - AddAll(() => method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false), $"{testClass.TestClassName}.{method.Name}"); + AddAll( + () => testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false), + () => testClass.Class.GetCustomAttributesData(), + testClass.TestClassName); + AddAll( + () => method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute), inherit: false), + () => method.GetCustomAttributesData(), + $"{testClass.TestClassName}.{method.Name}"); return dimensions; - void AddAll(Func read, string owner) + void AddAll(Func read, Func> readMetadata, string owner) { object[] attributes; try @@ -471,8 +484,16 @@ void AddAll(Func read, string owner) } catch (Exception ex) { + // Reading an attribute runs its constructor, and a constructor that throws is one + // of the ways a class gets here in the first place. The values are still in the + // assembly's metadata, which can be read without running anything, so the failure + // standing in for these tests can still carry the traits a leg selects on. Without + // this the failure would reach the runner bare, and the export and E2E legs select + // positively on a data store, so they would match nothing and report success. Console.WriteLine( - $"[FixtureArgumentSets] WARNING: the argument set attributes of '{owner}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + $"[FixtureArgumentSets] WARNING: the argument set attributes of '{owner}' could not be read, so their values are being taken from metadata instead. {ex}"); + + AddAllFromMetadata(readMetadata, owner); return; } @@ -492,6 +513,85 @@ void AddAll(Func read, string owner) } } } + + void AddAllFromMetadata(Func> readMetadata, string owner) + { + IEnumerable attributeData; + try + { + attributeData = readMetadata() + .Where(d => typeof(FixtureArgumentSetsAttribute).IsAssignableFrom(d.AttributeType)) + .ToArray(); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the argument set metadata of '{owner}' could not be read either, so a trait filter may not select the failure standing in for its tests. {ex}"); + return; + } + + foreach (CustomAttributeData data in attributeData) + { + try + { + dimensions.Add(ExpandEnumFlagsFromMetadata(data)); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: an argument set attribute of '{owner}' could not be expanded from metadata, so a trait filter may not select the failure standing in for its tests. {ex}"); + } + } + } + } + + /// + /// Expands the argument sets an attribute declares by reading the assembly's metadata, without + /// constructing the attribute. + /// + /// + /// This is the fault path's way round an attribute whose constructor throws. It reads the same + /// enum values the constructor would have been handed, so the dimensions it produces are the + /// ones the tests would have carried had the class expanded. + /// + /// The metadata of one argument set attribute. + /// One array of single-valued flags per declared dimension. + private static SingleFlag[][] ExpandEnumFlagsFromMetadata(CustomAttributeData attributeData) + { + var dimensions = new List(); + + foreach (CustomAttributeTypedArgument argument in attributeData.ConstructorArguments) + { + // A derived attribute names its enums one per parameter, while the base takes them as + // a params array, so an argument is either an enum or a collection of them. + if (argument.Value is IReadOnlyCollection elements) + { + foreach (CustomAttributeTypedArgument element in elements) + { + AddDimension(element); + } + } + else + { + AddDimension(argument); + } + } + + return dimensions.ToArray(); + + void AddDimension(CustomAttributeTypedArgument argument) + { + Type argumentType = argument.ArgumentType; + + // Metadata stores an enum as its underlying value, so the declared type is what says + // which enum it was. Anything that is not an enum is not a dimension. + if (argumentType == null || !argumentType.IsEnum || argument.Value == null) + { + return; + } + + dimensions.Add(GetSingleValuedFlags((Enum)Enum.ToObject(argumentType, argument.Value)).ToArray()); + } } /// @@ -982,34 +1082,39 @@ private static string BuildVariantClassKey(string variantKey, Type testClass) } private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(FixtureArgumentSetsAttribute attribute) + { + return attribute.GetArgumentSets() + .Select(e => GetSingleValuedFlags(e).ToArray()) + .ToArray(); + } + + /// + /// Splits a flags enum value into the single-valued flags it names. + /// + /// The value to split, which may be null. + /// One per flag set in the value. + private static IEnumerable GetSingleValuedFlags(Enum e) { bool IsPowerOfTwo(long x) { return (x != 0) && ((x & (x - 1)) == 0); } - IEnumerable GetSingleValuedFlags(Enum e) + if (e is null) { - if (e is null) - { - yield break; - } + yield break; + } - var enumAsLong = Convert.ToInt64(e); + var enumAsLong = Convert.ToInt64(e); - foreach (Enum value in Enum.GetValues(e.GetType())) + foreach (Enum value in Enum.GetValues(e.GetType())) + { + var flagAsLong = Convert.ToInt64(value); + if (IsPowerOfTwo(flagAsLong) && (enumAsLong & flagAsLong) != 0) { - var flagAsLong = Convert.ToInt64(value); - if (IsPowerOfTwo(flagAsLong) && (enumAsLong & flagAsLong) != 0) - { - yield return new SingleFlag(value); - } + yield return new SingleFlag(value); } } - - return attribute.GetArgumentSets() - .Select(e => GetSingleValuedFlags(e).ToArray()) - .ToArray(); } /// From 5001ae3d780622d024c818effc69d43819c9a8b0 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 19:22:40 -0700 Subject: [PATCH 061/105] Pin the trait on a method expanded without a class-level declaration A class that declares no argument sets of its own but has a method that does takes a path of its own through discovery, and that path was only ever asserted unfiltered. The export and E2E legs select positively on the argument set trait, so a variant that lost it would be unreachable to them and those legs would report green having run nothing of the sort. Also flattens a nested ternary and corrects what the attribute's own documentation says a dimension is. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../PassthroughDiscoveryTests.cs | 24 +++++++++++++++++++ .../FixtureArgumentSetsAttribute.cs | 7 +++--- .../RetryTestCase.cs | 17 +++++++++---- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs index 70323b19fa..eb1ea37c05 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PassthroughDiscoveryTests.cs @@ -41,5 +41,29 @@ public void GivenAClassWithArgumentSetsOnOnlySomeMethods_WhenTheRunCompletes_The [ClassName + "PassthroughTheory(value: 3)"] = "Passed", }); } + + /// + /// The expanded method's variants have to carry the argument set trait even though the class + /// declares no argument sets of its own, because the export and E2E legs select positively on + /// that trait. A variant without it is not merely mislabelled: it is unreachable to those legs, + /// which then run nothing of the sort and report green. + /// + /// + /// The methods on the passthrough path carry no argument set trait, which is correct - they + /// were never expanded and there is no value to name - so this also pins that a leg naming one + /// data store gets the one variant and nothing else. + /// + [Fact] + public void GivenAClassWithArgumentSetsOnOnlySomeMethods_WhenALegSelectsOneDataStore_ThenOnlyThatVariantIsSelected() + { + TestAssetRun run = TestAssetRunner.Run("MixedFixtureAttributes", filterTrait: "AssetDataStore=Sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "ExpandedMethod (Sql)"] = "Passed", + }); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs index a3414e5efb..853f4af7d9 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetsAttribute.cs @@ -22,9 +22,10 @@ public abstract class FixtureArgumentSetsAttribute : Attribute /// Initializes a new instance of the class. /// /// - /// The flags enum values to pass to the class fixture's constructor. Each distinct enum type - /// contributes one argument, and the test class is expanded into one variant per combination - /// of the flags set within each type. + /// The flags enum values to pass to the class fixture's constructor. Each value passed becomes + /// one dimension, and the test class is expanded into one variant per combination of the flags + /// set across the dimensions. By convention each dimension uses a distinct flags enum type, so + /// that a fixture constructor can tell its parameters apart. /// protected FixtureArgumentSetsAttribute(params Enum[] argumentSets) { diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 8291910bfe..5fa7ca71c5 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -569,11 +569,18 @@ public bool QueueMessage(IMessageSinkMessage message) if (message is ITestFailed failed) { // Capture failure details for diagnostics - LastFailureMessage = failed.Messages != null && failed.Messages.Length > 0 - ? string.Join(Environment.NewLine, failed.Messages) - : failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 - ? string.Join(", ", failed.ExceptionTypes) - : "Unknown failure"; + if (failed.Messages != null && failed.Messages.Length > 0) + { + LastFailureMessage = string.Join(Environment.NewLine, failed.Messages); + } + else if (failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0) + { + LastFailureMessage = string.Join(", ", failed.ExceptionTypes); + } + else + { + LastFailureMessage = "Unknown failure"; + } LastFailureStackTrace = failed.StackTraces != null && failed.StackTraces.Length > 0 ? string.Join(Environment.NewLine, failed.StackTraces) From 9f7ba5581823701bbb9357fa4c661c3b035464f1 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 19:44:33 -0700 Subject: [PATCH 062/105] Cover retries and fixture argument set variants together The suite exercised retries and variants separately, though production combines them: an E2E test is routinely both [RetryFact] and expanded per data store, and it is the combination that has to keep its traits. Adds a scenario that is both, including a flaky retry that recovers per variant and two malformed theory shapes, and asserts the expanded output and two trait filters over it. Also aligns the traits copied by the retry discoverers with the case-insensitive comparer used everywhere else in the harness. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryVariants/RetryVariantTests.cs | 117 ++++++++++++++++++ .../RetryVariantTests.cs | 115 +++++++++++++++++ .../RetryFactDiscoverer.cs | 13 +- .../RetryTheoryDiscoverer.cs | 5 +- 4 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryVariants/RetryVariantTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryVariantTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryVariants/RetryVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryVariants/RetryVariantTests.cs new file mode 100644 index 0000000000..99c421c7b3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryVariants/RetryVariantTests.cs @@ -0,0 +1,117 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Concurrent; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryVariants +{ + /// + /// A class that is both expanded per data store and uses the retrying test attributes, which is + /// the shape most of this repository's integration tests take. + /// + /// + /// Variant expansion and retrying are separate mechanisms that meet here: expansion rewrites the + /// test method and its traits, while the retry discoverers build their own test case type and + /// copy the traits across. Either one can drop what the other added, and the result is a variant + /// that no leg selecting positively on a data store can see - so it never runs and the leg still + /// reports success. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + [Trait("Category", "RetryVariant")] + public class RetryVariantTests : IClassFixture + { + private static readonly ConcurrentDictionary Attempts = new ConcurrentDictionary(); + + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public RetryVariantTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// A retrying fact that passes, so that its variants and traits can be asserted on. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void PassingRetryFact() + { + Assert.NotEqual(default, _fixture.DataStore); + } + + /// + /// A retrying fact that fails once and then passes, proving retries still happen inside an + /// expanded variant and that each variant retries on its own count. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] + public void FlakyRetryFact() + { + int attempt = Attempts.AddOrUpdate(_fixture.DataStore.ToString(), 1, (_, current) => current + 1); + + if (attempt == 1) + { + Assert.Fail($"ASSET: {_fixture.DataStore} failing on attempt {attempt}"); + } + } + + /// + /// A retrying theory, so that a row of a theory is expanded per data store as well. + /// + /// Identifies the row. + [RetryTheory(MaxRetries = 2, DelayBetweenRetriesMs = 10)] + [InlineData(1)] + [InlineData(2)] + public void RetryTheoryRow(int value) + { + Assert.InRange(value, 1, 2); + } + +#pragma warning disable xUnit1003 // Theory methods must have test data +#pragma warning disable xUnit1026 // Theory methods should use all of their parameters + + /// + /// A retrying theory declaring no data, which xunit reports as an error case rather than a + /// test. It has to reach a leg selecting positively on a data store, carrying the reason it + /// could not run. + /// + /// Never supplied: the theory declares no data. + [RetryTheory] + public void MalformedRetryTheory(int value) + { + Assert.True(true); + } + +#pragma warning restore xUnit1026 +#pragma warning restore xUnit1003 + + /// + /// Supplies no rows at all, so that the theory below reaches xunit's error case through the + /// data attribute rather than through having no data attribute. + /// + /// An empty sequence. + public static TheoryData NoRows() + { + return new TheoryData(); + } + + /// + /// A retrying theory whose data attribute yields no rows, which is the other way xunit ends + /// up reporting a theory it cannot run. It has to reach a leg selecting positively on a data + /// store, carrying the reason it could not run. + /// + /// Never supplied: the data attribute yields no rows. + [RetryTheory] + [MemberData(nameof(NoRows))] + public void EmptyDataRetryTheory(int value) + { + Assert.InRange(value, 1, 2); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryVariantTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryVariantTests.cs new file mode 100644 index 0000000000..718b64528e --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryVariantTests.cs @@ -0,0 +1,115 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a class that is both expanded per data store and uses the retrying test attributes, + /// which is the shape most of this repository's integration tests take. + /// + /// + /// Expansion and retrying were only ever asserted apart. They meet in the test case the retry + /// discoverers build, which is a different type from the one expansion writes to and copies the + /// traits across by hand. A copy that lost or renamed a trait would leave a variant that no leg + /// selecting positively on a data store could see, and such a leg reports success having run + /// nothing. + /// + public class RetryVariantTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryVariants.RetryVariantTests."; + + /// + /// Every retrying test is expanded per data store, theory rows included, and the flaky one + /// still recovers on a later attempt within each variant. + /// + [Fact] + public void GivenRetryingTestsInAnExpandedClass_WhenTheRunCompletes_ThenEveryVariantIsReported() + { + TestAssetRun run = TestAssetRunner.Run("RetryVariants"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "PassingRetryFact (Sql)"] = "Passed", + [ClassName + "PassingRetryFact (Cosmos)"] = "Passed", + [ClassName + "FlakyRetryFact (Sql)"] = "Passed", + [ClassName + "FlakyRetryFact (Cosmos)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 1) (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 2) (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 1) (Cosmos)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 2) (Cosmos)"] = "Passed", + [ClassName + "MalformedRetryTheory (Sql)"] = "Failed", + [ClassName + "EmptyDataRetryTheory (Sql)"] = "Failed", + [ClassName + "MalformedRetryTheory (Cosmos)"] = "Failed", + [ClassName + "EmptyDataRetryTheory (Cosmos)"] = "Failed", + }); + } + + /// + /// The shape the export and E2E legs use. A retrying test has to be selectable by the same + /// compound positive filter as any other, on both the data store the expansion gave it and + /// the category its class declares. + /// + [Fact] + public void GivenRetryingTestsInAnExpandedClass_WhenALegSelectsOnBothTraits_ThenOnlyThatDataStoreRuns() + { + TestAssetRun run = TestAssetRunner.Run( + "RetryVariants", + filterQueryTraits: "(AssetDataStore=Sql)&(Category=RetryVariant)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "PassingRetryFact (Sql)"] = "Passed", + [ClassName + "FlakyRetryFact (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 1) (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 2) (Sql)"] = "Passed", + [ClassName + "MalformedRetryTheory (Sql)"] = "Failed", + [ClassName + "EmptyDataRetryTheory (Sql)"] = "Failed", + }); + } + + /// + /// Trait filtering compares names without regard to case, and a retrying test that is also + /// expanded goes through two separate pieces of trait handling to reach the runner. This + /// pins that it still answers to the same filter either way. + /// + [Fact] + public void GivenRetryingTestsInAnExpandedClass_WhenALegNamesTheTraitInAnotherCase_ThenTheSameTestsRun() + { + TestAssetRun run = TestAssetRunner.Run("RetryVariants", filterTrait: "assetdatastore=sql"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "PassingRetryFact (Sql)"] = "Passed", + [ClassName + "FlakyRetryFact (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 1) (Sql)"] = "Passed", + [ClassName + "RetryTheoryRow(value: 2) (Sql)"] = "Passed", + [ClassName + "MalformedRetryTheory (Sql)"] = "Failed", + [ClassName + "EmptyDataRetryTheory (Sql)"] = "Failed", + }); + } + + /// + /// A theory that declares no data is not a test that failed, it is a test that could not be + /// built, and the reason has to reach the results rather than being replaced by whatever + /// happens when the retrying case calls the method with no arguments. + /// + [Fact] + public void GivenAMalformedRetryingTheory_WhenTheRunCompletes_ThenTheReportedFailureNamesTheMissingData() + { + TestAssetRun run = TestAssetRunner.Run("RetryVariants"); + + Assert.Contains("No data found for", run.Output, System.StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs index bf6a411b2c..f3e800d049 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactDiscoverer.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -41,6 +42,16 @@ public ValueTask> Discover( timeout: null, baseDisplayName: null); + // xunit builds a test case's traits with an OrdinalIgnoreCase comparer, and the argument + // set traits are merged into them on the same terms, so the copy is made on those terms + // too. Filtering does not depend on it - the runner compares trait names itself - but a + // copy that compared differently from the dictionary it came from would let the same + // trait be held twice under two spellings. + var traits = testMethod.Traits.ToDictionary( + kvp => kvp.Key, + kvp => new HashSet(kvp.Value, StringComparer.OrdinalIgnoreCase), + StringComparer.OrdinalIgnoreCase); + var testCase = new RetryTestCase( details.ResolvedTestMethod, details.TestCaseDisplayName, @@ -51,7 +62,7 @@ public ValueTask> Discover( skipType: details.SkipType, skipUnless: details.SkipUnless, skipWhen: details.SkipWhen, - traits: testMethod.Traits.ToDictionary(kvp => kvp.Key, kvp => new HashSet(kvp.Value)), + traits: traits, testMethodArguments: null, sourceFile: details.SourceFilePath, sourceLine: details.SourceLineNumber, diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index dc92808114..2cca82f446 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -89,7 +89,10 @@ private static IXunitTestCase WrapTestCase(IXunitTestCase testCase, RetryTheoryA xunitTestCase.SkipType, xunitTestCase.SkipUnless, xunitTestCase.SkipWhen, - xunitTestCase.Traits.ToDictionary(kvp => kvp.Key, kvp => new HashSet(kvp.Value)), + xunitTestCase.Traits.ToDictionary( + kvp => kvp.Key, + kvp => new HashSet(kvp.Value, StringComparer.OrdinalIgnoreCase), + StringComparer.OrdinalIgnoreCase), xunitTestCase.TestMethodArguments, xunitTestCase.SourceFilePath, xunitTestCase.SourceLineNumber, From d4b78bcc183535ae30d7c8b088b65dd996a916d5 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 19:44:44 -0700 Subject: [PATCH 063/105] Fail discovery when a variant's traits cannot be applied A test case that is not an XunitTestCase could not be written to, so it was reported to the runner without its data store trait after a warning on stdout. The export and E2E legs filter positively on DataStore, so such a case would be absent from them and they would still pass - the outcome the rest of this class exists to prevent, and one nobody would see because CI does not fail on a warning. Discovery now throws instead. Verified empirically: the per-method handler catches it and reports a failing case per fixture argument set, carrying the traits that make it visible to those legs. Also resolves the test assets path with the local directory separator, as the project file writes it with the one MSBuild used to build it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../TestAssetRunner.cs | 4 +++ .../CustomXunitTestFrameworkDiscoverer.cs | 31 ++++++++++--------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index dec541116f..745515b2ee 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -188,6 +188,10 @@ private static string ResolveAssetsAssembly() "The TestAssetsAssembly metadata attribute is missing. It is injected by the project file and identifies the assembly holding the scenarios."); } + // The project file writes this path with the separator MSBuild used on the machine that + // built it, and a backslash is an ordinary filename character everywhere except Windows. + path = Path.GetFullPath(path.Replace('\\', Path.DirectorySeparatorChar)); + if (!File.Exists(path)) { throw new InvalidOperationException( diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index c9a39b09a4..fdc8dab5e2 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -973,14 +973,16 @@ private static void ApplyVariantDisplayName(ITestCase testCase, SingleFlag[] clo if (testCase is not XunitTestCase xunitTestCase) { - // Without the suffix every variant of this method reports under the same name, so - // a failure cannot be attributed to a fixture argument set. Every test case type + // Without the suffix every variant of this method reports under the same name, so a + // failure cannot be attributed to a fixture argument set. Every test case type // xunit.v3 produces derives from XunitTestCase, so this only happens if a custom - // discoverer introduces its own type -- say so rather than silently mis-naming. - Console.WriteLine( - $"[FixtureArgumentSets] WARNING: Test case '{testCase.TestCaseDisplayName}' is {testCase.GetType().Name}, not {nameof(XunitTestCase)}. " + - "Its fixture argument set will NOT be appended to the display name, so variants of this test will be indistinguishable in test results."); - return; + // discoverer introduces its own type. Fail rather than carry on: the per-method + // handler turns this into a reported failure per fixture argument set, which is + // visible to a filtering leg, whereas a warning on stdout is not. + throw new InvalidOperationException( + $"Test case '{testCase.TestCaseDisplayName}' is {testCase.GetType().Name}, not {nameof(XunitTestCase)}, " + + "so its fixture argument set can be applied to neither its display name nor its traits. " + + "Variants of this test would be indistinguishable in results and invisible to any CI leg selecting on a trait."); } if (TestCaseDisplayNameField == null) @@ -1024,13 +1026,14 @@ private static void ApplyVariantTraits(ITestCase testCase, FixtureArgumentSetTes { if (testCase is not XunitTestCase xunitTestCase) { - // ApplyVariantDisplayName has already reported the type, so this only adds what else - // is lost by it: nothing here can reach a case type this class does not know how to - // write to, and a case with no traits is one a filtering leg cannot see. - Console.WriteLine( - $"[FixtureArgumentSets] WARNING: the fixture argument set traits of '{testCase.TestCaseDisplayName}' could NOT be applied, " + - "so a CI leg selecting on them will not run it."); - return; + // A case whose traits cannot be written is one a filtering leg cannot see, which is + // the failure this class exists to prevent -- so fail discovery rather than let it + // through untagged. ApplyVariantDisplayName rejects the same case types, so in the + // current call order this is unreachable; it is kept because this method needs the + // guarantee on its own account, not because of the order it happens to be called in. + throw new InvalidOperationException( + $"The fixture argument set traits of '{testCase.TestCaseDisplayName}' could not be applied because it is " + + $"{testCase.GetType().Name}, not {nameof(XunitTestCase)}. A CI leg selecting on those traits would not run it."); } foreach (KeyValuePair> trait in variantMethod.Traits) From 259c205f1c515d6a6587f10155a184fa4f31079e Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 19:49:07 -0700 Subject: [PATCH 064/105] Retry a test that timed out A timeout reaches the retry decision as Xunit.Sdk.TestTimeoutException, whose name contains "Xunit", so the substring match that classifies assertion failures claimed it. RetryOnAssertionFailure defaults to false because an assertion failure is deterministic, so the test lost every remaining attempt - and a test that ran long because a dependency was slow once is exactly the flakiness these attributes exist to absorb. Verified: a test that exceeds its timeout on the first attempt and completes on the second was reported Failed, and passes with this change. TestTimeoutException derives from System.Exception rather than XunitException, so it was only ever mistaken for an assertion by way of that match. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryTimeouts/RetryTimeoutTests.cs | 45 +++++++++++++++++++ .../RetryTimeoutTests.cs | 43 ++++++++++++++++++ .../RetryTestCase.cs | 18 ++++++-- 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTimeouts/RetryTimeoutTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTimeouts/RetryTimeoutTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTimeouts/RetryTimeoutTests.cs new file mode 100644 index 0000000000..ef057976c2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryTimeouts/RetryTimeoutTests.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryTimeouts +{ + /// + /// A timeout is the canonical flaky failure, so a retry harness has to treat it as retriable. + /// Expected: 2 results, 1 passed and 1 failed. + /// + public class RetryTimeoutTests + { + private static int _flakyTimeoutAttempts; + + /// + /// A test that exceeds its timeout on the first attempt and completes on the second must be + /// reported as passed. xUnit reports a timeout as Xunit.Sdk.TestTimeoutException, + /// whose name contains "Xunit", so a retry decision made by matching that substring would + /// take it for a deterministic assertion failure and spend none of the remaining attempts. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, Timeout = 500)] + public async Task FlakyTimeout_IsRetriedAndPasses() + { + if (Interlocked.Increment(ref _flakyTimeoutAttempts) == 1) + { + await Task.Delay(5000); + } + } + + /// + /// A test that exceeds its timeout on every attempt must still be reported as failed once, + /// rather than losing the final attempt's failure the way a deferred failure would. + /// + [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = 10, Timeout = 500)] + public async Task AlwaysTimesOut_IsReportedFailedOnce() + { + await Task.Delay(5000); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs new file mode 100644 index 0000000000..860212fd57 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers what the retrying test attributes do with a test that exceeds its timeout. + /// + /// + /// A timeout is the failure a retry harness exists for: a test that runs long because a + /// dependency was slow once is the definition of flaky. It reaches the retry decision as + /// Xunit.Sdk.TestTimeoutException, and the decision is made by matching "Xunit" and + /// "Assert" against the exception type name, so a timeout is at risk of being read as a + /// deterministic assertion failure and denied the attempts it was configured for. + /// + public class RetryTimeoutTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryTimeouts.RetryTimeoutTests."; + + /// + /// A test that times out once and then completes has to be retried into a pass, and one + /// that times out on every attempt has to be reported failed exactly once. + /// + [Fact] + public void GivenTestsThatExceedTheirTimeout_WhenTheRunCompletes_ThenTheFlakyOneIsRetriedIntoAPass() + { + TestAssetRun run = TestAssetRunner.Run("RetryTimeouts"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "FlakyTimeout_IsRetriedAndPasses"] = "Passed", + [ClassName + "AlwaysTimesOut_IsReportedFailedOnce"] = "Failed", + }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 5fa7ca71c5..b03975ab93 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -596,10 +596,20 @@ public bool QueueMessage(IMessageSinkMessage message) // than spending one: an assertion failure is taken to be deterministic, so it is // not retried. The failure is still reported red either way, so the cost is a // flaky test going unretried, never a failure going unseen. - IsAssertionFailure = failed.ExceptionTypes != null && - failed.ExceptionTypes.Length > 0 && - (failed.ExceptionTypes[0].Contains("Xunit", StringComparison.Ordinal) || - failed.ExceptionTypes[0].Contains("Assert", StringComparison.Ordinal)); + // + // A timeout is excluded because it is the one failure that trade cannot be made + // for. Xunit.Sdk.TestTimeoutException contains "Xunit" and so matched here, but a + // test that ran long because a dependency was slow once is the definition of the + // flakiness these attributes exist to absorb -- and unlike the cases above, it is + // not an assertion at all. It derives from System.Exception rather than + // XunitException, so it is only ever mistaken for one by way of this match. + string failedExceptionType = failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 + ? failed.ExceptionTypes[0] + : string.Empty; + + IsAssertionFailure = !failedExceptionType.Contains("Timeout", StringComparison.Ordinal) && + (failedExceptionType.Contains("Xunit", StringComparison.Ordinal) || + failedExceptionType.Contains("Assert", StringComparison.Ordinal)); if (_deferFailures) { From 196ecaef95b03bc6423103c3927bd259ea3adb72 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:25:24 -0700 Subject: [PATCH 065/105] Pin the decision RetryOnAssertionFailure makes The retry scenarios used tests that fail on every attempt or on none. Both report the same result whether the retries were spent or not, so a regression that retried everything, or nothing, left them green: the policy itself was never asserted. Adds two tests that recover on their second attempt, which makes the reported outcome a direct statement about whether that attempt ran - an assertion failure must stay failed under the default policy, and a non-assertion exception must be retried into a pass. Verified by mutation: inverting the RetryOnAssertionFailure check fails the new test, and left every pre-existing retry test green. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Scenarios/RetryPolicy/RetryPolicyTests.cs | 55 +++++++++++++++++++ .../RetryPolicyTests.cs | 44 +++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs new file mode 100644 index 0000000000..4cf45139fb --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs @@ -0,0 +1,55 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryPolicy +{ + /// + /// Tests that tell the two RetryOnAssertionFailure policies apart. + /// Expected: 2 results, 1 failed and 1 passed. + /// + /// + /// A test that fails on every attempt is reported failed whether or not it was retried, and one + /// that fails on none is reported passed either way, so neither pins the policy that decides + /// between them. Both tests here fail once and would pass on a second attempt, so the reported + /// outcome states plainly whether the second attempt was spent. + /// + public class RetryPolicyTests + { + private static int _assertionAttempts; + private static int _exceptionAttempts; + + /// + /// An assertion failure is taken to be deterministic, so with the default policy the + /// remaining attempts are not spent and the recovery never happens: this must be reported + /// failed. Were it retried it would pass, so the outcome distinguishes the two. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = false)] + public void AssertionFailureUnderTheDefaultPolicy_IsNotRetried() + { + if (Interlocked.Increment(ref _assertionAttempts) == 1) + { + Assert.Fail("ASSET: first attempt fails an assertion, a second attempt would pass"); + } + } + + /// + /// A non-assertion exception is the transient kind these attributes exist for, so it is + /// retried even under the default policy: this must be reported passed. Were the policy + /// applied to every failure alike it would be reported failed. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = false)] + public void NonAssertionExceptionUnderTheDefaultPolicy_IsRetried() + { + if (Interlocked.Increment(ref _exceptionAttempts) == 1) + { + throw new InvalidOperationException("ASSET: first attempt throws, a second attempt would pass"); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs new file mode 100644 index 0000000000..eb41f699c2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs @@ -0,0 +1,44 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins the decision RetryOnAssertionFailure makes, rather than only the outcomes either + /// side of it. + /// + /// + /// The existing retry scenarios use tests that fail on every attempt or on none. Those report + /// the same result whether the retries were spent or not, so a regression that retried + /// everything, or nothing, would leave them green. Both tests here recover on their second + /// attempt, which makes the reported outcome a direct statement about whether that attempt ran. + /// + public class RetryPolicyTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryPolicy.RetryPolicyTests."; + + /// + /// Under the default policy an assertion failure is not retried and a non-assertion + /// exception is, so a test that recovers on its second attempt is reported failed in the + /// first case and passed in the second. + /// + [Fact] + public void GivenTestsThatRecoverOnASecondAttempt_WhenTheDefaultPolicyApplies_ThenOnlyTheNonAssertionFailureIsRetried() + { + TestAssetRun run = TestAssetRunner.Run("RetryPolicy"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "AssertionFailureUnderTheDefaultPolicy_IsNotRetried"] = "Failed", + [ClassName + "NonAssertionExceptionUnderTheDefaultPolicy_IsRetried"] = "Passed", + }); + } + } +} From cd886d52ce31a3cf2a023a65b7dd3db7d60c4cb8 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:25:35 -0700 Subject: [PATCH 066/105] Keep a fault's traits when one trait attribute throws xUnit computes a test method's traits on demand and hands them back as one dictionary, so reading them runs every trait attribute the method and its class declare. One that throws took the whole dictionary with it, and the failure reported in the lost method's place was left carrying none of the traits its other attributes named. That is the silence this fault path exists to break: the export leg selects positively with (DataStore=X)&(Category=Y), and a filter cannot match a trait that is absent, so the leg would report success with the method missing. The ordinary read is still tried first and is what runs for a healthy class. Only when it throws are the attributes read one at a time, so a single attribute that will not produce its traits costs only its own. Verified: with a trait attribute that throws on a method whose discovery faults, the two filtered regression tests failed before this change and pass after it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../ThrowingTrait/ThrowingTraitAttribute.cs | 28 +++++++ .../ThrowingTrait/ThrowingTraitTests.cs | 40 ++++++++++ .../ThrowingTraitTests.cs | 75 +++++++++++++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 64 +++++++++++++++- 4 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitAttribute.cs new file mode 100644 index 0000000000..cf304bd112 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitAttribute.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait +{ + /// + /// A trait attribute that throws when asked for its traits, standing in for one that computes a + /// trait from configuration that is not present. + /// + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = true)] + public sealed class ThrowingTraitAttribute : Attribute, ITraitAttribute + { + /// + /// Throws, as a trait attribute computing its value at discovery time may. + /// + /// Never returns. + public IReadOnlyCollection> GetTraits() + { + throw new InvalidOperationException("ASSET: this trait attribute cannot produce its traits"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitTests.cs new file mode 100644 index 0000000000..b1fa8fcfbc --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingTrait/ThrowingTraitTests.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait +{ + /// + /// A class whose discovery faults on a method that also carries a trait attribute that throws. + /// Expected: 1 result, failed, still carrying the ordinary traits its declaration named. + /// + /// + /// The failure standing in for the lost method has to carry the traits a CI leg selects on, and + /// those are read from the method as a whole. One trait attribute refusing to produce its value + /// must therefore not cost the method the traits its other attributes declared: a leg selecting + /// positively on Category would not match the failure, and would report success with the + /// method missing. + /// + [Trait("Category", "ThrowingTraitProbe")] + public class ThrowingTraitTests + { + /// + /// Never runs: it declares two fixture argument set attributes, so which one applies cannot + /// be determined. It also carries a trait attribute that throws, alongside ordinary ones. + /// + [Fact] + [Trait("Owner", "ThrowingTraitOwner")] + [ThrowingTrait] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs new file mode 100644 index 0000000000..dc51e39ae7 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs @@ -0,0 +1,75 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a discovery fault on a method that also carries a trait attribute that throws. + /// + /// + /// The traits of the lost method are read as one dictionary, and xUnit computes them on demand, + /// so a single trait attribute that throws can take every other trait down with it. The failure + /// reported in the method's place would then be missing the very traits a leg selects on, and a + /// leg filtering positively would report success without it - the outcome reporting these + /// failures exists to prevent. + /// + public class ThrowingTraitTests + { + private const string FaultCase = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait.ThrowingTraitTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string FaultCaseTwoDimensions = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait.ThrowingTraitTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + + private static Dictionary BothFaultCases() => + new Dictionary + { + [FaultCase] = "Failed", + [FaultCaseTwoDimensions] = "Failed", + }; + + /// + /// The failure standing in for the lost method is reported even though one of its trait + /// attributes threw. + /// + [Fact] + public void GivenAMethodWithATraitAttributeThatThrows_WhenItsDiscoveryFaults_ThenTheFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingTrait"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The trait the method's own declaration named still selects the failure. This is the + /// shape of the export leg's filter, which requires a positive Category. + /// + [Fact] + public void GivenAMethodWithATraitAttributeThatThrows_WhenALegSelectsOnAnOrdinaryTrait_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingTrait", filterQueryTraits: "(Owner=ThrowingTraitOwner)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The class-level trait selects it too, so the failure is visible to a leg filtering on the + /// category the class declares. + /// + [Fact] + public void GivenAMethodWithATraitAttributeThatThrows_WhenALegSelectsOnTheClassTrait_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingTrait", filterQueryTraits: "(Category=ThrowingTraitProbe)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index fdc8dab5e2..602e428e93 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -648,7 +648,9 @@ void Add(string key, string value) catch (Exception ex) { Console.WriteLine( - $"[FixtureArgumentSets] WARNING: the ordinary traits of '{anchor.TestClass.TestClassName}.{anchor.MethodName}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + $"[FixtureArgumentSets] WARNING: the ordinary traits of '{anchor.TestClass.TestClassName}.{anchor.MethodName}' could not be read in one go, so they are being read one attribute at a time. {ex}"); + + AddFaultTraitsPerAttribute(anchor, Add); } foreach (SingleFlag flag in argumentSetCombination) @@ -659,6 +661,66 @@ void Add(string key, string value) return traits; } + /// + /// Reads the ordinary traits of the method a discovery failure stands for one attribute at a + /// time, so that a single attribute refusing to produce its traits costs only its own. + /// + /// + /// xUnit computes a test method's traits on demand and returns them as one dictionary, so + /// asking for them runs every trait attribute the method and its class declare. One that + /// throws - a trait computed from configuration that is not present, say - therefore takes + /// the whole dictionary with it, leaving the failure carrying none of the traits its other + /// attributes named. That is the same silence this fault path exists to break: the export + /// leg selects positively with (DataStore=X)&(Category=Y), and a filter cannot + /// match a trait that is absent, so the leg would report success with the method missing. + /// + /// This is only the fallback. The ordinary read is tried first and is what runs for a + /// healthy class, because it is xUnit's own and needs no agreement with it about how traits + /// are gathered. + /// + /// + /// The method the failure is reported against. + /// Records one trait name and value. + private static void AddFaultTraitsPerAttribute(XunitTestMethod anchor, Action add) + { + foreach (MemberInfo declaration in new MemberInfo[] { anchor.TestClass?.Class, anchor.Method }) + { + if (declaration == null) + { + continue; + } + + ITraitAttribute[] traitAttributes; + + try + { + traitAttributes = declaration.GetCustomAttributes(inherit: true).OfType().ToArray(); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the trait attributes of '{declaration.Name}' could not be read, so a trait filter may not select the failure standing in for its tests. {ex}"); + continue; + } + + foreach (ITraitAttribute traitAttribute in traitAttributes) + { + try + { + foreach (KeyValuePair trait in traitAttribute.GetTraits()) + { + add(trait.Key, trait.Value); + } + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the trait attribute '{traitAttribute.GetType().Name}' of '{declaration.Name}' could not produce its traits, so a filter naming those traits may not select the failure standing in for its tests. {ex}"); + } + } + } + } + /// /// Builds the part of a discovery failure's unique ID that identifies the method it is /// reported against. From e89c80296c58c0e79962fb06d06a7e434c16390a Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:25:47 -0700 Subject: [PATCH 067/105] Ask whether an attempt failed, not whether it deferred The no-result and crash paths both decide which attempt's failure may reach the bus, and both asked the current attempt's bus for HasDeferredFailure. The last attempt is constructed with deferFailures: false, so its failures go straight through and it never has one deferred - the answer was always no, whatever it had seen. An earlier attempt's held-over failure could therefore be replayed on top of the failure the last attempt had already published, which is the double reporting the surrounding comments say must not happen. The two are equivalent on every other attempt, where a failure is both observed and deferred, so this only changes the last one. Not covered by a scenario: reaching it needs the runner to abort or throw after a failure was published but before the summary came back, which the test assets cannot induce. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryTestCase.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index b03975ab93..9236d72fd7 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -153,7 +153,7 @@ public async ValueTask Run( if (summary.Total == 0) { NoResultOutcome outcome = DecideNoResultOutcome( - currentAttemptObservedFailure: interceptingBus.HasDeferredFailure, + currentAttemptObservedFailure: interceptingBus.HasObservedFailure, earlierAttemptObservedFailure: pendingFailureBus?.HasDeferredFailure ?? false); Console.WriteLine( @@ -350,7 +350,7 @@ public async ValueTask Run( // path applies, so it is decided the same way here. Disposing replays whatever // is still deferred, and the finally below does that for the current attempt. NoResultOutcome disposition = DecideNoResultOutcome( - currentAttemptObservedFailure: interceptingBus?.HasDeferredFailure ?? false, + currentAttemptObservedFailure: interceptingBus?.HasObservedFailure ?? false, earlierAttemptObservedFailure: pendingFailureBus?.HasDeferredFailure ?? false); if (disposition == NoResultOutcome.ReplayEarlierAttempt) @@ -558,6 +558,21 @@ public FailureInterceptingMessageBus(IMessageBus innerBus, bool deferFailures) /// public bool HasDeferredFailure => _deferredMessages.Count > 0; + /// + /// Gets a value indicating whether this attempt saw a failure at all, whether or not it + /// was deferred. + /// + /// + /// This is not the same question as , and the difference + /// falls exactly on the last attempt: that one is constructed with + /// deferFailures: false, so its failures go straight through to the inner bus and + /// it never has one deferred. Asking whether the + /// current attempt failed therefore always answers no on the last attempt, and an + /// earlier attempt's held-over failure would then be replayed on top of the failure this + /// one already published - two results for a single test. + /// + public bool HasObservedFailure => LastFailureMessage != null; + public bool QueueMessage(IMessageSinkMessage message) { if (_deferring) From 5f6b89c4696ce363cfea1cd6c24935e1c2e4fdeb Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:31:09 -0700 Subject: [PATCH 068/105] Do not let a skip erase a failure the test already showed A [RetryFact] that failed on one attempt and called Assert.Skip on the next was reported NotExecuted: the failure was discarded as superseded, and the test then vanished from the results entirely. A real failure became CI-green. A pass legitimately supersedes an earlier failure - that is what a retry is for. A skip does not: it is an abstention, and it has not shown the test to be sound. Hold the abstention on any attempt that follows a failure so the earlier failure can be reported as the test's only result; replaying it without holding the skip reports the test twice. --- .../SkipAfterFailure/SkipAfterFailureTests.cs | 41 ++++++++++++++++++ .../SkipAfterFailureTests.cs | 40 +++++++++++++++++ .../RetryTestCase.cs | 43 ++++++++++++++++++- 3 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs new file mode 100644 index 0000000000..08832f01b4 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipAfterFailure +{ + /// + /// A retrying test that fails on its first attempt and skips itself on the second. + /// Expected: 1 result, skipped. + /// + /// + /// A skip is a definitive result, so it supersedes the earlier attempt's failure the same way a + /// pass does. This scenario exists to state that outcome rather than leave it to be discovered: + /// the first attempt's failure is discarded, so a test that fails and then decides it should not + /// have run is reported skipped, not failed. + /// + public class SkipAfterFailureTests + { + private static int _attempts; + + /// + /// Throws on the first attempt - a non-assertion exception, so it is retried under the + /// default policy - and skips itself on the second. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void FailsThenSkips() + { + if (Interlocked.Increment(ref _attempts) == 1) + { + throw new InvalidOperationException("ASSET: first attempt fails, the second skips"); + } + + Assert.Skip("ASSET: skipped on the second attempt"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs new file mode 100644 index 0000000000..c8b5d1d46a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins what a retrying test reports when an attempt skips after an earlier one failed. + /// + /// + /// An attempt that reports a real result supersedes a failure held over from an earlier one, and + /// a runtime skip is a real result. The earlier failure is therefore discarded and the test is + /// reported skipped: the same supersession that lets a retry turn a failure into a pass. This is + /// asserted rather than assumed because the alternative reading - that a failure seen once must + /// always be reported - would be a defensible design and a silent change if it were made by + /// accident. + /// + public class SkipAfterFailureTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipAfterFailure.SkipAfterFailureTests."; + + /// + /// A test that fails, is retried, and then skips itself is reported skipped exactly once, + /// with the superseded failure discarded rather than also reported. + /// + [Fact] + public void GivenARetryingTestThatSkipsAfterFailing_WhenTheRunCompletes_ThenItIsReportedSkippedOnce() + { + TestAssetRun run = TestAssetRunner.Run("SkipAfterFailure"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary { [ClassName + "FailsThenSkips"] = "Failed" }); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 9236d72fd7..6d890b2cf6 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -129,7 +129,10 @@ public async ValueTask Run( { // Always wrap the bus so the failure details are available for the CI log, but only // defer (hide) a failure while a further attempt could still supersede it. - interceptingBus = new FailureInterceptingMessageBus(messageBus, deferFailures: !isLastAttempt); + interceptingBus = new FailureInterceptingMessageBus( + messageBus, + deferFailures: !isLastAttempt, + deferAbstentions: pendingFailureBus?.HasDeferredFailure ?? false); var summary = await XunitRunnerHelper.RunXunitTestCase( this, @@ -198,6 +201,29 @@ public async ValueTask Run( return noResultSummary; } + // A skip is not a pass. An attempt that abstains has not shown the test to be + // sound, so it must not erase a failure an earlier attempt already demonstrated: + // that failure is real, and discarding it here reports the test as skipped or + // not run, which no CI leg treats as a failure. + if (summary.Failed == 0 && + (summary.Skipped > 0 || summary.NotRun > 0) && + (pendingFailureBus?.HasDeferredFailure ?? false)) + { + Console.WriteLine( + $"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' skipped itself on attempt {attempt}/{_maxRetries} after failing earlier. Reporting the failure rather than the skip."); + + // The abstention was held rather than published, so dropping it now leaves + // the replayed failure as the test's only result. + interceptingBus.DiscardDeferredMessages(); + + bool replayed = pendingFailureBus.ReplayDeferredMessages(); + pendingFailureBus = null; + StopRunIfRequested(replayed, cancellationTokenSource); + + runSummary.Failed = 1; + return runSummary; + } + // This attempt reported a real result, so a failure held over from an earlier // attempt has been superseded and must not also be reported. pendingFailureBus?.DiscardDeferredMessages(); @@ -537,13 +563,15 @@ private class FailureInterceptingMessageBus : IMessageBus { private readonly IMessageBus _innerBus; private readonly bool _deferFailures; + private readonly bool _deferAbstentions; private readonly List _deferredMessages = new List(); private bool _deferring; - public FailureInterceptingMessageBus(IMessageBus innerBus, bool deferFailures) + public FailureInterceptingMessageBus(IMessageBus innerBus, bool deferFailures, bool deferAbstentions = false) { _innerBus = innerBus; _deferFailures = deferFailures; + _deferAbstentions = deferAbstentions; } public string LastFailureMessage { get; private set; } @@ -581,6 +609,17 @@ public bool QueueMessage(IMessageSinkMessage message) return true; } + if (_deferAbstentions && (message is ITestSkipped || message is ITestNotRun)) + { + // An attempt that skips itself has abstained rather than shown the test to be + // sound, and this attempt follows one that failed. Hold the abstention so the + // caller can choose between it and that earlier failure; publishing it here + // would leave the failure no way to be reported without a second result. + _deferring = true; + _deferredMessages.Add(message); + return true; + } + if (message is ITestFailed failed) { // Capture failure details for diagnostics From 79ebf92009a1b9ea1d8c1a758bd8e80aec4ce943 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:32:41 -0700 Subject: [PATCH 069/105] Make an empty test leg fail instead of pass The E2E and export legs filter positively on DataStore, so a case that reaches the runner without that trait is invisible to them rather than failing - the shape of bug this PR already found twice. --minimum-expected-tests turns an emptied leg into a failure (exit 9, verified locally). Also drop empty elements when composing the E2E filter, so a caller passing no categoryFilter cannot produce /[(DataStore=X)&()]. --- build/jobs/e2e-tests.yml | 8 ++++++-- build/jobs/run-export-tests.yml | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index ced72768f1..5670e020f2 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -39,11 +39,15 @@ steps: # xUnit v3 MTP filter-query: match DataStore trait for this app service type and any extra trait filters $categoryFilter = '${{ parameters.categoryFilter }}' - $parts = $categoryFilter -split '&' + $parts = @($categoryFilter -split '&' | Where-Object { $_.Trim() -ne '' }) $allParts = @("DataStore=${{ parameters.appServiceType }}") + $parts $traitFilter = "[" + (($allParts | ForEach-Object { "($_)" }) -join '&') + "]" $filterQuery = "/$traitFilter" - $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx') + + # This filter is positive on DataStore, so a test case that reaches the runner without that + # trait is invisible to it rather than failing. Assert a floor on the count so a filter or + # trait regression that silently empties this leg cannot report success. + $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" & dotnet $dll.FullName @args diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index 4ebd2adb77..b87f3a2017 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -103,7 +103,9 @@ jobs: throw "Could not find $testName.dll under $testRoot" } - $args = @('--filter-query', '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx') + # Positive on DataStore, so a case missing that trait is invisible here rather than failing. + # The floor makes a filter or trait regression that empties this leg fail instead of pass. + $args = @('--filter-query', '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=CosmosDb)&(Category=ExportLongRunning)]" & dotnet $dll.FullName @args @@ -255,7 +257,9 @@ jobs: throw "Could not find $testName.dll under $testRoot" } - $args = @('--filter-query', '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx') + # Positive on DataStore, so a case missing that trait is invisible here rather than failing. + # The floor makes a filter or trait regression that empties this leg fail instead of pass. + $args = @('--filter-query', '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=SqlServer)&(Category=ExportLongRunning)]" & dotnet $dll.FullName @args From 6f45be1f01d4a3658f7b4fcfde702dde76e24996 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:36:16 -0700 Subject: [PATCH 070/105] Say what these comments actually describe Three comments claimed more than the code does: - The fault-reporting example had a method declaring DataStore=SqlServer twice reaching the Cosmos leg. It reaches neither: with no DataStore trait it is invisible to every leg that selects positively on one, which is the failure the surrounding code exists to prevent. - The clamping test claimed to cover a negative retry delay, but clamping MaxRetries to one means no retry, and the delay is only reached before one. Split out a test that fails once so the delay is actually reached. - RetryOnAssertionFailure documented itself in terms of XunitException. The message bus never sees the exception - it matches the reported type name - and timeouts are now excluded from that match. --- .../RetryOutcomes/RetryOutcomeTests.cs | 22 ++++++++++++++++--- .../RetryFactOutcomeTests.cs | 1 + .../CustomXunitTestFrameworkDiscoverer.cs | 5 +++-- .../RetryFactAttribute.cs | 9 +++++++- .../RetryTheoryAttribute.cs | 9 +++++++- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs index 6001879b9d..83b80e4e92 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs @@ -17,6 +17,7 @@ public class RetryOutcomeTests { private static int _flakyAttempts; private static int _clampedAttempts; + private static int _clampedDelayAttempts; /// /// An assertion failure with retries disabled must be reported once and not retried. @@ -83,13 +84,28 @@ public void SkippedAtRunTime_IsReportedSkipped() } /// - /// A MaxRetries below one would skip the attempt loop and report nothing, and a negative - /// delay would make Task.Delay throw. Both are clamped, so the test runs exactly once. + /// A MaxRetries below one would skip the attempt loop and report nothing, so it is clamped + /// up to one and the test runs exactly once. This test cannot say anything about the delay: + /// clamping MaxRetries to one means no retry, and the delay is only ever reached before one. /// - [RetryFact(MaxRetries = 0, DelayBetweenRetriesMs = -500, RetryOnAssertionFailure = true)] + [RetryFact(MaxRetries = 0, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = true)] public void ClampedRetryConfiguration_RunsExactlyOnce() { Assert.Equal(1, Interlocked.Increment(ref _clampedAttempts)); } + + /// + /// A negative delay would make Task.Delay throw, which would lose the retry this test needs + /// to pass. Reaching the delay at all takes a failed first attempt, so this fails once and + /// then succeeds: a passing result is the statement that the clamp held. + /// + [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = -500, RetryOnAssertionFailure = true)] + public void ClampedNegativeDelay_StillRetries() + { + if (Interlocked.Increment(ref _clampedDelayAttempts) == 1) + { + Assert.Fail("ASSET: failing so the retry delay is reached"); + } + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs index e6910b9a02..b990dc3b55 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactOutcomeTests.cs @@ -39,6 +39,7 @@ public void GivenRetryingTests_WhenTheRunCompletes_ThenEachTestIsReportedExactly [Prefix + "AlwaysPasses_IsReportedPassed"] = "Passed", [Prefix + "SkippedAtRunTime_IsReportedSkipped"] = "NotExecuted", [Prefix + "ClampedRetryConfiguration_RunsExactlyOnce"] = "Passed", + [Prefix + "ClampedNegativeDelay_StillRetries"] = "Passed", }); } } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 602e428e93..c8f821898a 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -423,8 +423,9 @@ internal static IReadOnlyList> BuildFaultArgumentSetCo /// would have used, because declaring two is one of the ways the expansion fails: asking for /// the single attribute throws, and answering that with no values at all leaves the failure /// carrying none of the traits its own declaration named. A method declaring - /// DataStore=SqlServer twice over would then be reported to the Cosmos leg and not to - /// the SQL one - the leg that would have run it passing green without it. + /// DataStore=SqlServer twice over would then reach the SQL leg with no + /// DataStore trait at all, and that leg selects positively on one - so it would match + /// nothing, and pass green with the method missing. /// /// /// Only this method's own attributes are read, never a sibling's. The failure stands for this diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs index bdbac82fad..e09440b554 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryFactAttribute.cs @@ -40,9 +40,16 @@ public RetryFactAttribute([CallerFilePath] string sourceFilePath = "", [CallerLi public int DelayBetweenRetriesMs { get; set; } = 5000; /// - /// Gets or sets whether to retry on assertion failures (XunitException). + /// Gets or sets whether to retry on assertion failures. /// Default is false - assertion failures usually indicate test bugs, not transient issues. /// Set to true for tests that validate eventually-consistent systems (e.g., cache refresh, reindex operations). + /// + /// What counts as an assertion failure is decided from the reported exception's type name - + /// one containing "Xunit" or "Assert", so that assertion libraries used alongside xUnit are + /// recognised too - rather than from the exception type itself, which the message bus does + /// not receive. Timeouts are deliberately excluded: they match that name test but are the + /// canonical transient failure, so they are retried whatever this is set to. + /// /// public bool RetryOnAssertionFailure { get; set; } = false; } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs index 624cb8234e..97e97fb759 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryAttribute.cs @@ -40,9 +40,16 @@ public RetryTheoryAttribute([CallerFilePath] string sourceFilePath = "", [Caller public int DelayBetweenRetriesMs { get; set; } = 5000; /// - /// Gets or sets whether to retry on assertion failures (XunitException). + /// Gets or sets whether to retry on assertion failures. /// Default is false - assertion failures usually indicate test bugs, not transient issues. /// Set to true for tests that validate eventually-consistent systems (e.g., cache refresh, reindex operations). + /// + /// What counts as an assertion failure is decided from the reported exception's type name - + /// one containing "Xunit" or "Assert", so that assertion libraries used alongside xUnit are + /// recognised too - rather than from the exception type itself, which the message bus does + /// not receive. Timeouts are deliberately excluded: they match that name test but are the + /// canonical transient failure, so they are retried whatever this is set to. + /// /// public bool RetryOnAssertionFailure { get; set; } = false; } From 458a1992d522e2a195482c79fa822f0dbf697393 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:56:31 -0700 Subject: [PATCH 071/105] Pin the trait a positively-filtered leg actually selects on The throwing-trait tests filtered on Owner and Category but never on the argument set trait - the one the E2E and export legs select positively on, and so the only one whose loss lets a leg pass green with the method missing. The shipping code is safe because the argument set trait is added outside the try that reads the method's traits, but nothing said so: moving that loop inside the try left all three tests passing. It now fails. Also make SingleFlag.Equals tolerate a default instance. SingleFlag is a struct, so default(SingleFlag) reaches Equals with a null EnumValue without passing through the constructor; GetHashCode already handled that, and an Equals that throws on the same value contradicts it. Pre-existing and not otherwise touched by this change, but the new discovery code reads SingleFlag and works around the inconsistency with null filters. --- .../ThrowingTraitTests.cs | 15 +++++++++++++++ .../SingleFlag.cs | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs index dc51e39ae7..8056e9397f 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingTraitTests.cs @@ -71,5 +71,20 @@ public void GivenAMethodWithATraitAttributeThatThrows_WhenALegSelectsOnTheClassT run, BothFaultCases()); } + + /// + /// The argument set trait still selects the failure. This is the trait the E2E and export + /// legs filter positively on, so it is the one whose loss would let such a leg pass green + /// with the method missing - the outcome this whole path exists to prevent. + /// + [Fact] + public void GivenAMethodWithATraitAttributeThatThrows_WhenALegSelectsOnTheArgumentSet_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingTrait", filterQueryTraits: "(AssetDataStore=Sql)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/SingleFlag.cs b/src/Microsoft.Health.Extensions.Xunit/SingleFlag.cs index b52088ef5f..9dbb201fb2 100644 --- a/src/Microsoft.Health.Extensions.Xunit/SingleFlag.cs +++ b/src/Microsoft.Health.Extensions.Xunit/SingleFlag.cs @@ -39,7 +39,10 @@ public SingleFlag(Enum enumValue) public bool Equals(SingleFlag other) { - return other.EnumValue.Equals(EnumValue); + // Not other.EnumValue.Equals(...): this is a struct, so default(SingleFlag) reaches here + // with a null EnumValue without ever passing through the constructor. GetHashCode already + // tolerates that, and an Equals that throws on the same value contradicts it. + return Equals(other.EnumValue, EnumValue); } public override bool Equals(object obj) From 5d8bf60a370082701a5d1955cec05768ab4a31a0 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 21:12:03 -0700 Subject: [PATCH 072/105] Report a declaration that names no value under every value it could have A fixture argument set naming no value expanded to nothing, so the product of the dimensions was empty and the failure standing in for the method carried no argument set trait at all. The E2E and export legs select positively on that trait, so such a failure was invisible to them: the leg reported success with the method's tests absent - the outcome this whole path exists to prevent, reached by the one route it did not cover. The value is unknown but the type is not, so the failure is now reported once per value that type declares. This fires only when a declaration produces no combination whatsoever: naming nothing in one position while naming something in another is how a method inherits that position from its class, and widening that would report the failure under values the declaration never asked for. Also correct the RetryOutcomeTests result count, which the added negative-delay test made stale. --- .../RetryOutcomes/RetryOutcomeTests.cs | 2 +- .../EmptyArgumentSetTests.cs | 28 +++++- .../PartialDiscoveryFaultTests.cs | 9 +- .../CustomXunitTestFrameworkDiscoverer.cs | 97 ++++++++++++++++++- 4 files changed, 128 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs index 83b80e4e92..4355cc84cc 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryOutcomes/RetryOutcomeTests.cs @@ -11,7 +11,7 @@ namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryOutcomes { /// /// The full outcome matrix, run without cancellation. - /// Expected: 7 results, 3 failed, 3 passed and 1 skipped, with exactly one result per test. + /// Expected: 8 results, 3 failed, 4 passed and 1 skipped, with exactly one result per test. /// public class RetryOutcomeTests { diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs index 2ba9b1808b..6c7886305a 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/EmptyArgumentSetTests.cs @@ -15,7 +15,15 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests /// public class EmptyArgumentSetTests { - private const string ErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.EmptyArgumentSet.EmptyArgumentSetTests.NeverRuns (fixture argument set discovery)"; + private const string ErrorCaseSql = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.EmptyArgumentSet.EmptyArgumentSetTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string ErrorCaseCosmos = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.EmptyArgumentSet.EmptyArgumentSetTests.NeverRuns (fixture argument set discovery: Cosmos)"; + + private static Dictionary BothErrorCases() => + new Dictionary + { + [ErrorCaseSql] = "Failed", + [ErrorCaseCosmos] = "Failed", + }; /// /// An argument set naming no flag collapses the product of the declared dimensions to nothing, @@ -29,11 +37,27 @@ public void GivenAnArgumentSetThatNamesNoValue_WhenTheClassIsDiscovered_ThenTheR { TestAssetRun run = TestAssetRunner.Run("EmptyArgumentSet"); + TestAssetRunAssertions.PublishedExactly(run, BothErrorCases()); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The declaration names no value, so there is none to report the failure under - but it still + /// names a type, and the failure is reported once per value that type declares. Without this a + /// leg selecting positively on the argument set would see nothing and report success with the + /// method's tests absent, which is the outcome reporting these failures exists to prevent. + /// + [Fact] + public void GivenAnArgumentSetThatNamesNoValue_WhenALegSelectsOnTheArgumentSet_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("EmptyArgumentSet", filterQueryTraits: "(AssetDataStore=Sql)"); + TestAssetRunAssertions.PublishedExactly( run, new Dictionary { - [ErrorCaseName] = "Failed", + [ErrorCaseSql] = "Failed", }); Assert.NotEqual(0, run.ExitCode); diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs index 2e0e079b50..1f6e381dcd 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartialDiscoveryFaultTests.cs @@ -16,7 +16,8 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests public class PartialDiscoveryFaultTests { private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartialDiscoveryFault.PartialDiscoveryFaultTests"; - private const string ErrorCaseName = ScenarioClass + ".NeverRuns (fixture argument set discovery)"; + private const string ErrorCaseSql = ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"; + private const string ErrorCaseCosmos = ScenarioClass + ".NeverRuns (fixture argument set discovery: Cosmos)"; /// /// Test cases are published to xUnit as the discoverer walks the class's methods, so a @@ -36,7 +37,8 @@ public void GivenOneMethodThatCannotBeExpanded_WhenTheClassIsDiscovered_ThenTheO { [ScenarioClass + ".RunsBeforeTheFault"] = "Passed", [ScenarioClass + ".RunsAfterTheFault"] = "Passed", - [ErrorCaseName] = "Failed", + [ErrorCaseSql] = "Failed", + [ErrorCaseCosmos] = "Failed", }); Assert.NotEqual(0, run.ExitCode); @@ -71,7 +73,8 @@ public void GivenOneMethodThatCannotBeExpanded_WhenTestsAreSelectedByItsOrdinary run, new Dictionary { - [ErrorCaseName] = "Failed", + [ErrorCaseSql] = "Failed", + [ErrorCaseCosmos] = "Failed", }); Assert.NotEqual(0, run.ExitCode); diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index c8f821898a..470b601d3b 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -505,7 +505,7 @@ void AddAll(Func read, Func> readMeta { try { - dimensions.Add(ExpandEnumFlagsFromAttributeData(attribute)); + dimensions.Add(ExpandFaultEnumFlagsFromAttributeData(attribute)); } catch (Exception ex) { @@ -560,6 +560,7 @@ void AddAllFromMetadata(Func> readMetadata, str private static SingleFlag[][] ExpandEnumFlagsFromMetadata(CustomAttributeData attributeData) { var dimensions = new List(); + var enumTypes = new List(); foreach (CustomAttributeTypedArgument argument in attributeData.ConstructorArguments) { @@ -578,7 +579,9 @@ private static SingleFlag[][] ExpandEnumFlagsFromMetadata(CustomAttributeData at } } - return dimensions.ToArray(); + // This path only ever runs while reporting a fault, so a declaration naming nothing is + // widened here for the same reason it is on the reflection path. + return WidenFaultDimensionsThatNameNothing(dimensions.ToArray(), enumTypes.ToArray()); void AddDimension(CustomAttributeTypedArgument argument) { @@ -592,6 +595,7 @@ void AddDimension(CustomAttributeTypedArgument argument) } dimensions.Add(GetSingleValuedFlags((Enum)Enum.ToObject(argumentType, argument.Value)).ToArray()); + enumTypes.Add(argumentType); } } @@ -1154,6 +1158,95 @@ private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(FixtureArgumentSe .ToArray(); } + /// + /// Expands an argument set declaration for the fault path, widening a dimension that names no + /// value to every value its type declares. + /// + /// + /// A dimension naming no value expands to nothing, so the product of the dimensions is empty + /// and the failure standing in for the method carries no argument set trait at all. The E2E + /// and export legs select positively on that trait, so such a failure is invisible to them: + /// the leg reports success with the method's tests simply absent, which is the outcome this + /// whole path exists to prevent. The value is unknown but the type is not - a dimension is + /// declared as an enum whether or not it names any of its flags - so the failure is reported + /// once per value that type declares instead. That over-reports, in the same deliberate + /// direction as the rest of this path: a leg seeing a failure for a variant that would not + /// have existed is loud and traceable to this declaration, where a leg seeing nothing is not. + /// + /// The declaration to expand. + /// One dimension per argument set, each holding at least one value where the type is known. + private static SingleFlag[][] ExpandFaultEnumFlagsFromAttributeData(FixtureArgumentSetsAttribute attribute) + { + IReadOnlyList argumentSets = attribute.GetArgumentSets(); + SingleFlag[][] dimensions = argumentSets + .Select(e => GetSingleValuedFlags(e).ToArray()) + .ToArray(); + + return WidenFaultDimensionsThatNameNothing(dimensions, argumentSets.Select(e => e?.GetType()).ToArray()); + } + + /// + /// Replaces the dimensions of a declaration that names no value at all with every value their + /// types declare, so the failure standing in for the method can still be selected. + /// + /// + /// This fires only when the declaration produces no combination whatsoever. A dimension that + /// names nothing alongside one that does is not this case: naming nothing in one position is + /// how a method asks to inherit that position from its class, and widening it would report + /// the failure under values the declaration never asked for. + /// + /// The dimensions as declared. + /// The type of each dimension, positionally, with null where unknown. + /// The dimensions to report the failure against. + private static SingleFlag[][] WidenFaultDimensionsThatNameNothing(SingleFlag[][] dimensions, Type[] enumTypes) + { + if (dimensions.Length == 0 || dimensions.Any(dimension => dimension.Length > 0)) + { + return dimensions; + } + + return dimensions + .Select((dimension, index) => AllSingleValuedFlagsOf(index < enumTypes.Length ? enumTypes[index] : null)) + .ToArray(); + } + + /// + /// Lists every single-valued flag an enum type declares, for reporting a failure whose own + /// declaration named none of them. + /// + /// The type to read, which may be null or not an enum. + /// The type's single-valued flags, or nothing if they cannot be read. + private static SingleFlag[] AllSingleValuedFlagsOf(Type enumType) + { + if (enumType == null || !enumType.IsEnum) + { + return Array.Empty(); + } + + try + { + return Enum.GetValues(enumType) + .Cast() + .Where(value => + { + long asLong = Convert.ToInt64(value); + return (asLong != 0) && ((asLong & (asLong - 1)) == 0); + }) + .Select(value => new SingleFlag(value)) + .ToArray(); + } + catch (Exception ex) + { + // Reading the type's values is the only thing that can fail here, and a failure leaves + // the dimension as it was: the failure is still reported, just without this trait. + // Losing the trait is the thing being guarded against, so it is said out loud. + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: '{enumType.FullName}' names no value in this declaration and its values could not be read, so the failure reported in the method's place carries no trait for that dimension and a leg selecting positively on it will not see the failure. {ex}"); + + return Array.Empty(); + } + } + /// /// Splits a flags enum value into the single-valued flags it names. /// From 23388e5752995e46ebc8b640e8ce0d0b4b707b90 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 21:12:38 -0700 Subject: [PATCH 073/105] Give the integration legs the same floor as the filtered ones These legs select by exclusion, so a filter that matches nothing does not fail - it runs zero tests and exits successfully. Same class of green-empty leg the E2E and export legs were just guarded against. --- build/jobs/run-cosmos-tests.yml | 2 +- build/jobs/run-sql-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index d7d2cdf624..0f7b8bbf3e 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -99,7 +99,7 @@ jobs: inputs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx --minimum-expected-tests 1' # Disable the task's built-in result publishing so the PublishTestResults@2 task below, # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index a96669f282..749f063bb2 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -91,7 +91,7 @@ jobs: inputs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx --minimum-expected-tests 1' # Disable the task's built-in result publishing so the PublishTestResults@2 task below, # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false From 688119e2d5ac2f8fe090c6f3b4fab4a14b09bdda Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 21:15:23 -0700 Subject: [PATCH 074/105] Say what these comments and names actually describe The skip-after-failure scenario still described the outcome it had before the fix, down to the test method name: it says skipped where the assertion and the code both say failed. A guard in the discoverer claimed to be unreachable in the current call order, but the method it named returns before its own type check when the variant closes no argument set, so the guard is reached. Also collapse a retry decision that spelled out both branches of a boolean it could return directly, and hoist the projection the two theory discoverer overrides duplicated. --- .../SkipAfterFailure/SkipAfterFailureTests.cs | 10 +++++----- .../SkipAfterFailureTests.cs | 18 ++++++++--------- .../CustomXunitTestFrameworkDiscoverer.cs | 6 +++--- .../RetryTestCase.cs | 12 ++--------- .../RetryTheoryDiscoverer.cs | 20 +++++++++++-------- 5 files changed, 31 insertions(+), 35 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs index 08832f01b4..77cd3362e2 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipAfterFailure/SkipAfterFailureTests.cs @@ -11,13 +11,13 @@ namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipAfterFailur { /// /// A retrying test that fails on its first attempt and skips itself on the second. - /// Expected: 1 result, skipped. + /// Expected: 1 result, failed. /// /// - /// A skip is a definitive result, so it supersedes the earlier attempt's failure the same way a - /// pass does. This scenario exists to state that outcome rather than leave it to be discovered: - /// the first attempt's failure is discarded, so a test that fails and then decides it should not - /// have run is reported skipped, not failed. + /// A pass supersedes an earlier failure because it is a claim that the code works - that is what + /// retrying is for. A skip claims nothing: it says the attempt should not have run. Letting it + /// supersede would erase a failure the test really did show, so the skip is discarded instead and + /// the failure stands. /// public class SkipAfterFailureTests { diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs index c8b5d1d46a..152263d5d7 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipAfterFailureTests.cs @@ -12,23 +12,23 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests /// Pins what a retrying test reports when an attempt skips after an earlier one failed. /// /// - /// An attempt that reports a real result supersedes a failure held over from an earlier one, and - /// a runtime skip is a real result. The earlier failure is therefore discarded and the test is - /// reported skipped: the same supersession that lets a retry turn a failure into a pass. This is - /// asserted rather than assumed because the alternative reading - that a failure seen once must - /// always be reported - would be a defensible design and a silent change if it were made by - /// accident. + /// A pass supersedes a failure held over from an earlier attempt, because a pass is a claim that + /// the code works. A skip makes no such claim - it says the attempt should not have run - so + /// letting it supersede would erase a failure the test really did show, and report NotExecuted + /// for a test that failed. The held failure therefore wins and the skip is dropped. This is + /// asserted rather than assumed because the opposite reading - that the newest result always + /// wins - is the simpler implementation and would be a silent regression. /// public class SkipAfterFailureTests { private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipAfterFailure.SkipAfterFailureTests."; /// - /// A test that fails, is retried, and then skips itself is reported skipped exactly once, - /// with the superseded failure discarded rather than also reported. + /// A test that fails, is retried, and then skips itself is reported failed exactly once, with + /// the abstaining skip discarded rather than reported alongside it. /// [Fact] - public void GivenARetryingTestThatSkipsAfterFailing_WhenTheRunCompletes_ThenItIsReportedSkippedOnce() + public void GivenARetryingTestThatSkipsAfterFailing_WhenTheRunCompletes_ThenItIsReportedFailedOnce() { TestAssetRun run = TestAssetRunner.Run("SkipAfterFailure"); diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 470b601d3b..cf2e3e09ec 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -1095,9 +1095,9 @@ private static void ApplyVariantTraits(ITestCase testCase, FixtureArgumentSetTes { // A case whose traits cannot be written is one a filtering leg cannot see, which is // the failure this class exists to prevent -- so fail discovery rather than let it - // through untagged. ApplyVariantDisplayName rejects the same case types, so in the - // current call order this is unreachable; it is kept because this method needs the - // guarantee on its own account, not because of the order it happens to be called in. + // through untagged. ApplyVariantDisplayName rejects the same case types, but it + // returns before that check when the variant closes no argument set, so this guard + // is reached rather than redundant. throw new InvalidOperationException( $"The fixture argument set traits of '{testCase.TestCaseDisplayName}' could not be applied because it is " + $"{testCase.GetType().Name}, not {nameof(XunitTestCase)}. A CI leg selecting on those traits would not run it."); diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 6d890b2cf6..1275235c5f 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -530,16 +530,8 @@ private bool ShouldRetry(Exception ex) // Don't retry assertion failures unless explicitly configured if (ex is XunitException) { - if (!_retryOnAssertionFailure) - { - Trace.WriteLine($"RetryFact: Not retrying XunitException because _retryOnAssertionFailure is false"); - return false; - } - else - { - Trace.WriteLine($"RetryFact: Retrying XunitException because _retryOnAssertionFailure is true"); - return true; - } + Trace.WriteLine($"RetryFact: {(_retryOnAssertionFailure ? "Retrying" : "Not retrying")} XunitException because _retryOnAssertionFailure is {_retryOnAssertionFailure.ToString().ToLowerInvariant()}"); + return _retryOnAssertionFailure; } // Retry everything else (network, timeout, SQL transient, etc.) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs index 2cca82f446..c5954fbda3 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTheoryDiscoverer.cs @@ -27,12 +27,9 @@ protected override async ValueTask> CreateTe ITheoryDataRow dataRow, object[] testMethodArguments) { - var attribute = (RetryTheoryAttribute)theoryAttribute; - - var baseCases = await base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments); - return baseCases - .Select(testCase => WrapTestCase(testCase, attribute)) - .ToArray(); + return WrapAll( + await base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments), + theoryAttribute); } protected override async ValueTask> CreateTestCasesForTheory( @@ -40,9 +37,16 @@ protected override async ValueTask> CreateTe IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute) { - var attribute = (RetryTheoryAttribute)theoryAttribute; + return WrapAll( + await base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute), + theoryAttribute); + } - var baseCases = await base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute); + private static IXunitTestCase[] WrapAll( + IReadOnlyCollection baseCases, + ITheoryAttribute theoryAttribute) + { + var attribute = (RetryTheoryAttribute)theoryAttribute; return baseCases .Select(testCase => WrapTestCase(testCase, attribute)) .ToArray(); From 59ed46acf42be2a11596efeff256db6617f82eb1 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 21:24:33 -0700 Subject: [PATCH 075/105] Settle whether an attempt can report both an abstention and a failure Holding a skip so it cannot erase an earlier failure also holds every message behind it, teardown's failure included, so a review asked whether one attempt could publish two results. It cannot, but only because of something xunit does rather than anything here: a runtime skip and a throwing teardown are collected into one AggregateException and reported as a single failure, so no skip is ever published and the holding path is never entered. That makes the answer worth pinning rather than reasoning out again. The scenario reaches the corner exactly - it was watched failing under a deliberately wrong expectation, with both the skip and the teardown throw present in the run - and asserts the one failure that comes out. --- .../SkipThenTeardownThrowsTests.cs | 56 +++++++++++++++++++ .../SkipThenTeardownThrowsTests.cs | 46 +++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipThenTeardownThrows/SkipThenTeardownThrowsTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipThenTeardownThrowsTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipThenTeardownThrows/SkipThenTeardownThrowsTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipThenTeardownThrows/SkipThenTeardownThrowsTests.cs new file mode 100644 index 0000000000..d4c9f738dd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/SkipThenTeardownThrows/SkipThenTeardownThrowsTests.cs @@ -0,0 +1,56 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipThenTeardownThrows +{ + /// + /// A retrying test that fails, then on its second attempt skips itself and throws from teardown. + /// Expected: 1 result, failed. + /// + /// + /// This is the narrowest corner of the retry state machine. Holding a skip so it cannot erase an + /// earlier failure also holds every message that follows it, teardown's failure among them, which + /// raised the question of whether one attempt could report both. + /// + /// It does not: xUnit collects the skip and the teardown exception into one AggregateException and + /// reports the attempt failed, so no skip is ever published and the abstention-holding path is not + /// entered. The test is therefore retried as the genuine failure it is, and reported failed once. + /// + /// + public class SkipThenTeardownThrowsTests : IDisposable + { + private static int _attempts; + + /// + /// Throws from teardown on every attempt after the first, which is the attempt that skips. + /// + public void Dispose() + { + if (Volatile.Read(ref _attempts) > 1) + { + throw new InvalidOperationException("ASSET: teardown throws after the skip"); + } + } + + /// + /// Throws on the first attempt - a non-assertion exception, so it is retried under the + /// default policy - and skips itself on the second, whose teardown then throws. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10)] + public void FailsThenSkipsWithThrowingTeardown() + { + if (Interlocked.Increment(ref _attempts) == 1) + { + throw new InvalidOperationException("ASSET: first attempt fails"); + } + + Assert.Skip("ASSET: skipped on the second attempt"); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipThenTeardownThrowsTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipThenTeardownThrowsTests.cs new file mode 100644 index 0000000000..bffd02c72d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SkipThenTeardownThrowsTests.cs @@ -0,0 +1,46 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins what a retrying test reports when an attempt both abstains and fails in its teardown, + /// after an earlier attempt had already failed. + /// + /// + /// Holding a skip so it cannot erase an earlier failure also holds every message that follows it, + /// because a reporter finalizes a test at ITestFinished and anything forwarded afterwards is + /// dropped. That raised the question of whether an attempt could arrive carrying both an + /// abstention and a failure of its own, and be reported twice. + /// + /// It cannot, and this records why rather than leaving the reasoning to be redone. xUnit collects + /// the skip and the teardown exception into one AggregateException and publishes a single + /// ITestFailed; no ITestSkipped is ever raised, so the abstention-holding path is not entered at + /// all. That is xUnit's behaviour rather than this code's, which is exactly why it is worth + /// pinning: were it to change, the retry machinery would start seeing a case it has no branch for. + /// + /// + public class SkipThenTeardownThrowsTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.SkipThenTeardownThrows.SkipThenTeardownThrowsTests."; + + /// + /// A test that fails, is retried, then skips itself while its teardown throws is reported + /// failed exactly once - not skipped, not twice, and not absent. + /// + [Fact] + public void GivenARetryingTestThatSkipsAndThrowsFromTeardown_WhenTheRunCompletes_ThenItIsReportedFailedOnce() + { + TestAssetRun run = TestAssetRunner.Run("SkipThenTeardownThrows"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary { [ClassName + "FailsThenSkipsWithThrowingTeardown"] = "Failed" }); + } + } +} From e00cda187cfdfab5a51dda246bf85c25afc62b9e Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 21:37:02 -0700 Subject: [PATCH 076/105] Honour the retry policy for an assertion failure that arrives wrapped The classifier that decides whether a failure is an assertion read only the first entry of the exception chain. An assertion failing inside Task.WhenAll or any parallel helper surfaces as AggregateException wrapping the real one, so the first entry said System.AggregateException, the failure was taken for an ordinary exception, and it was retried - the precise thing RetryOnAssertionFailure = false is set to prevent. The chain is now read whole. The timeout carve-out is applied the same way, because a timeout wrapped in something is still a timeout and still the failure most worth another attempt. A test that fails a wrapped assertion once and would pass on a second attempt now states which happened; it was watched being retried into a pass before the fix, and reported failed after. Also give the unit test legs the zero-test floor the other six legs have, and record why a trait value is deliberately dereferenced unguarded. --- build/jobs/build.yml | 4 ++-- .../Scenarios/RetryPolicy/RetryPolicyTests.cs | 20 +++++++++++++++- .../RetryPolicyTests.cs | 1 + .../FixtureArgumentSetTestMethod.cs | 12 ++++++++++ .../RetryTestCase.cs | 24 +++++++++++++------ 5 files changed, 51 insertions(+), 10 deletions(-) diff --git a/build/jobs/build.yml b/build/jobs/build.yml index e141473d81..ff6555c7d7 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -35,7 +35,7 @@ steps: inputs: command: test projects: '**/*UnitTests/*.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx' + arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --minimum-expected-tests 1' # Disable the task's built-in result publishing so the PublishTestResults@2 task below, # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false @@ -67,7 +67,7 @@ steps: inputs: command: test projects: '**/*UnitTests/*.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"' + arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --minimum-expected-tests 1 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"' # Disable the task's built-in result publishing so the PublishTestResults@2 task below, # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs index 4cf45139fb..957be64f64 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/RetryPolicy/RetryPolicyTests.cs @@ -6,12 +6,13 @@ using System; using System.Threading; using Xunit; +using Xunit.Sdk; namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.RetryPolicy { /// /// Tests that tell the two RetryOnAssertionFailure policies apart. - /// Expected: 2 results, 1 failed and 1 passed. + /// Expected: 3 results, 2 failed and 1 passed. /// /// /// A test that fails on every attempt is reported failed whether or not it was retried, and one @@ -23,6 +24,7 @@ public class RetryPolicyTests { private static int _assertionAttempts; private static int _exceptionAttempts; + private static int _wrappedAssertionAttempts; /// /// An assertion failure is taken to be deterministic, so with the default policy the @@ -51,5 +53,21 @@ public void NonAssertionExceptionUnderTheDefaultPolicy_IsRetried() throw new InvalidOperationException("ASSET: first attempt throws, a second attempt would pass"); } } + + /// + /// An assertion failure that reaches the policy wrapped in an + /// is the same deterministic failure and must be treated as one: reported failed, not retried. + /// Were the wrapper taken at face value it would look like an ordinary exception, be retried, + /// and pass - so the outcome states whether the wrapper was seen through. + /// + [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 10, RetryOnAssertionFailure = false)] + public void WrappedAssertionFailureUnderTheDefaultPolicy_IsNotRetried() + { + if (Interlocked.Increment(ref _wrappedAssertionAttempts) == 1) + { + throw new AggregateException( + new XunitException("ASSET: first attempt fails a wrapped assertion, a second attempt would pass")); + } + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs index eb41f699c2..348279ad3f 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryPolicyTests.cs @@ -38,6 +38,7 @@ public void GivenTestsThatRecoverOnASecondAttempt_WhenTheDefaultPolicyApplies_Th { [ClassName + "AssertionFailureUnderTheDefaultPolicy_IsNotRetried"] = "Failed", [ClassName + "NonAssertionExceptionUnderTheDefaultPolicy_IsRetried"] = "Passed", + [ClassName + "WrappedAssertionFailureUnderTheDefaultPolicy_IsNotRetried"] = "Failed", }); } } diff --git a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs index 37e94ba597..639147086e 100644 --- a/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs +++ b/src/Microsoft.Health.Extensions.Xunit/FixtureArgumentSetTestMethod.cs @@ -88,6 +88,18 @@ private static object[] BuildMethodArguments(IReadOnlyList fixtureAr return result; } + /// + /// Adds a trait naming each fixture argument set value this variant stands for, so a CI leg + /// selecting on those traits can see it. + /// + /// + /// A value here is never null: the only producer builds every through + /// its constructor, and a degenerate combination is rejected before reaching this point. It is + /// deliberately not guarded against anyway. Skipping a value that somehow was null would drop + /// the trait and leave this variant unselectable by the E2E and export legs, which each require + /// a positive match - the test would vanish from them silently. Throwing instead surfaces as a + /// discovery fault, which this assembly turns into a reported failure per argument set. + /// internal void ApplyArgumentSetTraits() { if (_fixtureArguments.Count == 0) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 1275235c5f..914d4d572d 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -649,13 +649,23 @@ public bool QueueMessage(IMessageSinkMessage message) // flakiness these attributes exist to absorb -- and unlike the cases above, it is // not an assertion at all. It derives from System.Exception rather than // XunitException, so it is only ever mistaken for one by way of this match. - string failedExceptionType = failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 - ? failed.ExceptionTypes[0] - : string.Empty; - - IsAssertionFailure = !failedExceptionType.Contains("Timeout", StringComparison.Ordinal) && - (failedExceptionType.Contains("Xunit", StringComparison.Ordinal) || - failedExceptionType.Contains("Assert", StringComparison.Ordinal)); + // A wrapped assertion failure is still an assertion failure. Xunit reports the + // whole exception chain here, outer type first, so reading only the first entry + // classifies AggregateException(XunitException) -- what an assertion failing + // inside Task.WhenAll or a parallel helper produces -- as an ordinary exception + // and retries it, which is exactly what the policy was set to prevent. + string[] failedExceptionTypes = failed.ExceptionTypes != null && failed.ExceptionTypes.Length > 0 + ? failed.ExceptionTypes + : new[] { string.Empty }; + + // The timeout carve-out is applied to the whole chain rather than one entry: a + // timeout wrapped in anything is still a timeout, and still the failure that most + // deserves another attempt. + IsAssertionFailure = + !failedExceptionTypes.Any(type => type != null && type.Contains("Timeout", StringComparison.Ordinal)) && + failedExceptionTypes.Any(type => type != null && + (type.Contains("Xunit", StringComparison.Ordinal) || + type.Contains("Assert", StringComparison.Ordinal))); if (_deferFailures) { From f9c0e2624c9d08124a7e3f76a2563279c5199019 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 21:55:50 -0700 Subject: [PATCH 077/105] Refuse a collection definition that tags classes it does not name This PR started from twelve Cosmos E2E cases that never ran while their leg stayed green: a collection definition carried a Category, xunit v3 applied it to every class in the collection, and an exclusion filter then skipped them. The trait was written in one file and took effect in others, with nothing at the joining classes to say so, which is why nobody saw it. Nothing stopped that being reintroduced. The assembly validations now refuse it, and say what to do instead - the traitless definition class the fix already used. A definition no other class joins is left alone, since its traits then reach only itself. Verified by giving a shared definition a trait: the check names it and every class that would have silently inherited it. Also correct two test remarks that described the retry classifier and the E2E filter shape as they were rather than as they are. --- .../ClassAttributeFaultTests.cs | 9 +- .../RetryTimeoutTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyChecker.cs | 82 +++++++++++++++++++ .../AssemblyValidationsTester.cs | 27 ++++++ .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- .../AssemblyValidationsTests.cs | 8 +- 18 files changed, 218 insertions(+), 20 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs index 03d50d6d20..d5fc233afb 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs @@ -152,9 +152,12 @@ public void GivenAClassLevelFault_WhenALegExcludesOneDataStore_ThenTheOtherMetho } /// - /// The same exclusion written as a query, which is how the E2E legs spell it. A case carrying - /// no value for the trait at all satisfies != as readily as one carrying a different - /// value, so this also pins that the failures carry their values rather than omitting them. + /// The same exclusion written as a query rather than as --filter-not-trait. Both + /// spellings are used by this repository's CI - the integration legs exclude a data store with + /// the flag, and the E2E legs pass a query whose Category exclusions take this form. A + /// case carrying no value for the trait at all satisfies != as readily as one carrying a + /// different value, so this also pins that the failures carry their values rather than + /// omitting them. /// [Fact] public void GivenAClassLevelFault_WhenAQueryExcludesOneDataStore_ThenOnlyTheOtherFailuresAreReported() diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs index 860212fd57..9b5ccd815d 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTimeoutTests.cs @@ -14,9 +14,11 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests /// /// A timeout is the failure a retry harness exists for: a test that runs long because a /// dependency was slow once is the definition of flaky. It reaches the retry decision as - /// Xunit.Sdk.TestTimeoutException, and the decision is made by matching "Xunit" and - /// "Assert" against the exception type name, so a timeout is at risk of being read as a - /// deterministic assertion failure and denied the attempts it was configured for. + /// Xunit.Sdk.TestTimeoutException, whose name contains "Xunit" and so matches the + /// substring test that classifies deterministic assertion failures - which are deliberately not + /// retried. A timeout is carved out of that match for exactly this reason, and these tests are + /// what would notice if the carve-out were removed: without it a timeout is denied every attempt + /// it was configured for. /// public class RetryTimeoutTests { diff --git a/src/Microsoft.Health.Fhir.Azure.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Azure.UnitTests/AssemblyValidationsTests.cs index 17db94a612..0e86beda26 100644 --- a/src/Microsoft.Health.Fhir.Azure.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Azure.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/AssemblyValidationsTests.cs index 3129ce935e..94143674ee 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyValidationsTests.cs index 51fccee340..217879340f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/AssemblyValidationsTests.cs index 00e30ede2a..591690d2c7 100644 --- a/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.SchemaManager.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyValidationsTests.cs index 65f75a5b47..cec3ebce3d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyValidationsTests.cs index 0dc8b17c31..31ecc88237 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs index c1e8771303..34f115b2eb 100644 --- a/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/AssemblyValidationsTests.cs index 87eb36697f..6d9525446e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs index f4e2e82517..f5d0a4787c 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs @@ -52,6 +52,88 @@ public static IEnumerable ScanTestsLookingForMissingTrait(Assembly assembl return notComplianceTestClass; } + /// + /// Looks for collection definitions that carry traits and are shared with other classes. + /// + /// + /// A trait on a [CollectionDefinition] class is applied to every class that joins that + /// collection, which is not obvious at either end: the trait is written in one file and takes + /// effect in others, and nothing at the joining class says so. This repository's CI legs select + /// on traits, so a collection that quietly adds a Category can move its members out of a + /// leg that excludes that category - and a leg that runs fewer tests than it should still + /// passes. That is not hypothetical; it is what this check was written after. + /// + /// A definition that carries traits but that no other class joins is left alone: it applies + /// them only to itself, which is the same as writing them on the class. The fix when this does + /// fire is the pattern already used elsewhere here - move the collection name onto a separate, + /// traitless definition class, and leave the traits on the test class that declared them. + /// + /// + /// Assembly under analysis. + /// The offending definition types, each with the classes that join its collection. + public static IReadOnlyList<(Type Definition, IReadOnlyList Members)> ScanForTraitCarryingSharedCollectionDefinitions(Assembly assembly) + { + EnsureArg.IsNotNull(assembly, nameof(assembly)); + + Type[] classes = assembly.GetTypes().Where(t => t.IsClass).ToArray(); + + var offenders = new List<(Type, IReadOnlyList)>(); + + foreach (Type definition in classes) + { + CustomAttributeData definitionAttribute = definition.CustomAttributes + .FirstOrDefault(a => a.AttributeType == typeof(CollectionDefinitionAttribute)); + + if (definitionAttribute == null || + !definition.CustomAttributes.Any(a => a.AttributeType == typeof(TraitAttribute))) + { + continue; + } + + // A collection definition can name its collection with a string or, since v3, be + // referenced by its own type. Both spellings have to be matched to find the members. + string collectionName = definitionAttribute.ConstructorArguments.Count > 0 + ? definitionAttribute.ConstructorArguments[0].Value as string + : null; + + List members = classes + .Where(candidate => candidate != definition && JoinsCollection(candidate, collectionName, definition)) + .ToList(); + + if (members.Count > 0) + { + offenders.Add((definition, members)); + } + } + + return offenders; + } + + private static bool JoinsCollection(Type candidate, string collectionName, Type definition) + { + foreach (CustomAttributeData attribute in candidate.CustomAttributes.Where(a => a.AttributeType == typeof(CollectionAttribute))) + { + if (attribute.ConstructorArguments.Count == 0) + { + continue; + } + + object value = attribute.ConstructorArguments[0].Value; + + if (value is string name && collectionName != null && string.Equals(name, collectionName, StringComparison.Ordinal)) + { + return true; + } + + if (value is Type referenced && referenced == definition) + { + return true; + } + } + + return false; + } + private static bool IsTestClass(Type type) { if (!type.IsClass) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs index e54bda668d..204f1da07a 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs @@ -27,6 +27,33 @@ public static void EnsureAllTestsHaveOwningTeamTrait(Assembly assembly) AssertTestClasses(assembly, Traits.OwningTeam, types); } + /// + /// Fails when a collection definition carrying traits is shared with other classes, because + /// those classes silently acquire the traits and the CI legs select on them. + /// + /// Assembly under analysis. + public static void EnsureNoSharedCollectionDefinitionCarriesTraits(Assembly assembly) + { + IReadOnlyList<(Type Definition, IReadOnlyList Members)> offenders = + AssemblyChecker.ScanForTraitCarryingSharedCollectionDefinitions(assembly); + + if (offenders.Count == 0) + { + return; + } + + var stringBuilder = new StringBuilder(); + foreach ((Type definition, IReadOnlyList members) in offenders) + { + stringBuilder.AppendLine($"{definition} applies its traits to: {string.Join(", ", members.Select(m => m.ToString()))}"); + } + + Assert.Fail( + $"Assembly '{assembly}' has collection definitions whose traits are applied to other classes. Those classes gain traits nothing at their declaration mentions, " + + $"and a CI leg selecting on traits will then run a different set of tests than it appears to - without failing. Move the collection name onto a separate traitless " + + $"definition class and leave the traits where they were declared.{Environment.NewLine}{stringBuilder}"); + } + private static void AssertTestClasses(Assembly currentAssembly, string traitName, IEnumerable types) { if (types == null || !types.Any()) diff --git a/src/Microsoft.Health.TaskManagement.UnitTests/AssemblyValidationsTests.cs b/src/Microsoft.Health.TaskManagement.UnitTests/AssemblyValidationsTests.cs index 919877d101..28b001a685 100644 --- a/src/Microsoft.Health.TaskManagement.UnitTests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.TaskManagement.UnitTests/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Crucible/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Crucible/AssemblyValidationsTests.cs index 822c474a15..5b227b2141 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Crucible/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Crucible/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs index 742acc5ab9..30d8e73211 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs index 7d5192bc53..5aa6358224 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs index 92e92b1d6f..45a261e56d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Smart/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Smart/AssemblyValidationsTests.cs index 0e124cf958..9aae72811b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Smart/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Smart/AssemblyValidationsTests.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -27,5 +27,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveOwningTeamTrait() { AssemblyValidationsTester.EnsureAllTestsHaveOwningTeamTrait(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionCarriesTraits() + { + AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); + } } } From b030602b7583a9d4b1f487784e8fcfaeb04823f2 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 22:08:46 -0700 Subject: [PATCH 078/105] Take a cancellation-shaped failure for cancellation only when cancelled A class is expanded by running the attributes declared on it, and an attribute that awaits anything with a timeout throws TaskCanceledException of its own accord. Discovery is not handed a cancellation token, so the exception type was the only thing separating that from a real Ctrl+C, and the type won: the exception was rethrown, the class was dropped, and the run ended green with its tests silently absent - the one outcome reporting discovery faults exists to prevent, reached through the door left open for cancellation. Ask the run whether it was actually cancelled instead, reading the ambient token xunit keeps. A cancelled run still goes back untouched, so Ctrl+C does not invent a red test naming whichever class was mid-expansion. The new scenario asset pins this end to end: with the type-only rule the inner run discovers nothing at all and exits zero. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...cellationLookalikeArgumentSetsAttribute.cs | 35 +++++++++++ .../CancellationLookalikeFaultTests.cs | 31 ++++++++++ .../CancellationLookalikeFaultTests.cs | 61 +++++++++++++++++++ .../DiscoveryFaultDispositionTests.cs | 43 +++++++++++-- .../CustomXunitTestFrameworkDiscoverer.cs | 40 ++++++++++-- 5 files changed, 201 insertions(+), 9 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeArgumentSetsAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeFaultTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/CancellationLookalikeFaultTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeArgumentSetsAttribute.cs new file mode 100644 index 0000000000..8fe3869992 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeArgumentSetsAttribute.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancellationLookalikeFault +{ + /// + /// An argument set attribute whose constructor throws an exception shaped like cancellation. + /// + /// + /// is what anything that awaits with a timeout throws, so an + /// attribute that reaches out to something while being constructed can raise it without the run + /// having been cancelled at all. Discovery is not handed a cancellation token, so the exception's + /// type is the only thing that distinguishes this from a real Ctrl+C - which is why the type + /// alone must not be allowed to decide. Taken for cancellation, this is rethrown, the class is + /// dropped, and the run ends green with its tests missing. + /// + public sealed class CancellationLookalikeArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the + /// class, which never completes. + /// + /// The data stores the class would have been expanded over. + public CancellationLookalikeArgumentSetsAttribute(AssetDataStore dataStore) + : base(dataStore) + { + throw new TaskCanceledException("This argument set attribute gave up waiting."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeFaultTests.cs new file mode 100644 index 0000000000..951ee7fbb6 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancellationLookalikeFault/CancellationLookalikeFaultTests.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancellationLookalikeFault +{ + /// + /// A class whose argument set attribute fails with an exception shaped like cancellation. + /// + /// + /// Nothing cancelled this run, so the failure belongs to the class and has to be reported as one. + /// Reading it as a cancelled run instead would rethrow it and drop the class, and because a + /// cancelled run is not a fault, the run would end green with these tests silently absent. + /// + [CancellationLookalikeArgumentSets(AssetDataStore.Sql)] + public class CancellationLookalikeFaultTests + { + /// + /// Never runs: the class carries an argument set attribute that cannot be constructed. + /// + [Fact] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/CancellationLookalikeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CancellationLookalikeFaultTests.cs new file mode 100644 index 0000000000..df1af07eea --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CancellationLookalikeFaultTests.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a discovery failure that is shaped like cancellation without the run having been + /// cancelled. + /// + /// + /// A cancelled run is rightly not reported as a fault, because a class that happened to be mid + /// expansion when Ctrl+C arrived has nothing wrong with it. That exclusion is only safe while it + /// recognises cancellation by the run actually having been cancelled. Recognising it by the + /// exception's type instead hands every class a way out of being reported: expansion runs the + /// attributes declared on the class, and an attribute that awaits anything with a timeout throws + /// of its own accord. Treated as + /// cancellation, that drops the class and leaves the run green with its tests missing, which is + /// the one outcome reporting discovery faults exists to prevent. + /// + public class CancellationLookalikeFaultTests + { + private const string ErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancellationLookalikeFault.CancellationLookalikeFaultTests.NeverRuns (fixture argument set discovery: Sql)"; + + /// + /// The failure is reported as a test case and the run fails, rather than the class being + /// dropped in the belief that the run was cancelled. + /// + [Fact] + public void GivenAnAttributeThatFailsLikeACancellation_WhenNothingWasCancelled_ThenTheFaultIsReported() + { + TestAssetRun run = TestAssetRunner.Run("CancellationLookalikeFault"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The cause has to survive into the output, or whoever reads the results is told a test + /// failed without being told the class never expanded. + /// + [Fact] + public void GivenAnAttributeThatFailsLikeACancellation_WhenItIsDiscovered_ThenTheCauseIsWrittenToTheOutput() + { + TestAssetRun run = TestAssetRunner.Run("CancellationLookalikeFault"); + + Assert.Contains("This argument set attribute gave up waiting.", run.Output, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs index d6d5f1dfa3..0a166d5c04 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultDispositionTests.cs @@ -33,7 +33,8 @@ public void GivenACancellation_WhenExpansionEnds_ThenItIsRethrownRatherThanRepor { Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( new OperationCanceledException(), - isCallbackFailure: false)); + isCallbackFailure: false, + isCancellationRequested: true)); } /// @@ -46,7 +47,8 @@ public void GivenATaskCancellation_WhenExpansionEnds_ThenItIsRethrownRatherThanR { Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( new TaskCanceledException(), - isCallbackFailure: false)); + isCallbackFailure: false, + isCancellationRequested: true)); } /// @@ -58,7 +60,8 @@ public void GivenAFailureFromTheCallback_WhenExpansionEnds_ThenItIsRethrownRathe { Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( new InvalidOperationException("the sink threw"), - isCallbackFailure: true)); + isCallbackFailure: true, + isCancellationRequested: false)); } /// @@ -71,7 +74,36 @@ public void GivenAnExpansionFailure_WhenExpansionEnds_ThenItIsReportedAsATest() { Assert.False(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( new InvalidOperationException("the argument set was misdeclared"), - isCallbackFailure: false)); + isCallbackFailure: false, + isCancellationRequested: false)); + } + + /// + /// An out of a run nobody cancelled did not come from + /// the runner: expansion reads the attributes declared on the class, and one of those is free + /// to throw it for reasons of its own. Rethrowing it would drop the class and leave the run + /// green with its tests missing, which is what reporting discovery faults exists to prevent. + /// + [Fact] + public void GivenACancellationExceptionButNoCancellation_WhenExpansionEnds_ThenItIsReportedAsATest() + { + Assert.False(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new OperationCanceledException("an attribute gave up waiting"), + isCallbackFailure: false, + isCancellationRequested: false)); + } + + /// + /// The derived type gets the same treatment, because it is the one an attribute that awaits + /// anything with a timeout will actually throw. + /// + [Fact] + public void GivenATaskCancellationButNoCancellation_WhenExpansionEnds_ThenItIsReportedAsATest() + { + Assert.False(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( + new TaskCanceledException("an attribute gave up waiting"), + isCallbackFailure: false, + isCancellationRequested: false)); } /// @@ -83,7 +115,8 @@ public void GivenACancellationFromTheCallback_WhenExpansionEnds_ThenItIsRethrown { Assert.True(CustomXunitTestFrameworkDiscoverer.ShouldRethrowRatherThanReport( new OperationCanceledException(), - isCallbackFailure: true)); + isCallbackFailure: true, + isCancellationRequested: true)); } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index cf2e3e09ec..da28da63d3 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -41,7 +41,7 @@ protected override async ValueTask FindTestsForType(IXunitTestClass testCl { return await FindTestsForTypeCore(testClass, discoveryOptions, tracker); } - catch (Exception ex) when (ShouldRethrowRatherThanReport(ex, tracker.IsCallbackFailure(ex))) + catch (Exception ex) when (ShouldRethrowRatherThanReport(ex, tracker.IsCallbackFailure(ex), IsRunCancelled())) { // Either the run was cancelled, or the caller's own sink threw. Neither is a failure // to expand fixture argument sets, so neither is reported as one. @@ -129,6 +129,15 @@ private static MethodInfo[] TryReadMethodsToReportAgainst(IXunitTestClass testCl /// with nothing wrong with it. A failure raised by xunit's own callback is not ours either, /// and the only way to report it would be to hand that same callback another test case. /// + /// Cancellation is recognised by asking whether the run was actually cancelled, not by the + /// exception's type alone. The type on its own is not evidence: expansion runs the attributes + /// declared on the class, and one of those is free to throw + /// for reasons of its own - a timeout inside an + /// attribute that reaches out to something, say. Taking that for a cancelled run rethrows it, + /// drops the class, and leaves the run green with its tests missing, which is the exact outcome + /// this mechanism exists to prevent, reached through the one door left open for it. + /// + /// /// This is a separate function so that the decision can be pinned directly. Reached only /// through a catch clause, the cancellation case in particular needs a run to be cancelled at /// the moment a class is being expanded, which no scenario can arrange reliably. @@ -136,9 +145,32 @@ private static MethodInfo[] TryReadMethodsToReportAgainst(IXunitTestClass testCl /// /// The exception that ended the expansion. /// Whether the exception came from xunit's own callback. + /// Whether the run was actually cancelled. /// true to rethrow; false to report the failure as a test case. - internal static bool ShouldRethrowRatherThanReport(Exception exception, bool isCallbackFailure) => - exception is OperationCanceledException || isCallbackFailure; + internal static bool ShouldRethrowRatherThanReport(Exception exception, bool isCallbackFailure, bool isCancellationRequested) => + (exception is OperationCanceledException && isCancellationRequested) || isCallbackFailure; + + /// + /// Reports whether the run has been cancelled, as far as it can be determined here. + /// + /// + /// Discovery is not handed a cancellation token, so this reads the ambient one xunit keeps for + /// the run. If it cannot be read the answer is no, which reports a fault rather than dropping a + /// class: a spurious red test naming a class is visible and can be argued with, while a class + /// silently absent from a green run is neither. + /// + private static bool IsRunCancelled() + { + try + { + return TestContext.Current?.CancellationToken.IsCancellationRequested ?? false; + } + catch (Exception ex) + { + Console.WriteLine($"[FixtureArgumentSets] WARNING: the run's cancellation state could not be read while handling a discovery failure, so it is treated as a fault. {ex}"); + return false; + } + } /// /// Determines whether xunit would treat a method as a test. @@ -846,7 +878,7 @@ private async ValueTask FindTestsForTypeCore(IXunitTestClass testClass, IT { continueDiscovery = await FindTestsForMethodCore(testClass, method, attribute, GetMethodArgumentSetsAttribute(method), classLevelOpenParameterSets, classLevelClosedParameterSets, discoveryOptions, tracker); } - catch (Exception ex) when (!ShouldRethrowRatherThanReport(ex, tracker.IsCallbackFailure(ex))) + catch (Exception ex) when (!ShouldRethrowRatherThanReport(ex, tracker.IsCallbackFailure(ex), IsRunCancelled())) { continueDiscovery = await ReportDiscoveryFault( testClass, From 4fd1fda297c4838135139fe4ad93eac9f18708d8 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 22:13:25 -0700 Subject: [PATCH 079/105] Recognise an assertion failure that shares its aggregate with others The retry attempt classified a failure by unwrapping aggregates that held exactly one exception. Anything awaited through Task.WhenAll, run by a parallel helper, or accompanied by a throwing teardown produces an aggregate holding several, and the walk stopped at that fork: the assertion inside went unnoticed and the failure was retried, which is what RetryOnAssertionFailure was set to prevent. Search the whole exception tree instead, so the attempt agrees with the message bus, which already reads the full chain of reported type names. The two decide the same question, and while the bus almost always gets there first, a rarely taken path that disagrees only ever surfaces as an unexplained retry. No timeout carve-out is needed here, unlike in the bus: that one matches type names and has to exclude the timeout whose name contains "Xunit", while this matches types, and the timeout derives from Exception. The tests pin that. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryAssertionClassificationTests.cs | 120 ++++++++++++++++++ .../RetryTestCase.cs | 73 ++++++++++- 2 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryAssertionClassificationTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryAssertionClassificationTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryAssertionClassificationTests.cs new file mode 100644 index 0000000000..a4e7481206 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryAssertionClassificationTests.cs @@ -0,0 +1,120 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Xunit; +using Xunit.Sdk; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers how a failure is recognised as an assertion failure when the retry attempt itself has + /// to classify it. + /// + /// + /// RetryOnAssertionFailure is what tells a retrying test not to spend attempts on a failure + /// that will fail the same way every time. It can only do that if an assertion failure is still + /// recognised once something has wrapped it, which is the normal shape rather than an exotic one: + /// anything awaited through , a parallel helper, or a teardown + /// that throws alongside the test arrives as an . Two places make + /// this call - the message bus, from the reported type names, and the attempt itself, from the + /// exception - and they have to agree, or whether the policy is honoured comes down to which one + /// happened to see the failure. + /// + public class RetryAssertionClassificationTests + { + /// + /// The plain case: an assertion failure with nothing around it. + /// + [Fact] + public void GivenABareAssertionFailure_WhenItIsClassified_ThenItIsAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure(new XunitException("assertion"))); + } + + /// + /// One wrapper deep, which is what awaiting a single faulted task produces. + /// + [Fact] + public void GivenAnAssertionFailureWrappedAlone_WhenItIsClassified_ThenItIsAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure( + new AggregateException(new XunitException("assertion")))); + } + + /// + /// The case that unwrapping only single-inner aggregates missed. A test that awaits several + /// tasks, or whose teardown throws alongside the assertion, produces an aggregate holding more + /// than one exception, and the assertion is no less deterministic for having company. + /// + [Fact] + public void GivenAnAssertionFailureAmongOthers_WhenItIsClassified_ThenItIsStillAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure( + new AggregateException( + new InvalidOperationException("the connection dropped"), + new XunitException("assertion")))); + } + + /// + /// Nesting is not always flat: an aggregate can hold an aggregate. + /// + [Fact] + public void GivenAnAssertionFailureNestedDeeply_WhenItIsClassified_ThenItIsStillAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure( + new AggregateException( + new InvalidOperationException("the connection dropped"), + new AggregateException(new XunitException("assertion"))))); + } + + /// + /// An assertion reached through an ordinary inner exception rather than an aggregate. + /// + [Fact] + public void GivenAnAssertionFailureAsAnInnerException_WhenItIsClassified_ThenItIsStillAnAssertionFailure() + { + Assert.True(RetryTestCase.ContainsAssertionFailure( + new InvalidOperationException("wrapped", new XunitException("assertion")))); + } + + /// + /// The other direction matters just as much. Classifying a transient failure as an assertion + /// would cost it every retry it was given the attribute for. + /// + [Fact] + public void GivenNoAssertionFailureAnywhere_WhenItIsClassified_ThenItIsNotAnAssertionFailure() + { + Assert.False(RetryTestCase.ContainsAssertionFailure( + new AggregateException( + new InvalidOperationException("the connection dropped"), + new TimeoutException("it took too long")))); + } + + /// + /// A timeout needs no carve-out here, unlike in the message bus, which matches on type names + /// and so has to exclude the one whose name contains "Xunit". This classifies by type, and the + /// invariant that makes that safe is pinned here: TestTimeoutException does not derive + /// from , so a timeout stays retryable - which is the whole point, + /// as a test that ran long because a dependency was slow once is what retrying is for. The + /// type cannot be constructed from here, so the relationship is asserted rather than exercised. + /// + [Fact] + public void GivenTheTimeoutType_WhenItIsInspected_ThenItIsNotAnAssertionType() + { + Assert.False(typeof(XunitException).IsAssignableFrom(typeof(TestTimeoutException))); + } + + /// + /// Nothing to classify is not an assertion failure, and must not throw on the way to saying so. + /// + [Fact] + public void GivenNoException_WhenItIsClassified_ThenItIsNotAnAssertionFailure() + { + Assert.False(RetryTestCase.ContainsAssertionFailure(null)); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 914d4d572d..592edaa004 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -512,6 +512,23 @@ protected override void Deserialize(IXunitSerializationInfo info) /// /// Determines if an exception should trigger a retry. /// + /// + /// The whole exception tree is searched, not just the outermost exception or a chain of + /// single-inner aggregates. An assertion that fails inside Task.WhenAll, a parallel + /// helper, or any teardown that runs alongside it arrives wrapped in an + /// that may hold several inner exceptions, and stopping at + /// the first fork would classify it as an ordinary exception and retry it - the very thing + /// was set to prevent. This matches how the message bus + /// classifies the same failure, which reads the full chain of reported type names; the two + /// have to agree, or whether the policy is honoured would depend on which of them happened to + /// see the failure first. + /// + /// A timeout needs no carve-out here. The bus classifies by type name and so has to exclude + /// Xunit.Sdk.TestTimeoutException explicitly, but that type derives from + /// rather than , so it never matches the + /// test below and is retried, which is what a timeout deserves. + /// + /// private bool ShouldRetry(Exception ex) { // If exception is null, we should retry (something went wrong with exception capture) @@ -521,14 +538,8 @@ private bool ShouldRetry(Exception ex) return true; // Retry when we can't determine the exception type } - // Unwrap aggregate exceptions - while (ex is AggregateException aggEx && aggEx.InnerExceptions.Count == 1) - { - ex = aggEx.InnerException; - } - // Don't retry assertion failures unless explicitly configured - if (ex is XunitException) + if (ContainsAssertionFailure(ex)) { Trace.WriteLine($"RetryFact: {(_retryOnAssertionFailure ? "Retrying" : "Not retrying")} XunitException because _retryOnAssertionFailure is {_retryOnAssertionFailure.ToString().ToLowerInvariant()}"); return _retryOnAssertionFailure; @@ -539,6 +550,54 @@ private bool ShouldRetry(Exception ex) return true; } + /// + /// Reports whether an exception is, or contains anywhere within it, an assertion failure. + /// + /// + /// Separate and internal so the decision can be pinned directly. In a real run the message bus + /// almost always classifies the failure first, because xunit reports failures through the bus + /// rather than letting them out of the attempt, which leaves this path reachable but rarely + /// taken - and a rarely taken path that disagrees with the common one is exactly the kind of + /// difference that only shows up as an unexplained retry long after the fact. + /// + /// The exception to classify. + /// true if an assertion failure appears anywhere in the exception tree. + internal static bool ContainsAssertionFailure(Exception exception) => + exception != null && EnumerateExceptionTree(exception).Any(inner => inner is XunitException); + + /// + /// Walks an exception and everything nested inside it, including every branch of an + /// . + /// + /// The exception to walk. + /// The exception followed by each exception nested within it. + private static IEnumerable EnumerateExceptionTree(Exception exception) + { + var pending = new Stack(); + pending.Push(exception); + + while (pending.Count > 0) + { + Exception current = pending.Pop(); + yield return current; + + if (current is AggregateException aggregate) + { + foreach (Exception inner in aggregate.InnerExceptions) + { + if (inner != null) + { + pending.Push(inner); + } + } + } + else if (current.InnerException != null) + { + pending.Push(current.InnerException); + } + } + } + /// /// Message bus that defers failure messages (ITestFailed) so that an attempt which is /// about to be retried leaves no trace in the test results. From 70c12cc39a2d144e3ce487ae2b3d69c34359e977 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 22:21:05 -0700 Subject: [PATCH 080/105] Give a fault the traits of the collection its class joined Xunit v3 gives every member of a collection the traits its definition declares, which is the propagation this branch exists because of. When one trait attribute throws, the traits of the failure standing in for the lost tests are gathered one attribute at a time instead, and that fallback read only the class and the method - so a trait that came from the collection was silently dropped. The export and E2E legs select positively, so a leg naming that trait matched nothing and reported success with the method missing. The new scenario shows it: with the collection left out, the filtered run discovers no tests at all. The fallback now reads the collection definition first, in xunit's own order. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../CollectionTraitFallbackCollection.cs | 24 +++++++ .../CollectionTraitFallbackTests.cs | 41 +++++++++++ .../CollectionTraitFallbackTests.cs | 68 +++++++++++++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 32 ++++++++- 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackCollection.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitFallbackTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackCollection.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackCollection.cs new file mode 100644 index 0000000000..a8c099c8cf --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackCollection.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraitFallback +{ + /// + /// A collection definition that declares a trait its members inherit, and declares it nowhere else. + /// + /// + /// Xunit v3 gives every member of a collection the traits its definition carries, which is how a + /// class can end up filtered on a trait that appears nowhere in its own source. That propagation + /// is the whole reason this scenario exists: it makes the collection the only place the trait can + /// be read from. + /// + [CollectionDefinition("CollectionTraitFallbackProbe")] + [Trait("Category", "CollectionTraitFallbackProbe")] + public class CollectionTraitFallbackCollection + { + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackTests.cs new file mode 100644 index 0000000000..0f72dbe41f --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CollectionTraitFallback/CollectionTraitFallbackTests.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraitFallback +{ + /// + /// A class whose discovery faults, whose traits cannot be read in one go, and whose only trait + /// comes from the collection it joined. Expected: 1 result, failed, carrying that trait. + /// + /// + /// The throwing trait attribute forces the failure's traits to be gathered one attribute at a + /// time. Gathering them from the class and the method alone would silently drop the collection's + /// trait, because the class does not declare it - and the export and E2E legs select positively, + /// so a leg naming that trait would match nothing and report success with the method missing. + /// That is the same silence reporting discovery faults exists to break, so the fallback has to be + /// no quieter than the ordinary read it stands in for. + /// + [Collection("CollectionTraitFallbackProbe")] + public class CollectionTraitFallbackTests + { + /// + /// Never runs: it declares two fixture argument set attributes, so which one applies cannot be + /// determined. It also carries a trait attribute that throws. + /// + [Fact] + [ThrowingTrait] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitFallbackTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitFallbackTests.cs new file mode 100644 index 0000000000..0fb3aaaef9 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/CollectionTraitFallbackTests.cs @@ -0,0 +1,68 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a discovery failure whose traits have to be gathered one attribute at a time, and whose + /// only trait comes from the collection its class joined. + /// + /// + /// Xunit v3 gives every member of a collection the traits its definition declares, so a class can + /// be selected on a trait that appears nowhere in its own source - the propagation this PR exists + /// because of. When one trait attribute throws, the failure standing in for the lost method has + /// its traits gathered attribute by attribute instead, and that gathering has to reach as far as + /// the ordinary read it replaces. Reading only the class and the method drops the collection's + /// trait, and a leg selecting positively on it would then match nothing and report success with + /// the method missing. + /// + public class CollectionTraitFallbackTests + { + private const string OneDimensionErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraitFallback.CollectionTraitFallbackTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string TwoDimensionErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CollectionTraitFallback.CollectionTraitFallbackTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + + /// + /// Unfiltered, the failure is reported and the run fails. + /// + [Fact] + public void GivenAFaultWhoseTraitsMustBeGatheredSeparately_WhenItIsDiscovered_ThenTheFaultIsReported() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraitFallback"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [OneDimensionErrorCaseName] = "Failed", + [TwoDimensionErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The shape that matters. A leg naming the collection's trait still has to see the failure, + /// which it can only do if the fallback read the collection definition as well as the class. + /// + [Fact] + public void GivenAFaultWhoseTraitsMustBeGatheredSeparately_WhenALegSelectsTheCollectionTrait_ThenThatLegStillSeesTheFailure() + { + TestAssetRun run = TestAssetRunner.Run("CollectionTraitFallback", filterTrait: "Category=CollectionTraitFallbackProbe"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [OneDimensionErrorCaseName] = "Failed", + [TwoDimensionErrorCaseName] = "Failed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index da28da63d3..116be9e607 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -720,7 +720,13 @@ void Add(string key, string value) /// Records one trait name and value. private static void AddFaultTraitsPerAttribute(XunitTestMethod anchor, Action add) { - foreach (MemberInfo declaration in new MemberInfo[] { anchor.TestClass?.Class, anchor.Method }) + // The collection definition is read first, and for the same reason it is read at all: + // xunit v3 gives every member of a collection the traits its definition declares, so a + // class can carry a trait that appears nowhere on the class itself. Leaving it out here + // would make the fallback quieter than the ordinary read it stands in for - the failure + // would lose exactly those traits, and a leg selecting on one would not see it. The order + // matches xunit's own: collection, then class, then method. + foreach (MemberInfo declaration in new MemberInfo[] { ReadCollectionDefinition(anchor), anchor.TestClass?.Class, anchor.Method }) { if (declaration == null) { @@ -758,6 +764,30 @@ private static void AddFaultTraitsPerAttribute(XunitTestMethod anchor, Action + /// Reads the collection definition class the method's collection was declared by, if any. + /// + /// + /// A collection assembled without a definition class has none, which is the ordinary case for + /// a class that never joined one. Read defensively because this runs while handling a failure + /// that may itself have come from a type that cannot be loaded. + /// + /// The method the failure is reported against. + /// The collection definition class, or null if there is not one. + private static Type ReadCollectionDefinition(XunitTestMethod anchor) + { + try + { + return anchor.TestClass?.TestCollection?.CollectionDefinition; + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the collection definition of '{anchor.TestClass?.TestClassName}' could not be read, so a filter naming a trait it declares may not select the failure standing in for its tests. {ex}"); + return null; + } + } + /// /// Builds the part of a discovery failure's unique ID that identifies the method it is /// reported against. From 28d4b7eff26db30380a0bd51c8319edaff24279d Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 22:23:07 -0700 Subject: [PATCH 081/105] Name the variant in retry diagnostics, not just the method Every retry message named the test by class and method, which many test cases share: a method expanded over fixture argument sets, and a theory with several rows, all report the same name. These messages are read when something has already gone wrong, and one that cannot say which variant retried, or which one gave up after exhausting its attempts, is of little use at that point. Use the display name, which carries the variant and the row, falling back to the class and method when it is not set. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryTestCase.cs | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 592edaa004..64adf31115 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -84,6 +84,22 @@ public RetryTestCase( _retryOnAssertionFailure = retryOnAssertionFailure; } + /// + /// Names the test in diagnostics. + /// + /// + /// The display name is used rather than the class and method names because those are not + /// unique: a method expanded over fixture argument sets, or a theory with several rows, + /// produces many test cases sharing them. Retry diagnostics are read when something has + /// already gone wrong, and messages that cannot say which variant retried, or failed after + /// exhausting its attempts, are worth little at that point. The display name carries the + /// variant and the row, and falls back to the class and method if it is not set. + /// + private string TestDescription => + !string.IsNullOrEmpty(TestCaseDisplayName) + ? TestCaseDisplayName + : $"{TestMethod?.TestClass?.TestClassName}.{TestMethod?.MethodName}"; + /// /// Runs the test case, retrying transient failures up to the configured number of attempts. /// @@ -103,7 +119,7 @@ public async ValueTask Run( { // Trace output is only visible to an attached debugger, never in CI logs. Anything // that needs to show up in a pipeline log has to go to Console instead. - Trace.WriteLine($"RetryFact starting test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' with MaxRetries={_maxRetries}, DelayMs={_delayMs}, RetryOnAssertionFailure={_retryOnAssertionFailure}"); + Trace.WriteLine($"RetryFact starting test '{TestDescription}' with MaxRetries={_maxRetries}, DelayMs={_delayMs}, RetryOnAssertionFailure={_retryOnAssertionFailure}"); var runSummary = new RunSummary { Total = 1 }; Exception lastException = null; @@ -118,7 +134,7 @@ public async ValueTask Run( { var isLastAttempt = attempt == _maxRetries; - Trace.WriteLine($"RetryFact attempt {attempt}/{_maxRetries} for test '{TestMethod.MethodName}'"); + Trace.WriteLine($"RetryFact attempt {attempt}/{_maxRetries} for test '{TestDescription}'"); // Create a fresh aggregator for each attempt var attemptAggregator = new ExceptionAggregator(); @@ -161,8 +177,8 @@ public async ValueTask Run( Console.WriteLine( outcome == NoResultOutcome.ReportNothing - ? $"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted), and no earlier attempt observed a failure, so there is nothing to report for it." - : $"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted). Reporting the last observed failure rather than a pass."); + ? $"[RetryFact] Test '{TestDescription}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted), and no earlier attempt observed a failure, so there is nothing to report for it." + : $"[RetryFact] Test '{TestDescription}' reported no result for attempt {attempt}/{_maxRetries} (the run was cancelled or aborted). Reporting the last observed failure rather than a pass."); // At most one bus is replayed. A failure this attempt saw supersedes anything held // over from an earlier one, and replaying both would publish two results for a @@ -210,7 +226,7 @@ public async ValueTask Run( (pendingFailureBus?.HasDeferredFailure ?? false)) { Console.WriteLine( - $"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' skipped itself on attempt {attempt}/{_maxRetries} after failing earlier. Reporting the failure rather than the skip."); + $"[RetryFact] Test '{TestDescription}' skipped itself on attempt {attempt}/{_maxRetries} after failing earlier. Reporting the failure rather than the skip."); // The abstention was held rather than published, so dropping it now leaves // the replayed failure as the test's only result. @@ -231,9 +247,9 @@ public async ValueTask Run( if (summary.Failed == 0) { // Test passed - success message already went through to Test Explorer - Trace.WriteLine($"RetryFact test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' passed on attempt {attempt}/{_maxRetries}"); + Trace.WriteLine($"RetryFact test '{TestDescription}' passed on attempt {attempt}/{_maxRetries}"); messageBus.QueueMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' passed on attempt {attempt}/{_maxRetries}")); + new DiagnosticMessage($"[RetryFact] Test '{TestDescription}' passed on attempt {attempt}/{_maxRetries}")); runSummary.Failed = 0; @@ -285,9 +301,9 @@ public async ValueTask Run( if (!shouldRetry) { - Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed with a non-retriable exception on attempt {attempt}/{_maxRetries}. Skipping remaining retries."); + Console.WriteLine($"[RetryFact] Test '{TestDescription}' failed with a non-retriable exception on attempt {attempt}/{_maxRetries}. Skipping remaining retries."); messageBus.QueueMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.MethodName}' failed with non-retriable exception. Skipping retries.")); + new DiagnosticMessage($"[RetryFact] Test '{TestDescription}' failed with non-retriable exception. Skipping retries.")); // This attempt ran on the intercepting bus, which deferred the // ITestFailed message so the test could be retried. We are not @@ -315,9 +331,9 @@ public async ValueTask Run( // finally does not dispose it, which would replay the failure immediately. interceptingBus = null; - Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}, retrying after {_delayMs}ms. Error: {lastException?.Message ?? "No exception message"}"); + Console.WriteLine($"[RetryFact] Test '{TestDescription}' failed on attempt {attempt}/{_maxRetries}, retrying after {_delayMs}ms. Error: {lastException?.Message ?? "No exception message"}"); messageBus.QueueMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed on attempt {attempt}/{_maxRetries}. Retrying after {_delayMs}ms delay. Error: {lastException?.Message ?? "No exception message"}")); + new DiagnosticMessage($"[RetryFact] Test '{TestDescription}' failed on attempt {attempt}/{_maxRetries}. Retrying after {_delayMs}ms delay. Error: {lastException?.Message ?? "No exception message"}")); try { @@ -327,7 +343,7 @@ public async ValueTask Run( { // Cancelled while waiting to retry. The failure from this attempt is // still deferred, so report it instead of letting the test vanish. - Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' was cancelled during the retry delay after attempt {attempt}/{_maxRetries}. Reporting the failure from that attempt."); + Console.WriteLine($"[RetryFact] Test '{TestDescription}' was cancelled during the retry delay after attempt {attempt}/{_maxRetries}. Reporting the failure from that attempt."); pendingFailureBus.ReplayDeferredMessages(); @@ -345,9 +361,9 @@ public async ValueTask Run( else { // Last attempt - failure message already went through to Test Explorer - Console.WriteLine($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed after all {_maxRetries} attempts. Final error: {lastException?.Message ?? "No exception captured"}"); + Console.WriteLine($"[RetryFact] Test '{TestDescription}' failed after all {_maxRetries} attempts. Final error: {lastException?.Message ?? "No exception captured"}"); messageBus.QueueMessage( - new DiagnosticMessage($"[RetryFact] Test '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}' failed after {_maxRetries} attempts. Last exception: {lastException?.Message ?? "No exception message"}")); + new DiagnosticMessage($"[RetryFact] Test '{TestDescription}' failed after {_maxRetries} attempts. Last exception: {lastException?.Message ?? "No exception message"}")); if (lastException != null) { @@ -368,7 +384,7 @@ public async ValueTask Run( } catch (Exception ex) { - Console.WriteLine($"[RetryFact] Unexpected exception while running '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}': {ex.GetType().FullName}: {ex.Message}"); + Console.WriteLine($"[RetryFact] Unexpected exception while running '{TestDescription}': {ex.GetType().FullName}: {ex.Message}"); // A crash must not swallow a failure an earlier attempt already produced, but at // most one attempt's failure may reach the bus: replaying both would publish two @@ -397,7 +413,7 @@ public async ValueTask Run( } // Should never reach here - Console.WriteLine($"[RetryFact] Reached the end of the retry loop unexpectedly for '{TestMethod.TestClass.TestClassName}.{TestMethod.MethodName}'. Reporting the test as failed."); + Console.WriteLine($"[RetryFact] Reached the end of the retry loop unexpectedly for '{TestDescription}'. Reporting the test as failed."); runSummary.Failed = 1; return runSummary; } From 736331ee72f909faaa2a01e6851e13937578fbd7 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 22:56:31 -0700 Subject: [PATCH 082/105] Let the E2E legs see the checks that guard the E2E assembly Every E2E leg selects positively on a data store, and a positive match cannot select a test case carrying no DataStore trait at all - the file that builds the filter says as much in its own comment. Real E2E classes get the trait from HttpIntegrationFixtureArgumentSets, but AssemblyValidationsTests has no server fixture and wants none, so it carried no DataStore trait and was filtered out of every leg. Listing the tests the way CI does returns none of its three checks. So the assembly-wide checks never ran anywhere for E2E - including the one added on this branch to catch a collection definition tagging classes it does not name, which is the propagation that hid twelve E2E cases behind a green run. The protection was inert in exactly the assembly the incident came from. Name both stores explicitly so the checks run under whichever leg is executing. Listing under the real filter strings now returns three checks for each store, and running them passes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AssemblyValidationsTests.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs index 5aa6358224..b977ac8b41 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs @@ -5,13 +5,29 @@ using System.Reflection; using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; namespace Microsoft.Health.Fhir.Tests.E2E { + /// + /// Assembly-wide checks over the E2E tests. + /// + /// + /// Every E2E leg selects positively on a data store - /[(DataStore=CosmosDb)&...] - and + /// a positive match cannot select a test case that carries no DataStore trait at all. Real + /// E2E classes get theirs from HttpIntegrationFixtureArgumentSets, but this class has no + /// server fixture and wants none: it only reads the assembly's metadata. Without the traits below + /// it is filtered out of every leg and never runs anywhere, which would leave these checks - one + /// of which exists to catch the collection-trait propagation that hid twelve E2E cases behind a + /// green run - quietly inert, in the one assembly the incident came from. Both stores are named + /// so the checks run whichever leg is executing. + /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.AssemblyValidation)] + [Trait(nameof(DataStore), nameof(DataStore.CosmosDb))] + [Trait(nameof(DataStore), nameof(DataStore.SqlServer))] public sealed class AssemblyValidationsTests { private static Assembly _currentAssembly = Assembly.GetAssembly(typeof(AssemblyValidationsTests)); From 490f1a38e9b3cf344bb551df9727fedd3a177b7f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 23:01:53 -0700 Subject: [PATCH 083/105] Ask whether a collection definition carries a trait, not which attribute wrote it The check named TraitAttribute exactly, but xUnit propagates any ITraitAttribute from a collection definition to the classes that join it - TraitAttribute is merely the built-in spelling of one. A custom trait attribute on a definition would therefore tag its members while the guard saw nothing, which is the same silent leg-reshaping the guard exists to catch, arriving by a slightly different route. The discoverer this branch changes already reasons in terms of the interface; this now agrees with it. Verified by putting a custom ITraitAttribute on a definition another class joins: under the old exact-type test the assembly checks all passed, and under this one all three report the offender by name. The scan above is left matching TraitAttribute exactly on purpose - it reads a trait's name out of the first constructor argument, a shape only that attribute guarantees - and a comment now says so. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs index f5d0a4787c..a0a27919d0 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs @@ -9,6 +9,7 @@ using System.Reflection; using EnsureThat; using Xunit; +using Xunit.v3; namespace Microsoft.Health.Fhir.Tests.Common { @@ -84,8 +85,12 @@ public static IEnumerable ScanTestsLookingForMissingTrait(Assembly assembl CustomAttributeData definitionAttribute = definition.CustomAttributes .FirstOrDefault(a => a.AttributeType == typeof(CollectionDefinitionAttribute)); + // Any ITraitAttribute propagates, not just the [Trait] spelling of one, so ask the + // question xUnit asks rather than naming a single attribute type. The scan above is + // deliberately narrower: it reads a trait's name out of the first constructor + // argument, which is a shape only TraitAttribute guarantees. if (definitionAttribute == null || - !definition.CustomAttributes.Any(a => a.AttributeType == typeof(TraitAttribute))) + !definition.CustomAttributes.Any(a => typeof(ITraitAttribute).IsAssignableFrom(a.AttributeType))) { continue; } From fb672b7d31edf9096d4704d08fc2c20261c14086 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 23:05:22 -0700 Subject: [PATCH 084/105] See a class join a collection by the v3 generic spelling Membership was read only from the non-generic [Collection] attribute, and only when it carried a constructor argument. xUnit v3 adds [Collection], which names the definition in its generic argument and takes no constructor argument at all - so it missed the guard twice over. On a branch whose whole purpose is moving this repository to v3, the spelling people are about to start writing was the one spelling that could reintroduce the incident unseen. Ask the interface question instead, then read membership from whichever of the three forms the class used: a name, a definition type, or a generic argument. Verified with a definition another class joins as [Collection]: the assembly checks passed with the previous code and report the offender by name with this one. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AssemblyChecker.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs index a0a27919d0..26a37c693f 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs @@ -116,8 +116,23 @@ public static IEnumerable ScanTestsLookingForMissingTrait(Assembly assembl private static bool JoinsCollection(Type candidate, string collectionName, Type definition) { - foreach (CustomAttributeData attribute in candidate.CustomAttributes.Where(a => a.AttributeType == typeof(CollectionAttribute))) + foreach (CustomAttributeData attribute in candidate.CustomAttributes + .Where(a => typeof(ICollectionAttribute).IsAssignableFrom(a.AttributeType))) { + // A class joins a collection by name, by naming the definition type, or - the v3 + // spelling this repository is moving to - as [Collection], which carries + // the definition in its generic argument and takes no constructor argument at all. + if (attribute.AttributeType.IsGenericType && + attribute.AttributeType.GetGenericTypeDefinition() == typeof(CollectionAttribute<>)) + { + if (attribute.AttributeType.GetGenericArguments()[0] == definition) + { + return true; + } + + continue; + } + if (attribute.ConstructorArguments.Count == 0) { continue; From dda973abe39aa02df3a84dfc8dceb9258c8075d2 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 23:49:56 -0700 Subject: [PATCH 085/105] Drop the losing attempt's messages instead of leaving them to be replayed An intercepting bus replays whatever it still holds when it is disposed, so a failure can never be lost by forgetting to report it. The cost is that silence is not something a caller can express by omission: leaving a bus alone publishes it anyway, a moment later. Deciding which attempt wins was therefore only half the job. The empty-summary path discarded the earlier attempt when the current one won, but not the current attempt when the earlier one won; the crash path did the same, and its comment said so - it expected the finally to replay the current attempt's messages, next to the ones just published for the earlier attempt. When the current attempt was holding an abstention, that is an ITestSkipped arriving immediately after an ITestFailed for the same test, which is the reordering the skip-after-failure path a few lines above exists to prevent. Disposal even announced it, logging an internal error while replaying. Both paths now go through one rule that publishes the winner and discards the loser, so the two cannot drift apart again. Pinned with the real bus: under the previous behaviour the test records ITestFailed followed by ITestSkipped, and disposal is asserted to have nothing left to fall back on. Reachable only if an exception escapes the runner after an abstention was held, which xUnit normally converts into a failure - hence a unit-level pin rather than a test asset. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SingleResultReportingTests.cs | 188 ++++++++++++++++++ .../RetryTestCase.cs | 74 ++++--- 2 files changed, 234 insertions(+), 28 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs new file mode 100644 index 0000000000..3de0879934 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs @@ -0,0 +1,188 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Sdk; +using Xunit.v3; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Pins the rule that resolves two attempts competing to report a single test. + /// + /// + /// An intercepting bus replays anything it still holds when it is disposed, so that a failure + /// can never be lost. The cost of that fail-safe is that a caller cannot stay silent about a + /// bus by saying nothing: the loser of the competition has to be discarded explicitly, or it + /// publishes a second result moments after the winner. When the loser is holding an abstention + /// and the winner is a failure, that second result is a skip arriving after a failure - which + /// is how a red test can be read as green. + /// + [Trait("Category", "UnitTests")] + public class SingleResultReportingTests + { + [Fact] + public void GivenAnEarlierFailureAndAnAbstainingCurrentAttempt_WhenTheEarlierAttemptWins_ThenOnlyTheFailureIsPublished() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus earlier = DeferFailure(inner); + RetryTestCase.FailureInterceptingMessageBus current = DeferAbstention(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayEarlierAttempt, current, earlier); + + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + + // Disposal is the fail-safe, and it must have nothing left to fall back on. + current.Dispose(); + earlier.Dispose(); + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + } + + [Fact] + public void GivenBothAttemptsHoldingMessages_WhenTheCurrentAttemptWins_ThenOnlyItsMessagesArePublished() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus earlier = DeferFailure(inner); + RetryTestCase.FailureInterceptingMessageBus current = DeferFailure(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayCurrentAttempt, current, earlier); + + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + + current.Dispose(); + earlier.Dispose(); + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + } + + [Fact] + public void GivenNothingObservedAFailure_WhenThereIsNothingToReport_ThenTheCurrentAbstentionSurvivesToBePublished() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus current = DeferAbstention(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReportNothing, current, earlierAttempt: null); + + // The abstention is the only result this test has, so it is left for disposal to publish + // rather than being dropped. + Assert.Empty(inner.PublishedKinds); + current.Dispose(); + Assert.Equal(new[] { nameof(ITestSkipped) }, inner.PublishedKinds); + } + + [Fact] + public void GivenAnEarlierFailureThatLostToNothing_WhenThereIsNothingToReport_ThenItIsNotPublished() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus earlier = DeferFailure(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReportNothing, currentAttempt: null, earlierAttempt: earlier); + + earlier.Dispose(); + Assert.Empty(inner.PublishedKinds); + } + + [Fact] + public void GivenAMissingBus_WhenTheRuleIsApplied_ThenItReportsThatTheRunMayContinue() + { + Assert.True(RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayEarlierAttempt, currentAttempt: null, earlierAttempt: null)); + Assert.True(RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayCurrentAttempt, currentAttempt: null, earlierAttempt: null)); + } + + [Fact] + public void GivenTheUnderlyingBusAsksToStop_WhenTheWinnerIsReplayed_ThenTheRequestIsPassedOn() + { + var inner = new RecordingMessageBus { ContinueRunning = false }; + RetryTestCase.FailureInterceptingMessageBus earlier = DeferFailure(inner); + + Assert.False(RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayEarlierAttempt, currentAttempt: null, earlierAttempt: earlier)); + } + + private static RetryTestCase.FailureInterceptingMessageBus DeferFailure(RecordingMessageBus inner) + { + var bus = new RetryTestCase.FailureInterceptingMessageBus(inner, deferFailures: true); + bus.QueueMessage(new StubTestFailed()); + return bus; + } + + private static RetryTestCase.FailureInterceptingMessageBus DeferAbstention(RecordingMessageBus inner) + { + var bus = new RetryTestCase.FailureInterceptingMessageBus(inner, deferFailures: true, deferAbstentions: true); + bus.QueueMessage(new StubTestSkipped()); + return bus; + } + + private sealed class RecordingMessageBus : IMessageBus + { + private readonly List _published = new List(); + + public bool ContinueRunning { get; set; } = true; + + public IReadOnlyList PublishedKinds => _published + .ToList(); + + public bool QueueMessage(IMessageSinkMessage message) + { + _published.Add(message switch + { + ITestFailed => nameof(ITestFailed), + ITestSkipped => nameof(ITestSkipped), + _ => message.GetType().Name, + }); + + return ContinueRunning; + } + + public void Dispose() + { + } + } + + private abstract class StubResultMessage + { + public string AssemblyUniqueID => "assembly"; + + public string TestCollectionUniqueID => "collection"; + + public string TestClassUniqueID => "class"; + + public string TestMethodUniqueID => "method"; + + public string TestCaseUniqueID => "case"; + + public string TestUniqueID => "test"; + + public decimal ExecutionTime => 0m; + + public string Output => string.Empty; + + public string[] Warnings => null; + + public DateTimeOffset FinishTime => DateTimeOffset.MinValue; + + public string ToJson() => "{}"; + } + + private sealed class StubTestFailed : StubResultMessage, ITestFailed + { + public FailureCause Cause => FailureCause.Assertion; + + public int[] ExceptionParentIndices => new[] { -1 }; + + public string[] ExceptionTypes => new[] { typeof(InvalidOperationException).FullName }; + + public string[] Messages => new[] { "stub failure" }; + + public string[] StackTraces => new string[] { null }; + } + + private sealed class StubTestSkipped : StubResultMessage, ITestSkipped + { + public string Reason => "stub skip"; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 64adf31115..3d112c5fda 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -183,22 +183,7 @@ public async ValueTask Run( // At most one bus is replayed. A failure this attempt saw supersedes anything held // over from an earlier one, and replaying both would publish two results for a // single test. - bool continueRunning = true; - - switch (outcome) - { - case NoResultOutcome.ReplayCurrentAttempt: - pendingFailureBus?.DiscardDeferredMessages(); - continueRunning = interceptingBus.ReplayDeferredMessages(); - break; - - case NoResultOutcome.ReplayEarlierAttempt: - continueRunning = pendingFailureBus.ReplayDeferredMessages(); - break; - - default: - break; - } + bool continueRunning = ReportSingleResult(outcome, interceptingBus, pendingFailureBus); StopRunIfRequested(continueRunning, cancellationTokenSource); @@ -387,22 +372,15 @@ public async ValueTask Run( Console.WriteLine($"[RetryFact] Unexpected exception while running '{TestDescription}': {ex.GetType().FullName}: {ex.Message}"); // A crash must not swallow a failure an earlier attempt already produced, but at - // most one attempt's failure may reach the bus: replaying both would publish two + // most one attempt's result may reach the bus: replaying both would publish two // results for a single test. That is the same supersession rule the no-result - // path applies, so it is decided the same way here. Disposing replays whatever - // is still deferred, and the finally below does that for the current attempt. + // path applies, so it is decided - and carried out - the same way here. NoResultOutcome disposition = DecideNoResultOutcome( currentAttemptObservedFailure: interceptingBus?.HasObservedFailure ?? false, earlierAttemptObservedFailure: pendingFailureBus?.HasDeferredFailure ?? false); - if (disposition == NoResultOutcome.ReplayEarlierAttempt) - { - pendingFailureBus?.Dispose(); - } - else - { - pendingFailureBus?.DiscardDeferredMessages(); - } + // The run is already unwinding, so a request to stop has nothing left to stop. + ReportSingleResult(disposition, interceptingBus, pendingFailureBus); throw; } @@ -460,6 +438,46 @@ internal static NoResultOutcome DecideNoResultOutcome(bool currentAttemptObserve : NoResultOutcome.ReportNothing; } + /// + /// Publishes at most one attempt's held-back messages, and drops the other attempt's. + /// + /// The outcome chosen by . + /// The bus this attempt ran on, or null once its ownership has moved. + /// The bus carrying a failure held over from an earlier attempt, if any. + /// false when the underlying bus asked for the run to stop. + /// + /// Deciding which attempt wins is not enough on its own: the loser's messages have to be + /// dropped as well, because a bus still holding messages replays them when it is disposed. + /// That fail-safe exists so a failure can never be lost, but it makes silence the one thing + /// a caller cannot express by omission - leaving the loser alone publishes it anyway, a + /// moment after the winner. An abstention held by an attempt that then crashed would arrive + /// as a second result on top of the earlier attempt's failure, and a skip landing after a + /// failure is exactly the reordering that turns a red test green. + /// + internal static bool ReportSingleResult( + NoResultOutcome outcome, + FailureInterceptingMessageBus currentAttempt, + FailureInterceptingMessageBus earlierAttempt) + { + switch (outcome) + { + case NoResultOutcome.ReplayCurrentAttempt: + earlierAttempt?.DiscardDeferredMessages(); + return currentAttempt?.ReplayDeferredMessages() ?? true; + + case NoResultOutcome.ReplayEarlierAttempt: + currentAttempt?.DiscardDeferredMessages(); + return earlierAttempt?.ReplayDeferredMessages() ?? true; + + default: + // Nothing observed a failure, so there is no competition to resolve. Whatever the + // current attempt is still holding - an abstention, typically - is the only + // result the test has, and is left for its disposal to publish. + earlierAttempt?.DiscardDeferredMessages(); + return true; + } + } + /// /// Chooses the exception that belongs with the failure being reported for an attempt that /// published no result of its own. @@ -626,7 +644,7 @@ private static IEnumerable EnumerateExceptionTree(Exception exception /// attempt keeps the messages in their original order, so the caller can either discard /// them (retrying) or replay them intact (reporting the failure). /// - private class FailureInterceptingMessageBus : IMessageBus + internal class FailureInterceptingMessageBus : IMessageBus { private readonly IMessageBus _innerBus; private readonly bool _deferFailures; From 67601822383a71af03d718252548b849552d6b00 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 23:58:42 -0700 Subject: [PATCH 086/105] Keep the traits a broken attribute did not break Reading a declaration's attributes constructs all of them, so a single attribute whose constructor throws costs that declaration every trait beside it - including the Category a CI leg selects on. The attribute that throws need not have anything to do with traits, which is what makes this easy to arrive at without meaning to. The fallback now names the attribute types from metadata and constructs them one at a time, so the sound ones survive the broken one. Reaching this path needs two faults at once: a trait attribute that throws, which is what sends the read down the fallback, and a second attribute that throws when constructed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../ThrowingAttributeConstructorTests.cs | 49 ++++++++++++ .../ThrowingConstructorAttribute.cs | 27 +++++++ .../ThrowingAttributeConstructorTests.cs | 78 +++++++++++++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 62 ++++++++++++++- 4 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingAttributeConstructorTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingConstructorAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingAttributeConstructorTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingAttributeConstructorTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingAttributeConstructorTests.cs new file mode 100644 index 0000000000..0ac8099886 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingAttributeConstructorTests.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingTrait; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingAttributeConstructor +{ + /// + /// A class whose discovery faults, and which carries an attribute whose constructor throws + /// alongside the trait a CI leg would select on. + /// Expected: 1 result, failed, still carrying the class trait. + /// + /// + /// Two things have to go wrong at once to reach the read this covers. The method carries a trait + /// attribute that throws, which is what makes xUnit's own trait read fail and sends the failure + /// down the per-attribute fallback. The class then carries an attribute whose constructor + /// throws, and because a declaration's attributes are constructed together, that read fails for + /// the class as a whole. If the fallback gives up on the declaration at that point, the class + /// trait goes with it, and a leg selecting positively on Category cannot match the + /// failure standing in for these tests - reporting success with the class missing. + /// + /// The attribute that throws is not a trait attribute and knows nothing about traits. It just + /// happens to sit on the same class, which is what makes this easy to arrive at without meaning + /// to. + /// + /// + [Trait("Category", "ThrowingConstructorProbe")] + [ThrowingConstructor] + public class ThrowingAttributeConstructorTests + { + /// + /// Never runs: it declares two fixture argument set attributes, so which one applies cannot + /// be determined. Its throwing trait attribute is what forces the fallback trait read. + /// + [Fact] + [ThrowingTrait] + [AssetArgumentSets(AssetDataStore.Sql)] + [TwoDimensionArgumentSets(AssetDataStore.Sql, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingConstructorAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingConstructorAttribute.cs new file mode 100644 index 0000000000..056d30f265 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/ThrowingAttributeConstructor/ThrowingConstructorAttribute.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingAttributeConstructor +{ + /// + /// An ordinary attribute - not a trait attribute - whose constructor throws. + /// + /// + /// Reading a declaration's attributes constructs all of them, so this one is enough to make that + /// read fail for the declaration it sits on, taking the sound trait attributes beside it with it. + /// It deliberately is not an ITraitAttribute: the traits are lost because of where the + /// attribute sits, not because of what it is. + /// + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] + public sealed class ThrowingConstructorAttribute : Attribute + { + public ThrowingConstructorAttribute() + { + throw new InvalidOperationException("This attribute's constructor always throws."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingAttributeConstructorTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingAttributeConstructorTests.cs new file mode 100644 index 0000000000..7b1c03b6a3 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ThrowingAttributeConstructorTests.cs @@ -0,0 +1,78 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a discovery fault on a class that also carries an attribute whose constructor throws. + /// + /// + /// This is a wider hole than a trait attribute that refuses to produce its traits. A + /// declaration's attributes are constructed together, so one throwing constructor fails the read + /// for the whole declaration - and the attribute that throws need not have anything to do with + /// traits, which is what makes it easy to arrive at without meaning to. The class trait would go + /// with it, and a leg selecting positively on that trait would report success with the class's + /// tests missing. + /// + public class ThrowingAttributeConstructorTests + { + private const string FaultCase = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingAttributeConstructor.ThrowingAttributeConstructorTests.NeverRuns (fixture argument set discovery: Sql)"; + private const string FaultCaseTwoDimensions = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.ThrowingAttributeConstructor.ThrowingAttributeConstructorTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + + private static Dictionary BothFaultCases() => + new Dictionary + { + [FaultCase] = "Failed", + [FaultCaseTwoDimensions] = "Failed", + }; + + /// + /// The failure standing in for the lost tests is reported even though constructing the + /// class's attributes throws. + /// + [Fact] + public void GivenAClassWithAnAttributeConstructorThatThrows_WhenItsDiscoveryFaults_ThenTheFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingAttributeConstructor"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The class trait still selects the failure, even though the attribute that throws sits on + /// the same class. This is the shape of the export leg's filter, which requires a positive + /// Category, and the reason the sound attributes must survive the broken one. + /// + [Fact] + public void GivenAClassWithAnAttributeConstructorThatThrows_WhenALegSelectsOnTheClassTrait_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingAttributeConstructor", filterQueryTraits: "(Category=ThrowingConstructorProbe)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + + /// + /// The argument set trait selects it too. That trait is built from the argument sets rather + /// than read from the declaration, so this pins that the failed read costs the failure + /// nothing beyond the declaration it happened on. + /// + [Fact] + public void GivenAClassWithAnAttributeConstructorThatThrows_WhenALegSelectsOnTheArgumentSet_ThenTheFailureIsStillSelected() + { + TestAssetRun run = TestAssetRunner.Run("ThrowingAttributeConstructor", filterQueryTraits: "(AssetDataStore=Sql)"); + + TestAssetRunAssertions.PublishedExactly( + run, + BothFaultCases()); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index 116be9e607..d08d43497d 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -741,9 +741,16 @@ private static void AddFaultTraitsPerAttribute(XunitTestMethod anchor, Action + /// Reads a declaration's trait attributes one type at a time, keeping the ones that can be + /// constructed. + /// + /// + /// This runs only when reading them together has already failed. The attribute types are + /// taken from metadata, which names them without constructing anything, so a constructor + /// that throws costs only the attributes of its own type rather than every attribute on the + /// declaration. Attributes inherited from a base declaration are not enumerated here: doing + /// so means walking the hierarchy by hand, and the ordinary read - which this only stands in + /// for - already covers them. + /// + /// The collection definition, class, or method being read. + /// The trait attributes that could be constructed, which may be none. + private static ITraitAttribute[] ReadTraitAttributesIndividually(MemberInfo declaration) + { + List traitAttributeTypes; + + try + { + traitAttributeTypes = declaration.CustomAttributes + .Select(attribute => attribute.AttributeType) + .Where(type => typeof(ITraitAttribute).IsAssignableFrom(type)) + .Distinct() + .ToList(); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the trait attributes of '{declaration.Name}' could not be named, so a trait filter may not select the failure standing in for its tests. {ex}"); + return Array.Empty(); + } + + var traitAttributes = new List(); + + foreach (Type traitAttributeType in traitAttributeTypes) + { + try + { + traitAttributes.AddRange(Attribute.GetCustomAttributes(declaration, traitAttributeType, inherit: true).OfType()); + } + catch (Exception ex) + { + Console.WriteLine( + $"[FixtureArgumentSets] WARNING: the trait attribute '{traitAttributeType.Name}' of '{declaration.Name}' could not be constructed, so a filter naming the traits it declares may not select the failure standing in for its tests. {ex}"); + } + } + + return traitAttributes.ToArray(); + } + /// /// Reads the collection definition class the method's collection was declared by, if any. /// From 6fc4f562d870feae50fe47a1728af6c124eb6db7 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 00:06:47 -0700 Subject: [PATCH 087/105] Widen every argument set dimension that names nothing, not only all of them A declaration whose dimensions all name nothing was already rescued: the failure standing in for its tests is reported once per value each dimension's type declares, so a leg selecting positively on one of them still sees it. A declaration that names nothing in only some of its dimensions has the same empty product and the same missing tests, but was left carrying only the dimensions it did name - and a leg selecting positively on a silent dimension, which is how every E2E and export leg selects a data store, matched nothing and passed green and empty. The dimensions a declaration leaves open are now widened individually. Where the open position was meant to be inherited from the class, the class's own declaration is closed over separately and still contributes its combinations, so the only cost is a failure reported under a value the declaration did not name - loud, and traceable back to it. The alternative is silence. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../PartiallyEmptyArgumentSetTests.cs | 42 +++++++++++ .../DiscoveryFaultTests.cs | 11 +++ .../PartiallyEmptyArgumentSetTests.cs | 74 +++++++++++++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 46 ++++++++---- 4 files changed, 157 insertions(+), 16 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartiallyEmptyArgumentSet/PartiallyEmptyArgumentSetTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/PartiallyEmptyArgumentSetTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartiallyEmptyArgumentSet/PartiallyEmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartiallyEmptyArgumentSet/PartiallyEmptyArgumentSetTests.cs new file mode 100644 index 0000000000..e0d47ea5cd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/PartiallyEmptyArgumentSet/PartiallyEmptyArgumentSetTests.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartiallyEmptyArgumentSet +{ + /// + /// A class declaring a fixture argument set that names a value in one dimension and none in the + /// other, so it expands to no variants. + /// + /// + /// The product of the declared dimensions is empty, exactly as it is when no dimension names + /// anything, so the method produces no test cases and the failure standing in for it is all a + /// leg can see. What differs is that one dimension does name a value, which is enough to keep + /// the other dimension from being widened - so the stand-in carries the dimension that was + /// declared and says nothing about the one that was not. A leg selecting positively on the + /// undeclared dimension, which is how the E2E and export legs select a data store, matches + /// nothing and reports success with the method's tests absent. + /// + /// The class deliberately declares nothing itself. A class-level declaration would contribute + /// its own combinations to the stand-in and hide the hole. + /// + /// + public class PartiallyEmptyArgumentSetTests + { + /// + /// Never runs: one of the two argument sets it asks for expands to nothing, so their product + /// does too. + /// + [Fact] + [TwoDimensionArgumentSets((AssetDataStore)0, AssetOtherDimension.Some)] + public void NeverRuns() + { + Assert.True(true); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs index e15949ed1e..b74d7af99b 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DiscoveryFaultTests.cs @@ -18,6 +18,13 @@ public class DiscoveryFaultTests private const string SqlErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Sql)"; private const string CosmosErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Cosmos)"; + // The class declares one dimension and the method two, the second of which names nothing. + // The two declarations are closed over separately, so the class's contributes the cases + // above and the method's contributes these - its silent dimension widened to the only value + // its type declares, so the failure can be selected on that dimension as well as the other. + private const string SqlWidenedErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + private const string CosmosWidenedErrorCaseName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DiscoveryFault.DiscoveryFaultTests.NeverRuns (fixture argument set discovery: Cosmos, Some)"; + /// /// xUnit reports an exception thrown out of discovery only as a diagnostic message, which is /// suppressed unless the run was started with --xunit-diagnostics, and carries on @@ -37,6 +44,8 @@ public void GivenAClassThatCannotBeExpanded_WhenItIsDiscovered_ThenTheFaultIsRep { [SqlErrorCaseName] = "Failed", [CosmosErrorCaseName] = "Failed", + [SqlWidenedErrorCaseName] = "Failed", + [CosmosWidenedErrorCaseName] = "Failed", }); Assert.NotEqual(0, run.ExitCode); @@ -96,6 +105,7 @@ public void GivenAClassThatCannotBeExpanded_WhenTestsAreSelectedByArgumentSetTra new Dictionary { [SqlErrorCaseName] = "Failed", + [SqlWidenedErrorCaseName] = "Failed", }); Assert.NotEqual(0, run.ExitCode); @@ -119,6 +129,7 @@ public void GivenAClassThatCannotBeExpanded_WhenTestsAreSelectedByExcludingAnArg new Dictionary { [CosmosErrorCaseName] = "Failed", + [CosmosWidenedErrorCaseName] = "Failed", }); Assert.NotEqual(0, run.ExitCode); diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartiallyEmptyArgumentSetTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartiallyEmptyArgumentSetTests.cs new file mode 100644 index 0000000000..96385f15ec --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/PartiallyEmptyArgumentSetTests.cs @@ -0,0 +1,74 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers a declaration that names a value in one argument set dimension and none in the other. + /// + /// + /// Naming nothing in every dimension is already rescued: the failure standing in for the method + /// is reported once per value each dimension's type declares, so a leg selecting positively on + /// one of them still sees it. Naming nothing in only some of them is the same hole with a + /// narrower mouth - the product is still empty, the method still produces no tests, but the + /// stand-in says nothing about the dimension that named nothing and a leg selecting on that + /// dimension still sees an empty, green run. + /// + public class PartiallyEmptyArgumentSetTests + { + private const string SqlCase = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartiallyEmptyArgumentSet.PartiallyEmptyArgumentSetTests.NeverRuns (fixture argument set discovery: Sql, Some)"; + private const string CosmosCase = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.PartiallyEmptyArgumentSet.PartiallyEmptyArgumentSetTests.NeverRuns (fixture argument set discovery: Cosmos, Some)"; + + private static Dictionary EveryDataStore() => + new Dictionary + { + [SqlCase] = "Failed", + [CosmosCase] = "Failed", + }; + + /// + /// The dimension that named nothing is widened to every value its type declares, so the + /// failure is reported once per data store rather than once with no data store at all. + /// + [Fact] + public void GivenADeclarationNamingNothingInOneDimension_WhenItExpandsToNoVariants_ThenTheFailureIsReportedForEveryValueOfThatDimension() + { + TestAssetRun run = TestAssetRunner.Run("PartiallyEmptyArgumentSet"); + + TestAssetRunAssertions.PublishedExactly(run, EveryDataStore()); + } + + /// + /// This is the shape of the E2E and export legs' filter. Without the widening the stand-in + /// carries no data store trait, this selects nothing, and the leg passes with the method's + /// tests missing. + /// + [Fact] + public void GivenADeclarationNamingNothingInOneDimension_WhenALegSelectsPositivelyOnThatDimension_ThenTheFailureIsSelected() + { + TestAssetRun run = TestAssetRunner.Run("PartiallyEmptyArgumentSet", filterQueryTraits: "(AssetDataStore=Sql)"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary { [SqlCase] = "Failed" }); + } + + /// + /// The dimension that did name a value keeps it. Widening the one that named nothing must + /// not spill into the one that did, or the failure would be reported under values the + /// declaration explicitly did not ask for. + /// + [Fact] + public void GivenADeclarationNamingNothingInOneDimension_WhenALegSelectsOnTheDeclaredDimension_ThenTheFailureIsSelected() + { + TestAssetRun run = TestAssetRunner.Run("PartiallyEmptyArgumentSet", filterQueryTraits: "(AssetOtherDimension=Some)"); + + TestAssetRunAssertions.PublishedExactly(run, EveryDataStore()); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index d08d43497d..afde5c640b 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -1284,14 +1284,16 @@ private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(FixtureArgumentSe /// /// /// A dimension naming no value expands to nothing, so the product of the dimensions is empty - /// and the failure standing in for the method carries no argument set trait at all. The E2E - /// and export legs select positively on that trait, so such a failure is invisible to them: - /// the leg reports success with the method's tests simply absent, which is the outcome this - /// whole path exists to prevent. The value is unknown but the type is not - a dimension is - /// declared as an enum whether or not it names any of its flags - so the failure is reported - /// once per value that type declares instead. That over-reports, in the same deliberate - /// direction as the rest of this path: a leg seeing a failure for a variant that would not - /// have existed is loud and traceable to this declaration, where a leg seeing nothing is not. + /// and the failure standing in for the method carries no argument set trait at all - or, when + /// another dimension does name a value, carries only that one and stays silent about this. The + /// E2E and export legs select positively on the data store trait, so such a failure is + /// invisible to them: the leg reports success with the method's tests simply absent, which is + /// the outcome this whole path exists to prevent. The value is unknown but the type is not - a + /// dimension is declared as an enum whether or not it names any of its flags - so the failure + /// is reported once per value that type declares instead. That over-reports, in the same + /// deliberate direction as the rest of this path: a leg seeing a failure for a variant that + /// would not have existed is loud and traceable to this declaration, where a leg seeing + /// nothing is not. /// /// The declaration to expand. /// One dimension per argument set, each holding at least one value where the type is known. @@ -1306,27 +1308,39 @@ private static SingleFlag[][] ExpandFaultEnumFlagsFromAttributeData(FixtureArgum } /// - /// Replaces the dimensions of a declaration that names no value at all with every value their - /// types declare, so the failure standing in for the method can still be selected. + /// Replaces each dimension that names no value with every value its type declares, so the + /// failure standing in for the method can still be selected on that dimension. /// /// - /// This fires only when the declaration produces no combination whatsoever. A dimension that - /// names nothing alongside one that does is not this case: naming nothing in one position is - /// how a method asks to inherit that position from its class, and widening it would report - /// the failure under values the declaration never asked for. + /// A dimension naming nothing is widened whether or not the dimensions beside it name + /// anything. Naming nothing in one position may be how a method asks to inherit that + /// position from its class, but the class's own declaration is closed over separately and + /// contributes its combinations regardless, so nothing is lost by also reporting this + /// declaration across the position it left open - and a class that declares nothing leaves + /// the position open with no second declaration to fill it. Not widening it there is what + /// makes the failure carry the dimension that was declared and say nothing about the one + /// that was not, which a leg selecting positively on the silent dimension cannot match. + /// + /// The alternative is the outcome this whole path exists to prevent: the method produces no + /// tests, the leg selects nothing standing in for them, and the run is green and empty. A + /// failure reported under a value the declaration did not name is loud and traceable back to + /// this declaration, which is the direction every trade-off on the fault path is made in. + /// /// /// The dimensions as declared. /// The type of each dimension, positionally, with null where unknown. /// The dimensions to report the failure against. private static SingleFlag[][] WidenFaultDimensionsThatNameNothing(SingleFlag[][] dimensions, Type[] enumTypes) { - if (dimensions.Length == 0 || dimensions.Any(dimension => dimension.Length > 0)) + if (dimensions.Length == 0 || dimensions.All(dimension => dimension.Length > 0)) { return dimensions; } return dimensions - .Select((dimension, index) => AllSingleValuedFlagsOf(index < enumTypes.Length ? enumTypes[index] : null)) + .Select((dimension, index) => dimension.Length > 0 + ? dimension + : AllSingleValuedFlagsOf(index < enumTypes.Length ? enumTypes[index] : null)) .ToArray(); } From 286b1d09c0647e173d489cf66e3b3f2ebd743ebc Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 00:13:54 -0700 Subject: [PATCH 088/105] Fail when the assembly stops declaring the framework its traits come from The custom framework is what expands fixture argument sets into variants, and the variants are what carry the DataStore trait every E2E and export leg selects positively on. Losing the declaration throws nothing: the classes are discovered as ordinary xUnit tests, carry no data store trait, and are filtered out. A filtered-out class never constructs its fixture, so no connection is attempted and nothing errors - the leg runs the handful of classes that declare traits by hand, reports success, and some two thousand E2E tests are simply absent. Also traits the E2E-only copy of the assembly checks, which compiled into the E2E assemblies but was invisible to their filters, so a check added there would have run nowhere. The copy shared with the integration assemblies stays untraited on purpose - those legs select by excluding a store, and an exclusion drops a class that names both - and now says so. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AssemblyValidationsTests.cs | 13 ++++++++ .../AssemblyValidationsTester.cs | 33 +++++++++++++++++++ .../AssemblyValidationsTests.cs | 16 +++++++++ .../AssemblyValidationsTests.cs | 7 ++++ .../AssemblyValidationsTests.cs | 7 ++++ 5 files changed, 76 insertions(+) diff --git a/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs b/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs index 34f115b2eb..43fc83f87d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Tests/AssemblyValidationsTests.cs @@ -10,6 +10,19 @@ namespace Microsoft.Health.Fhir.Shared.Tests { + /// + /// Assembly-wide checks, from the shared project the unit, integration, and E2E assemblies all + /// compile. + /// + /// + /// This copy deliberately carries no DataStore trait. It compiles into the integration + /// assemblies, whose legs select by excluding the other store, and an exclusion drops a + /// case when any of its values under that trait matches - so naming both stores here would hide + /// these checks from both integration legs rather than reveal them. The consequence is that in + /// an E2E assembly this copy is invisible to the positively-filtered legs, and a check added + /// only here would not run there. E2E-visible checks belong in the copy under + /// Microsoft.Health.Fhir.Shared.Tests.E2E, which is traited for exactly that reason. + /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.AssemblyValidation)] public sealed class AssemblyValidationsTests diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs index 204f1da07a..e8eefc6100 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs @@ -54,6 +54,39 @@ public static void EnsureNoSharedCollectionDefinitionCarriesTraits(Assembly asse $"definition class and leave the traits where they were declared.{Environment.NewLine}{stringBuilder}"); } + /// + /// Fails when the assembly does not declare the test framework its tests are written for. + /// + /// Assembly under analysis. + /// The test framework the assembly must declare. + /// + /// The custom framework is what expands fixture argument sets into variants, and the variants + /// are what carry the DataStore trait every E2E and export leg selects positively on. + /// Lose the declaration and nothing throws: the classes are discovered as ordinary xUnit + /// tests, carry no data store trait, and are filtered out of every leg. A class that is + /// filtered out never constructs its fixture, so no connection is attempted and nothing + /// errors - the leg runs whatever handful of tests declared their traits by hand, reports + /// success, and the entire suite is missing. This check turns that into a failure. + /// + public static void EnsureTestFrameworkIsDeclared(Assembly assembly, Type expectedFrameworkType) + { + TestFrameworkAttribute[] declared = assembly.GetCustomAttributes().ToArray(); + + if (declared.Length == 1 && declared[0].FrameworkType == expectedFrameworkType) + { + return; + } + + string found = declared.Length == 0 + ? "none" + : string.Join(", ", declared.Select(attribute => attribute.FrameworkType?.FullName ?? "")); + + Assert.Fail( + $"Assembly '{assembly}' must declare exactly one test framework, '{expectedFrameworkType.FullName}', but declares: {found}. " + + $"Without it the fixture argument sets are never expanded, so the tests carry no data store trait, every leg that selects positively on one " + + $"filters them out, and the leg passes with the suite missing rather than failing."); + } + private static void AssertTestClasses(Assembly currentAssembly, string traitName, IEnumerable types) { if (types == null || !types.Any()) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs index 30d8e73211..181832f8ca 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs @@ -5,13 +5,29 @@ using System.Reflection; using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; using Xunit; namespace Microsoft.Health.Fhir.Shared.Tests.E2E.Common { + /// + /// Assembly-wide checks over the E2E tests, from the shared project the E2E assemblies alone + /// compile. + /// + /// + /// This project is imported by the four E2E assemblies and nothing else, and every E2E leg + /// selects positively on a data store, so without the traits below this copy is filtered out of + /// every leg and never runs. It would still compile, and a check added to it would still look + /// like it was covering the E2E assemblies while running nowhere. Naming both stores is safe + /// here precisely because only positively-filtered legs compile it: a leg that selected by + /// excluding a store would drop a class naming both, which is why the copy shared with + /// the integration assemblies deliberately carries neither. + /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.AssemblyValidation)] + [Trait(nameof(DataStore), nameof(DataStore.CosmosDb))] + [Trait(nameof(DataStore), nameof(DataStore.SqlServer))] public sealed class AssemblyValidationsTests { private static Assembly _currentAssembly = Assembly.GetAssembly(typeof(AssemblyValidationsTests)); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs index b977ac8b41..074d01ac86 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/AssemblyValidationsTests.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System.Reflection; +using Microsoft.Health.Extensions.Xunit; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; @@ -49,5 +50,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionC { AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureTheCustomTestFrameworkIsDeclared() + { + AssemblyValidationsTester.EnsureTestFrameworkIsDeclared(_currentAssembly, typeof(CustomXunitTestFramework)); + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs index 45a261e56d..d60876dbe5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/AssemblyValidationsTests.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System.Reflection; +using Microsoft.Health.Extensions.Xunit; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using Xunit; @@ -33,5 +34,11 @@ public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionC { AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); } + + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureTheCustomTestFrameworkIsDeclared() + { + AssemblyValidationsTester.EnsureTestFrameworkIsDeclared(_currentAssembly, typeof(CustomXunitTestFramework)); + } } } From 0db80b5c8af40ecfcb2d691c9c5adf49e54fe634 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 00:31:43 -0700 Subject: [PATCH 089/105] Count the tests a leg selects before trusting it to have run them A leg that filters positively on DataStore reports success when its filter matches nothing, so the count is the only thing separating everything passed from nothing ran. Each leg now asserts a floor on what discovery selects, well below its real count, before the run starts. The floor deliberately does not ride on --minimum-expected-tests: that option is applied again to the much smaller pass --retry-failed-tests starts, so any floor above the size of a retry pass turns a flake that recovered into a failed leg with exit 9. Measured on a probe: three passing tests and one that fails only on its first run exit 9 under --retry-failed-tests 3 --minimum-expected-tests 3. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/e2e-tests.yml | 35 +++++++++++++++++++-- build/jobs/run-cosmos-tests.yml | 5 +++ build/jobs/run-export-tests.yml | 56 +++++++++++++++++++++++++++++---- build/jobs/run-sql-tests.yml | 7 +++++ 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index 5670e020f2..f540ae4e11 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -11,6 +11,16 @@ parameters: - name: testRunTitleSuffix type: string default: '' +# The floor this leg's discovered test count must clear. A leg that selects positively on a +# DataStore trait reports success when its filter matches nothing, so the count is the only thing +# that distinguishes "everything passed" from "nothing ran". It is checked against a discovery +# pass rather than passed to the run as --minimum-expected-tests, because that option is applied +# again to the much smaller pass --retry-failed-tests starts: a floor above the size of a retry +# pass turns a flake that recovered into a failed leg (exit 9). Set well below the real counts so +# version differences and legitimately removed tests do not fail a leg that is working. +- name: minimumExpectedTests + type: string + default: '1' steps: - template: e2e-tests-extract.yml @@ -45,8 +55,29 @@ steps: $filterQuery = "/$traitFilter" # This filter is positive on DataStore, so a test case that reaches the runner without that - # trait is invisible to it rather than failing. Assert a floor on the count so a filter or - # trait regression that silently empties this leg cannot report success. + # trait is invisible to it rather than failing. Count what the filter selects before running + # anything, so a trait or filter regression that empties this leg fails it instead of + # passing it green and empty. Discovery needs no server, and unlike the runner's own + # --minimum-expected-tests this count is not re-applied to the smaller pass that + # --retry-failed-tests starts, where any real floor would fail a recovered flake. + $listOutput = & dotnet $dll.FullName --list-tests --filter-query $filterQuery + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for filter $filterQuery failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for filter $filterQuery from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for filter $filterQuery (floor ${{ parameters.minimumExpectedTests }})" + if ($discovered -lt ${{ parameters.minimumExpectedTests }}) { + throw "Filter $filterQuery selected $discovered test(s), below this leg's floor of ${{ parameters.minimumExpectedTests }}. The leg would have run a fraction of its tests and reported success." + } + + # Kept at 1 as a last-ditch check that the run itself published something. Anything higher + # is unsafe here for the retry reason above; the real floor is the discovery check. $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index 0f7b8bbf3e..ef6dce3358 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -165,6 +165,9 @@ jobs: containerAppName: '${{ parameters.containerAppName }}' appServiceType: 'CosmosDb' categoryFilter: '${{ parameters.mainCategoryFilter }}' + # Around 1350 cases across the FHIR versions. A floor of 700 leaves room for version + # differences and removed tests while failing a leg that has quietly lost its traits. + minimumExpectedTests: '700' - ${{ if eq(parameters.runReindexJob, true) }}: - job: 'cosmosE2eTests_Reindex' @@ -185,3 +188,5 @@ jobs: appServiceType: 'CosmosDb' categoryFilter: 'Category=IndexAndReindex' testRunTitleSuffix: ' Reindex' + # Around 52 cases. A floor of 25 fails a leg that has lost most of them. + minimumExpectedTests: '25' diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index b87f3a2017..111aec7ed7 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -104,10 +104,32 @@ jobs: } # Positive on DataStore, so a case missing that trait is invisible here rather than failing. - # The floor makes a filter or trait regression that empties this leg fail instead of pass. - $args = @('--filter-query', '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') + # Count what the filter selects before running, so a regression that empties this leg fails + # it. The floor cannot ride on --minimum-expected-tests: that is re-applied to the smaller + # pass --retry-failed-tests starts, so a floor above a retry pass fails a recovered flake. + # Around 3 cases run here, so 2 is below any legitimate count and above an emptied leg. + $filterQuery = '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]' + $minimumExpectedTests = 2 - Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=CosmosDb)&(Category=ExportLongRunning)]" + $listOutput = & dotnet $dll.FullName --list-tests --filter-query $filterQuery + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for filter $filterQuery failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for filter $filterQuery from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for filter $filterQuery (floor $minimumExpectedTests)" + if ($discovered -lt $minimumExpectedTests) { + throw "Filter $filterQuery selected $discovered test(s), below this leg's floor of $minimumExpectedTests. The leg would have run a fraction of its tests and reported success." + } + + $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') + + Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" & dotnet $dll.FullName @args if ($LASTEXITCODE -ne 0) { @@ -258,10 +280,32 @@ jobs: } # Positive on DataStore, so a case missing that trait is invisible here rather than failing. - # The floor makes a filter or trait regression that empties this leg fail instead of pass. - $args = @('--filter-query', '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') + # Count what the filter selects before running, so a regression that empties this leg fails + # it. The floor cannot ride on --minimum-expected-tests: that is re-applied to the smaller + # pass --retry-failed-tests starts, so a floor above a retry pass fails a recovered flake. + # Around 5 cases run here, so 3 is below any legitimate count and above an emptied leg. + $filterQuery = '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]' + $minimumExpectedTests = 3 + + $listOutput = & dotnet $dll.FullName --list-tests --filter-query $filterQuery + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for filter $filterQuery failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for filter $filterQuery from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for filter $filterQuery (floor $minimumExpectedTests)" + if ($discovered -lt $minimumExpectedTests) { + throw "Filter $filterQuery selected $discovered test(s), below this leg's floor of $minimumExpectedTests. The leg would have run a fraction of its tests and reported success." + } + + $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') - Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=SqlServer)&(Category=ExportLongRunning)]" + Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" & dotnet $dll.FullName @args if ($LASTEXITCODE -ne 0) { diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 749f063bb2..f6b677a1a1 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -155,6 +155,9 @@ jobs: containerAppName: '${{ parameters.containerAppName }}' appServiceType: 'SqlServer' categoryFilter: '${{ parameters.mainCategoryFilter }}' + # Around 2000 cases across the FHIR versions. A floor of 1000 leaves room for version + # differences and removed tests while failing a leg that has quietly lost its traits. + minimumExpectedTests: '1000' - ${{ if eq(parameters.runReindexJob, true) }}: - job: 'sqlE2eTests_Reindex' @@ -175,6 +178,8 @@ jobs: appServiceType: 'SqlServer' categoryFilter: 'Category=IndexAndReindex' testRunTitleSuffix: ' Reindex' + # Around 52 cases. A floor of 25 fails a leg that has lost most of them. + minimumExpectedTests: '25' - ${{ if eq(parameters.runBulkUpdateJob, true) }}: - job: 'sqlE2eTests_BulkUpdate' @@ -195,3 +200,5 @@ jobs: appServiceType: 'SqlServer' categoryFilter: 'Category=BulkUpdate' testRunTitleSuffix: ' BulkUpdate' + # Around 35 cases. A floor of 15 fails a leg that has lost most of them. + minimumExpectedTests: '15' From 8b6517c6c88a3293ea08159e786bc03eaa236f1b Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 00:31:43 -0700 Subject: [PATCH 090/105] Stop the last-resort fail-safe from crying wolf on a routine path Disposal replaying held messages is both the fail-safe for a caller that forgot them and the ordinary way the last surviving result reaches the bus, and it announced an internal error for both. Every run printed the line, so nobody had reason to believe it the one time it meant something. The deliberate handoff now says so, and the warning is left for the case it was written for. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SingleResultReportingTests.cs | 44 +++++++++++++++++++ .../RetryTestCase.cs | 27 ++++++++++-- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs index 3de0879934..e17b50ff3d 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs @@ -102,6 +102,50 @@ public void GivenTheUnderlyingBusAsksToStop_WhenTheWinnerIsReplayed_ThenTheReque Assert.False(RetryTestCase.ReportSingleResult(NoResultOutcome.ReplayEarlierAttempt, currentAttempt: null, earlierAttempt: earlier)); } + [Fact] + public void GivenADeliberateHandoffToDisposal_WhenTheBusIsDisposed_ThenItPublishesWithoutClaimingAnInternalError() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus current = DeferAbstention(inner); + + RetryTestCase.ReportSingleResult(NoResultOutcome.ReportNothing, current, earlierAttempt: null); + + string console = CaptureConsole(current.Dispose); + + Assert.Equal(new[] { nameof(ITestSkipped) }, inner.PublishedKinds); + Assert.DoesNotContain("Internal error", console, StringComparison.Ordinal); + } + + [Fact] + public void GivenMessagesNobodyResolved_WhenTheBusIsDisposed_ThenItStillPublishesAndSaysSo() + { + var inner = new RecordingMessageBus(); + RetryTestCase.FailureInterceptingMessageBus orphaned = DeferFailure(inner); + + string console = CaptureConsole(orphaned.Dispose); + + Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); + Assert.Contains("Internal error", console, StringComparison.Ordinal); + } + + private static string CaptureConsole(Action action) + { + System.IO.TextWriter original = Console.Out; + var captured = new System.IO.StringWriter(); + + try + { + Console.SetOut(captured); + action(); + } + finally + { + Console.SetOut(original); + } + + return captured.ToString(); + } + private static RetryTestCase.FailureInterceptingMessageBus DeferFailure(RecordingMessageBus inner) { var bus = new RetryTestCase.FailureInterceptingMessageBus(inner, deferFailures: true); diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 3d112c5fda..90e7d87704 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -474,6 +474,7 @@ internal static bool ReportSingleResult( // current attempt is still holding - an abstention, typically - is the only // result the test has, and is left for its disposal to publish. earlierAttempt?.DiscardDeferredMessages(); + currentAttempt?.PublishDeferredMessagesOnDispose(); return true; } } @@ -651,6 +652,7 @@ internal class FailureInterceptingMessageBus : IMessageBus private readonly bool _deferAbstentions; private readonly List _deferredMessages = new List(); private bool _deferring; + private bool _replayOnDisposeIsIntended; public FailureInterceptingMessageBus(IMessageBus innerBus, bool deferFailures, bool deferAbstentions = false) { @@ -808,14 +810,33 @@ public void DiscardDeferredMessages() _deferring = false; } + /// + /// Records that whatever is still held is meant to be published by disposal rather than + /// by an explicit replay, so that disposal does not report it as a leak. + /// + /// + /// Disposal replaying held messages is both the fail-safe for a caller that forgot them + /// and the ordinary way the last surviving result reaches the bus. Without a way to tell + /// those apart, the fail-safe announces an internal error on a routine path, and a reader + /// who sees that line on every run has no reason to believe it the one time it means + /// something. This marks the deliberate case so the warning stays rare enough to trust. + /// + public void PublishDeferredMessagesOnDispose() + { + _replayOnDisposeIsIntended = true; + } + public void Dispose() { // Don't dispose the inner bus - it's owned by the caller. - // Anything still deferred was never explicitly replayed or discarded. Dropping it - // would remove the test from the run, so fail safe by reporting it. + // Anything still deferred is reported either way, because dropping it would remove + // the test from the run. Only an unannounced one is a defect worth saying so about. if (_deferredMessages.Count > 0) { - Console.WriteLine($"[RetryFact] Internal error: {_deferredMessages.Count} deferred test message(s) were neither replayed nor discarded. Replaying them so the failure is not lost."); + if (!_replayOnDisposeIsIntended) + { + Console.WriteLine($"[RetryFact] Internal error: {_deferredMessages.Count} deferred test message(s) were neither replayed nor discarded. Replaying them so the failure is not lost."); + } try { From 0fec0a39cbc4d2a5af659829292e7266e410b8e4 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 00:43:29 -0700 Subject: [PATCH 091/105] Say why a deferred failure is not captured, and where parallelism comes from Three things that were only obvious from the outside: a failure arriving after an abstention has been deferred deliberately never sets LastFailureMessage, because the outcome that follows reports a failure anyway and capturing it would publish two results for one test; the E2E runner settings are not published, so the legs' parallel-none switch is the only thing keeping suites that share a server off each other; and the bus now writes diagnostics through an injectable sink so a test can read them without redirecting the console, which is process-wide and would have made those tests unsafe to run beside the rest. The extensions harness also joins the R4 and R5 solution filters, so the tests guarding this machinery are present in the solution a developer actually opens. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- R4.slnf | 1 + R5.slnf | 1 + build/jobs/e2e-tests.yml | 3 ++ .../SingleResultReportingTests.cs | 31 +++++-------------- .../RetryTestCase.cs | 21 +++++++++++-- ...oft.Health.Fhir.Shared.Tests.E2E.projitems | 5 +++ 6 files changed, 37 insertions(+), 25 deletions(-) diff --git a/R4.slnf b/R4.slnf index 2786e2ab36..4aa8e6bf07 100644 --- a/R4.slnf +++ b/R4.slnf @@ -4,6 +4,7 @@ "projects": [ "samples\\apps\\SmartLauncher\\SmartLauncher.csproj", "src\\Microsoft.Health.Extensions.Xunit\\Microsoft.Health.Extensions.Xunit.csproj", + "src\\Microsoft.Health.Extensions.Xunit.UnitTests\\Microsoft.Health.Extensions.Xunit.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Api.UnitTests\\Microsoft.Health.Fhir.R4.Api.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Api\\Microsoft.Health.Fhir.Api.csproj", "src\\Microsoft.Health.Fhir.Api.OpenIddict\\Microsoft.Health.Fhir.Api.OpenIddict.csproj", diff --git a/R5.slnf b/R5.slnf index 092fe04bdc..e10c9ccb30 100644 --- a/R5.slnf +++ b/R5.slnf @@ -4,6 +4,7 @@ "projects": [ "samples\\apps\\SmartLauncher\\SmartLauncher.csproj", "src\\Microsoft.Health.Extensions.Xunit\\Microsoft.Health.Extensions.Xunit.csproj", + "src\\Microsoft.Health.Extensions.Xunit.UnitTests\\Microsoft.Health.Extensions.Xunit.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Api.UnitTests\\Microsoft.Health.Fhir.R5.Api.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Api\\Microsoft.Health.Fhir.Api.csproj", "src\\Microsoft.Health.Fhir.Api.OpenIddict\\Microsoft.Health.Fhir.Api.OpenIddict.csproj", diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index f540ae4e11..515fa591d9 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -78,6 +78,9 @@ steps: # Kept at 1 as a last-ditch check that the run itself published something. Anything higher # is unsafe here for the retry reason above; the real floor is the discovery check. + # --parallel none is not optional and must not be dropped: these tests share one server, and + # the xunit.runner.json that would otherwise disable parallelism is deliberately not copied + # to a published layout, so the command line is the only place this is guaranteed. $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx', '--minimum-expected-tests', '1') Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery" diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs index e17b50ff3d..2dfc8352b8 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/SingleResultReportingTests.cs @@ -106,44 +106,29 @@ public void GivenTheUnderlyingBusAsksToStop_WhenTheWinnerIsReplayed_ThenTheReque public void GivenADeliberateHandoffToDisposal_WhenTheBusIsDisposed_ThenItPublishesWithoutClaimingAnInternalError() { var inner = new RecordingMessageBus(); + var diagnostics = new List(); RetryTestCase.FailureInterceptingMessageBus current = DeferAbstention(inner); + current.DiagnosticLog = diagnostics.Add; RetryTestCase.ReportSingleResult(NoResultOutcome.ReportNothing, current, earlierAttempt: null); - - string console = CaptureConsole(current.Dispose); + current.Dispose(); Assert.Equal(new[] { nameof(ITestSkipped) }, inner.PublishedKinds); - Assert.DoesNotContain("Internal error", console, StringComparison.Ordinal); + Assert.DoesNotContain(diagnostics, d => d.Contains("Internal error", StringComparison.Ordinal)); } [Fact] public void GivenMessagesNobodyResolved_WhenTheBusIsDisposed_ThenItStillPublishesAndSaysSo() { var inner = new RecordingMessageBus(); + var diagnostics = new List(); RetryTestCase.FailureInterceptingMessageBus orphaned = DeferFailure(inner); + orphaned.DiagnosticLog = diagnostics.Add; - string console = CaptureConsole(orphaned.Dispose); + orphaned.Dispose(); Assert.Equal(new[] { nameof(ITestFailed) }, inner.PublishedKinds); - Assert.Contains("Internal error", console, StringComparison.Ordinal); - } - - private static string CaptureConsole(Action action) - { - System.IO.TextWriter original = Console.Out; - var captured = new System.IO.StringWriter(); - - try - { - Console.SetOut(captured); - action(); - } - finally - { - Console.SetOut(original); - } - - return captured.ToString(); + Assert.Contains(diagnostics, d => d.Contains("Internal error", StringComparison.Ordinal)); } private static RetryTestCase.FailureInterceptingMessageBus DeferFailure(RecordingMessageBus inner) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 90e7d87704..7357dc67ab 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -663,6 +663,15 @@ public FailureInterceptingMessageBus(IMessageBus innerBus, bool deferFailures, b public string LastFailureMessage { get; private set; } + /// + /// Gets or sets where this bus writes its own diagnostics, defaulting to the console. + /// + /// + /// Exists so a test can read what was reported without redirecting the console, which is + /// process-wide state and would make any test doing it unsafe to run alongside others. + /// + public Action DiagnosticLog { get; set; } = Console.WriteLine; + public string LastFailureStackTrace { get; private set; } public bool IsAssertionFailure { get; private set; } @@ -692,6 +701,14 @@ public bool QueueMessage(IMessageSinkMessage message) { if (_deferring) { + // Once this attempt has held something back, everything after it is held too, so + // a failure arriving here never reaches the capture below and never sets + // LastFailureMessage. That under-reporting is deliberate. The only way to get + // here is for an abstention to have been deferred first, and the outcome that + // follows - the earlier attempt's failure being replayed - publishes a failure + // either way, so the run stays loud. Capturing it here instead would make this + // attempt look like a failure in its own right, and the two would be published + // as separate results for one test. _deferredMessages.Add(message); return true; } @@ -835,7 +852,7 @@ public void Dispose() { if (!_replayOnDisposeIsIntended) { - Console.WriteLine($"[RetryFact] Internal error: {_deferredMessages.Count} deferred test message(s) were neither replayed nor discarded. Replaying them so the failure is not lost."); + DiagnosticLog($"[RetryFact] Internal error: {_deferredMessages.Count} deferred test message(s) were neither replayed nor discarded. Replaying them so the failure is not lost."); } try @@ -848,7 +865,7 @@ public void Dispose() // already unwinding. Throwing from here would replace whatever actually went // wrong with a reporting error, so the failure to replay is logged and the // original exception is left to propagate. - Console.WriteLine($"[RetryFact] Failed to replay {_deferredMessages.Count} deferred test message(s): {e.GetType().FullName}: {e.Message}"); + DiagnosticLog($"[RetryFact] Failed to replay {_deferredMessages.Count} deferred test message(s): {e.GetType().FullName}: {e.Message}"); } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems index 4d13e103dc..39f4f03907 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems @@ -160,6 +160,11 @@ xunit.runner.json PreserveNewest + Never From b3c2a871e09ada9f7a1fa699725d14d3f2a382e1 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 01:52:29 -0700 Subject: [PATCH 092/105] Prove which test classes no CI leg is able to select A leg that filters on a DataStore trait can only select a class that either carries that trait by hand or gets it from an argument-set attribute the framework expands into per-store variants. A class with neither is silently invisible: no leg selects it, every leg still exits zero, and nothing says so. Four E2E classes are in that state today, and were on main too, so this adds a guard rather than a behaviour change. The guard fails on an unexpected unreachable class and equally on an allow-list entry that has since become reachable, so the list cannot quietly rot into a blanket exemption. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AssemblyChecker.cs | 52 +++++++++++++++++++ .../AssemblyValidationsTester.cs | 46 ++++++++++++++++ .../AssemblyValidationsTests.cs | 41 +++++++++++++++ 3 files changed, 139 insertions(+) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs index 26a37c693f..e142e34837 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs @@ -154,6 +154,58 @@ private static bool JoinsCollection(Type candidate, string collectionName, Type return false; } + /// + /// Looks for test classes that no data-store-filtered leg can select. + /// + /// + /// Every E2E and export leg selects positively on the DataStore trait. A class gets + /// that trait one of two ways: the custom framework expands a + /// into one variant per store and traits each, or + /// the class writes the trait out by hand. A class that does neither is not skipped and does + /// not error - it simply matches no leg's filter, so it is never constructed, never run, and + /// never counted, while every leg reports success. Passing a store through + /// [InlineData] looks like it should help and does not: inline arguments are values, + /// not traits, and the filter never sees them. + /// + /// Assembly under analysis. + /// The trait the legs select on. + /// + /// The attribute the framework expands into per-store variants. Passed in because it is + /// declared alongside the tests rather than here. + /// + /// The test classes no leg selects. + public static IReadOnlyList ScanForTestClassesNoStoreFilteredLegSelects(Assembly assembly, string traitName, Type argumentSetAttributeType) + { + EnsureArg.IsNotNull(assembly, nameof(assembly)); + EnsureArg.IsNotNullOrWhiteSpace(traitName, nameof(traitName)); + EnsureArg.IsNotNull(argumentSetAttributeType, nameof(argumentSetAttributeType)); + + return assembly.GetTypes() + .Where(t => t.IsClass && IsTestClass(t)) + .Where(t => !IsExpandedIntoStoreVariants(t, argumentSetAttributeType) && !DeclaresTrait(t, traitName)) + .ToList(); + } + + private static bool IsExpandedIntoStoreVariants(Type type, Type argumentSetAttributeType) + { + // The attribute is declared on the class or on individual methods, and either is enough + // for the framework to expand that class into per-store variants. + return CarriesArgumentSets(type.CustomAttributes, argumentSetAttributeType) || + type.GetMethods().Any(m => CarriesArgumentSets(m.CustomAttributes, argumentSetAttributeType)); + } + + private static bool CarriesArgumentSets(IEnumerable attributes, Type argumentSetAttributeType) + { + return attributes.Any(a => argumentSetAttributeType.IsAssignableFrom(a.AttributeType)); + } + + private static bool DeclaresTrait(Type type, string traitName) + { + return type.CustomAttributes + .Where(a => a.AttributeType == typeof(TraitAttribute) && a.ConstructorArguments.Count > 0) + .Any(a => string.Equals(a.ConstructorArguments[0].Value as string, traitName, StringComparison.Ordinal)); + } + private static bool IsTestClass(Type type) { if (!type.IsClass) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs index e8eefc6100..2599f4fb64 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs @@ -87,6 +87,52 @@ public static void EnsureTestFrameworkIsDeclared(Assembly assembly, Type expecte $"filters them out, and the leg passes with the suite missing rather than failing."); } + /// + /// Fails when a test class is invisible to every leg that selects on a data store. + /// + /// Assembly under analysis. + /// The trait the legs select on. + /// The attribute the framework expands into per-store variants. + /// + /// Classes already known to run in no leg, named in full. Each one is a test nobody runs; the + /// list exists so that state is written down and cannot grow without someone saying so. + /// + public static void EnsureEveryTestClassIsSelectedBySomeLeg( + Assembly assembly, + string traitName, + Type argumentSetAttributeType, + IReadOnlyCollection knownUnreachableClasses) + { + IReadOnlyList unreachable = AssemblyChecker.ScanForTestClassesNoStoreFilteredLegSelects(assembly, traitName, argumentSetAttributeType); + + string[] unexpected = unreachable + .Select(type => type.FullName) + .Where(name => !knownUnreachableClasses.Contains(name)) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + string[] staleEntries = knownUnreachableClasses + .Where(name => !unreachable.Any(type => string.Equals(type.FullName, name, StringComparison.Ordinal))) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + if (unexpected.Length > 0) + { + Assert.Fail( + $"Assembly '{assembly}' has test classes that no leg selects, because they neither declare a '{traitName}' trait nor carry the fixture argument sets " + + $"the framework expands into one variant per store. Nothing skips or errors for these: every leg filters them out, runs without them, and passes. " + + $"Give each one argument sets or an explicit '{traitName}' trait, or add it to the known list with a reason.{Environment.NewLine}" + + string.Join(Environment.NewLine, unexpected)); + } + + if (staleEntries.Length > 0) + { + Assert.Fail( + $"Assembly '{assembly}' names classes as unreachable that a leg now selects. Remove them from the known list so it keeps describing what is " + + $"actually unrun.{Environment.NewLine}{string.Join(Environment.NewLine, staleEntries)}"); + } + } + private static void AssertTestClasses(Assembly currentAssembly, string traitName, IEnumerable types) { if (types == null || !types.Any()) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs index 181832f8ca..311e244abf 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs @@ -3,7 +3,10 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; +using System.Collections.Generic; using System.Reflection; +using Microsoft.Health.Extensions.Xunit; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; @@ -32,6 +35,28 @@ public sealed class AssemblyValidationsTests { private static Assembly _currentAssembly = Assembly.GetAssembly(typeof(AssemblyValidationsTests)); + private static readonly HashSet KnownUnreachableTestClasses = new HashSet(StringComparer.Ordinal) + { + // Untraited on purpose. This copy is also compiled into the integration assemblies, whose + // legs filter by excluding a store rather than naming one, and an exclusion drops a class + // that names both. Traiting it here would silence it there, so the E2E copy above carries + // the traits instead and this one runs in the integration legs only. + "Microsoft.Health.Fhir.Shared.Tests.AssemblyValidationsTests", + + // Pre-existing, and unrun before this PR as well: the legs used to select by name + // substring, which these matched no better than they match the trait filters. They need a + // container registry and a convert-data configuration that no leg sets up, so traiting + // them would move them from never running to reliably failing. + "Microsoft.Health.Fhir.Tests.E2E.Rest.ContainerRegistryTemplateUploaderTests", + "Microsoft.Health.Fhir.Tests.E2E.Rest.ConvertDataTestModeTests", + + // Takes its data store as an [InlineData] argument, which is a value and not a trait, so + // no filter can see it. Traiting the class would run every row in both legs, including + // the row written for the other store, so this needs per-row traits rather than a + // one-line fix. + "Microsoft.Health.Fhir.Tests.E2E.Rest.Search.SearchParameterInitializationTests", + }; + [Fact] public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveCategoryTrait() { @@ -49,5 +74,21 @@ public void GivenCurrentAssembly_WhenScanned_EnsureNoSharedCollectionDefinitionC { AssemblyValidationsTester.EnsureNoSharedCollectionDefinitionCarriesTraits(_currentAssembly); } + + /// + /// Every leg selects positively on , so a class that carries no such + /// trait runs nowhere while every leg stays green. The classes below are in that state + /// already, from before the legs filtered on traits at all; they are listed so the situation + /// is visible and cannot quietly grow. + /// + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureEveryTestClassIsSelectedBySomeLeg() + { + AssemblyValidationsTester.EnsureEveryTestClassIsSelectedBySomeLeg( + _currentAssembly, + nameof(DataStore), + typeof(FixtureArgumentSetsAttribute), + KnownUnreachableTestClasses); + } } } From 91efe88c024aa51ea011c1081d016dd57bb8d0fb Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 01:52:44 -0700 Subject: [PATCH 093/105] Give the integration legs a floor too, and make every leg state its own The integration legs select by excluding the other store's trait. An exclusion is the mirror hazard of a positive filter: a class that gains a trait it did not have before drops out of the leg, and the leg still passes. They had no floor, so add the same pre-run discovery count the other legs use, with the identical filter arguments the run itself uses. The E2E floor parameter loses its default. A default is the one value that is certainly wrong for a leg whose author never considered the question, and it would apply silently. All five call sites already pass one. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/e2e-tests.yml | 5 +++-- build/jobs/run-cosmos-tests.yml | 32 +++++++++++++++++++++++++++++++- build/jobs/run-sql-tests.yml | 32 +++++++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index 515fa591d9..738445143a 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -17,10 +17,11 @@ parameters: # pass rather than passed to the run as --minimum-expected-tests, because that option is applied # again to the much smaller pass --retry-failed-tests starts: a floor above the size of a retry # pass turns a flake that recovered into a failed leg (exit 9). Set well below the real counts so -# version differences and legitimately removed tests do not fail a leg that is working. +# version differences and legitimately removed tests do not fail a leg that is working. There is +# deliberately no default: a new leg has to state its own floor, because the only value that would +# be safe to inherit is one too low to ever fire. - name: minimumExpectedTests type: string - default: '1' steps: - template: e2e-tests-extract.yml diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index ef6dce3358..f0a9bab941 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -72,7 +72,7 @@ jobs: arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' - task: PowerShell@2 - displayName: 'Verify xUnit config' + displayName: 'Verify xUnit config and test counts' inputs: targetType: inline pwsh: true @@ -83,6 +83,15 @@ jobs: throw "No integration test projects found under $(Pipeline.Workspace)/source/test." } + # Around 293 tests select here. This leg filters by excluding the other store, and an + # exclusion drops a case that names either value, so a class that starts carrying both - + # by inheriting traits from a shared collection definition, for instance - disappears from + # this leg and from the other one at the same time, leaving both green and much smaller. + # Counting discovery is the only thing that notices. The floor is checked here rather than + # passed as --minimum-expected-tests because that option is applied again to the smaller + # pass --retry-failed-tests starts, where any real floor would fail a recovered flake. + $minimumExpectedTests = 150 + foreach ($project in $projects) { $outputDir = [System.IO.Path]::Combine($project.Directory.FullName, "bin", "$(buildConfiguration)", "$(defaultBuildFramework)") $configPath = Join-Path $outputDir "xunit.runner.json" @@ -92,6 +101,27 @@ jobs: Write-Host "xUnit config for $($project.BaseName): $configPath" Get-Content -Path $configPath | ForEach-Object { Write-Host $_ } + + $assemblyPath = Join-Path $outputDir "$($project.BaseName).dll" + if (-not (Test-Path $assemblyPath)) { + throw "Test assembly not found for $($project.FullName) at $assemblyPath" + } + + $listOutput = & dotnet $assemblyPath --list-tests --filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer" + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for $($project.BaseName) failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for $($project.BaseName) from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for $($project.BaseName) (floor $minimumExpectedTests)" + if ($discovered -lt $minimumExpectedTests) { + throw "$($project.BaseName) selected $discovered test(s), below this leg's floor of $minimumExpectedTests. The leg would have run a fraction of its tests and reported success." + } } - task: DotNetCoreCLI@2 diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index f6b677a1a1..32ebaa6294 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -64,7 +64,7 @@ jobs: arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' - task: PowerShell@2 - displayName: 'Verify xUnit config' + displayName: 'Verify xUnit config and test counts' inputs: targetType: inline pwsh: true @@ -75,6 +75,15 @@ jobs: throw "No integration test projects found under $(Pipeline.Workspace)/source/test." } + # Around 441 tests select here. This leg filters by excluding the other store, and an + # exclusion drops a case that names either value, so a class that starts carrying both - + # by inheriting traits from a shared collection definition, for instance - disappears from + # this leg and from the other one at the same time, leaving both green and much smaller. + # Counting discovery is the only thing that notices. The floor is checked here rather than + # passed as --minimum-expected-tests because that option is applied again to the smaller + # pass --retry-failed-tests starts, where any real floor would fail a recovered flake. + $minimumExpectedTests = 250 + foreach ($project in $projects) { $outputDir = [System.IO.Path]::Combine($project.Directory.FullName, "bin", "$(buildConfiguration)", "$(defaultBuildFramework)") $configPath = Join-Path $outputDir "xunit.runner.json" @@ -84,6 +93,27 @@ jobs: Write-Host "xUnit config for $($project.BaseName): $configPath" Get-Content -Path $configPath | ForEach-Object { Write-Host $_ } + + $assemblyPath = Join-Path $outputDir "$($project.BaseName).dll" + if (-not (Test-Path $assemblyPath)) { + throw "Test assembly not found for $($project.FullName) at $assemblyPath" + } + + $listOutput = & dotnet $assemblyPath --list-tests --filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb" + if ($LASTEXITCODE -ne 0) { + throw "Listing tests for $($project.BaseName) failed with exit code $LASTEXITCODE" + } + + $found = $listOutput | Select-String -Pattern 'found\s+(\d+)\s+test' | Select-Object -First 1 + if (-not $found) { + throw "Could not read a discovered test count for $($project.BaseName) from the runner's output." + } + + $discovered = [int]$found.Matches[0].Groups[1].Value + Write-Host "Discovered $discovered test(s) for $($project.BaseName) (floor $minimumExpectedTests)" + if ($discovered -lt $minimumExpectedTests) { + throw "$($project.BaseName) selected $discovered test(s), below this leg's floor of $minimumExpectedTests. The leg would have run a fraction of its tests and reported success." + } } - task: DotNetCoreCLI@2 From 493afaf5f4779cd199d48d7741eb12e0f03ce449 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 01:52:45 -0700 Subject: [PATCH 094/105] Correct two comments that described behaviour the code does not have One claimed a deferral can only originate from an abstention; ordinary deferred failures, the common retry path, set the same flag. The other described retry selection as applying to transient failures, which reads as a transience test that ShouldRetry does not perform. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryTestCase.cs | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 7357dc67ab..3c8ff23ea1 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -101,8 +101,15 @@ public RetryTestCase( : $"{TestMethod?.TestClass?.TestClassName}.{TestMethod?.MethodName}"; /// - /// Runs the test case, retrying transient failures up to the configured number of attempts. + /// Runs the test case, retrying a failed attempt up to the configured number of attempts. /// + /// + /// Nothing here decides whether a failure was transient. Every failure is retried except one + /// classified as an assertion failure, which is retried only when the test asks for it with + /// RetryOnAssertionFailure - see . A deterministic failure is + /// therefore retried too, and fails every attempt, which costs time but reports the same + /// result. Read a passing retry as "this attempt passed", not as "the failure was transient". + /// /// Whether explicit tests should be run. /// The message bus that test results are reported to. /// The arguments to pass to the test class constructor. @@ -701,14 +708,19 @@ public bool QueueMessage(IMessageSinkMessage message) { if (_deferring) { - // Once this attempt has held something back, everything after it is held too, so - // a failure arriving here never reaches the capture below and never sets - // LastFailureMessage. That under-reporting is deliberate. The only way to get - // here is for an abstention to have been deferred first, and the outcome that - // follows - the earlier attempt's failure being replayed - publishes a failure - // either way, so the run stays loud. Capturing it here instead would make this - // attempt look like a failure in its own right, and the two would be published - // as separate results for one test. + // This attempt is already holding messages back - either a failure it means to + // retry, or an abstention that has to compete with an earlier failure - so + // everything after it is held too, and is neither published now nor examined + // below. The usual thing arriving here is the ITestFinished that follows a + // deferred failure, and it must not reach the inner bus while the result it + // belongs to is still being decided. + // + // A failure arriving here therefore never sets LastFailureMessage. On the retry + // path that costs nothing, because the failure that started the deferral set it + // already. On the abstention path it is deliberate: the outcome that follows + // replays the earlier attempt's failure, so a failure is published either way, + // and treating this attempt as a failure in its own right would publish two + // results for one test. _deferredMessages.Add(message); return true; } From 7fb265ed523673d3029f6d1253a33ca742fc0788 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 02:31:58 -0700 Subject: [PATCH 095/105] Stop the cancellation scenarios deciding by stopwatch who goes first Both scenarios need one collection to cancel the run while the other is mid-attempt, and both arranged that by sleeping: the canceller slept, the other slept longer, and the interleave was whatever the machine's speed made it. On a loaded agent the cancellation lands after the other side's window has closed, and the run then reports a failure about the scenario rather than about the product. That is a test suite lying about the code, which is the one thing this whole change exists to prevent. Each side now waits to be told, so the order holds at any speed. The waits are asynchronous because a blocking one holds a runner thread: with a blocking wait the collection being waited for did not get a thread until the waiter had already given up, which was measurably worse than the sleeps it replaced. The run-duration backstop moves from 20s to 60s. It only has to separate a cancelled run from one that sat through three 30s retry delays, so 60s still says exactly that while no longer failing a correct run that was merely slow to start a child process. Verified: the old timing fails deterministically when the canceller is made 8s late, the new handshake passes under the same condition, and the harness is green over eight consecutive runs including three with every core saturated. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../CancellationTriggerTests.cs | 13 +++- .../PassesWhileCancellingTests.cs | 19 +++-- .../PassingAttemptHandshake.cs | 72 +++++++++++++++++++ .../CancellationTriggerTests.cs | 15 +++- .../DeferredFailureTests.cs | 4 ++ .../RetryDelayHandshake.cs | 48 +++++++++++++ .../RetryFactCancellationTests.cs | 20 ++++-- 7 files changed, 175 insertions(+), 16 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassingAttemptHandshake.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/RetryDelayHandshake.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs index 76e4e09e8a..3001899c56 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/CancellationTriggerTests.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Threading; +using System.Threading.Tasks; using Xunit; namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt @@ -19,9 +19,16 @@ public class CancellationTriggerTests /// Deliberately fails to trigger cancellation of the whole run. /// [Fact] - public void FailsToTriggerCancellation() + public async Task FailsToTriggerCancellation() { - Thread.Sleep(800); + // Waiting for the sibling collection to announce its second attempt is what makes this + // failure land while that attempt is still running. A sleep here would only be a guess + // at how long the sibling takes to get there. The wait is asynchronous so that it does + // not hold a worker thread the sibling collection may need in order to get there at all. + await Task.WhenAny( + PassingAttemptHandshake.SecondAttemptRunning, + Task.Delay(PassingAttemptHandshake.Budget)); + Assert.Fail("ASSET: deliberate failure that cancels the run"); } } diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs index 34670ccaf9..7116c4a038 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassesWhileCancellingTests.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System.Threading; +using System.Threading.Tasks; using Xunit; namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt @@ -27,20 +28,26 @@ public class PassesWhileCancellingTests /// the sibling collection to cancel the run, and passes anyway. /// [RetryFact(MaxRetries = 2, DelayBetweenRetriesMs = 50, RetryOnAssertionFailure = true)] - public void PassingAttemptIsNotOverriddenByEarlierFailure() + public async Task PassingAttemptIsNotOverriddenByEarlierFailure() { if (Interlocked.Increment(ref _attempts) == 1) { Assert.Fail("ASSET: first attempt fails so a failure is deferred"); } - // Stay in the second attempt while the sibling collection cancels the run, then prove - // that is really what happened. Without this the attempt simply passes on its own and - // the scenario would report the expected result while exercising nothing. - Thread.Sleep(3000); + // Tell the sibling collection it may now cancel the run. Until this point cancelling + // would land before the attempt that has to survive it had started. + PassingAttemptHandshake.AnnounceSecondAttempt(); + + // Stay in the second attempt until the sibling collection actually cancels the run, then + // prove that is really what happened. Without this the attempt simply passes on its own + // and the scenario would report the expected result while exercising nothing. Waiting for + // the cancellation rather than sleeping for a fixed period keeps that true on a machine + // slow enough that a fixed window would close first. + bool cancelled = await PassingAttemptHandshake.WaitForCancellationAsync(TestContext.Current.CancellationToken); Assert.True( - TestContext.Current.CancellationToken.IsCancellationRequested, + cancelled, "ASSET: the run was not cancelled while this attempt was running, so this scenario did not exercise a pass during cancellation."); } } diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassingAttemptHandshake.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassingAttemptHandshake.cs new file mode 100644 index 0000000000..a88187d6d9 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringPassingAttempt/PassingAttemptHandshake.cs @@ -0,0 +1,72 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt +{ + /// + /// Orders the two collections in this scenario against each other. + /// + /// + /// + /// The scenario needs one collection to cancel the run while the other is part way through a + /// retry attempt. Arranging that with sleeps makes the outcome depend on how fast the machine + /// is: on a loaded agent the cancelling collection's failure can land after the other + /// collection's window has closed, and the scenario then fails with a message about itself + /// rather than reporting anything about the behaviour under test. Waiting for an announcement + /// orders the two by construction instead. + /// + /// + /// Both waits here are asynchronous, which is not a matter of taste. The runner executes + /// collections on a bounded set of worker threads, so a collection that blocks a thread while + /// waiting for its sibling can leave the sibling with no thread to run on. That was measured: + /// with a blocking wait the collection being waited for did not start until after the waiter + /// had already given up, which is the same failure this handshake exists to remove. + /// + /// + internal static class PassingAttemptHandshake + { + /// + /// Bounds every wait so that a scenario which has stopped interleaving fails with its own + /// explanation, rather than hanging until the runner's timeout reports nothing useful. + /// + internal static readonly TimeSpan Budget = TimeSpan.FromSeconds(30); + + private static readonly TaskCompletionSource SecondAttemptRunningSource = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Gets a task that completes once the retrying test's second attempt is running, which is + /// the point from which cancelling the run exercises what this scenario exists to check. + /// + internal static Task SecondAttemptRunning => SecondAttemptRunningSource.Task; + + /// + /// Announces that the retrying test's second attempt is running. + /// + internal static void AnnounceSecondAttempt() => SecondAttemptRunningSource.TrySetResult(); + + /// + /// Waits for the run to be cancelled, giving up after . + /// + /// The run's cancellation token. + /// true if the run was cancelled; false if the budget expired first. + internal static async Task WaitForCancellationAsync(CancellationToken cancellationToken) + { + try + { + await Task.Delay(Budget, cancellationToken); + return false; + } + catch (OperationCanceledException) + { + return true; + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs index 23fca9965a..30864941b3 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/CancellationTriggerTests.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Threading; +using System.Threading.Tasks; using Xunit; namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay @@ -19,9 +19,18 @@ public class CancellationTriggerTests /// Deliberately fails to trigger cancellation of the whole run. /// [Fact] - public void FailsToTriggerCancellation() + public async Task FailsToTriggerCancellation() { - Thread.Sleep(700); + // Wait to be told the sibling has failed an attempt, so this failure cannot land before + // there is a retry delay to cancel during. The wait is asynchronous so it does not hold + // a worker thread the sibling collection may need. + await Task.WhenAny(RetryDelayHandshake.AttemptFailing, Task.Delay(RetryDelayHandshake.Budget)); + + // A short settle on top of that ordering, so the sibling is inside its 30s delay rather + // than on the boundary of entering it. This is a margin, not a race: the ordering above + // already happened. + await Task.Delay(700); + Assert.Fail("ASSET: deliberate failure that cancels the run"); } } diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs index 0fdedee152..2fe41a867b 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/DeferredFailureTests.cs @@ -28,6 +28,10 @@ public class DeferredFailureTests [RetryFact(MaxRetries = 3, DelayBetweenRetriesMs = 30000, RetryOnAssertionFailure = true)] public void FailureSurvivesCancellationDuringRetryDelay() { + // Tell the sibling collection it may now cancel the run: the retry delay this scenario + // needs it cancelled during begins the moment this attempt fails. + RetryDelayHandshake.AnnounceAttemptFailing(); + Assert.Fail("ASSET: failure that must survive cancellation"); } } diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/RetryDelayHandshake.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/RetryDelayHandshake.cs new file mode 100644 index 0000000000..d13dd7a074 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/CancelledDuringRetryDelay/RetryDelayHandshake.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay +{ + /// + /// Orders the two collections in this scenario against each other. + /// + /// + /// + /// The scenario needs one collection to cancel the run while the other is waiting to retry. + /// Deciding when to cancel by sleeping makes that depend on how fast the machine is, so the + /// cancelling collection waits to be told the retrying test has failed an attempt instead. + /// + /// + /// The wait is asynchronous so that it does not hold one of the runner's bounded worker + /// threads. A blocking wait here can leave the collection being waited for with no thread to + /// run on, which turns the handshake into the stall it was meant to prevent. + /// + /// + internal static class RetryDelayHandshake + { + /// + /// Bounds the wait so that a scenario which has stopped interleaving fails with its own + /// explanation, rather than hanging until the runner's timeout reports nothing useful. + /// + internal static readonly TimeSpan Budget = TimeSpan.FromSeconds(30); + + private static readonly TaskCompletionSource AttemptFailingSource = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Gets a task that completes as the retrying test fails an attempt, immediately before the + /// retry delay this scenario needs the run to be cancelled during. + /// + internal static Task AttemptFailing => AttemptFailingSource.Task; + + /// + /// Announces that the retrying test is failing an attempt. + /// + internal static void AnnounceAttemptFailing() => AttemptFailingSource.TrySetResult(); + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs index b241285735..c60c72fb75 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryFactCancellationTests.cs @@ -14,17 +14,29 @@ namespace Microsoft.Health.Extensions.Xunit.UnitTests /// cancelled part way through, which is where a retrying test can either lose a real failure /// or invent one that never happened. /// + /// + /// Both scenarios need their two collections to make progress at the same time: one cancels the + /// run while the other is mid-attempt. The scenarios arrange that themselves by waiting + /// asynchronously for each other, so it holds however many threads the runner uses. Pinning a + /// thread count from here was measured to make it worse rather than better: with two threads and + /// a blocking wait, the collection being waited for did not get a thread until the waiter had + /// already given up. + /// public class RetryFactCancellationTests { private const string RetryDelayPrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringRetryDelay."; private const string PassingAttemptPrefix = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.CancelledDuringPassingAttempt."; /// - /// Comfortably above what a cancelled run needs, and comfortably below the 30s retry delay - /// the scenario would otherwise sit through, so the assertion distinguishes the two without - /// being sensitive to how quickly the run starts up. + /// What this budget has to separate is a cancelled run from one that sat through its retry + /// delays instead. Those are 30s apart and there are three of them, so a run that was not + /// cancelled cannot finish in less than 30s and in practice takes about 90s. A cancelled run + /// takes seconds. Setting the line at 60s therefore still distinguishes the two, while + /// leaving room for a loaded agent to be slow at starting a child process without turning a + /// correct run red: this assertion is a backstop, and the scenario's own handshake is what + /// makes the cancellation land where it should. /// - private static readonly TimeSpan CancellationBudget = TimeSpan.FromSeconds(20); + private static readonly TimeSpan CancellationBudget = TimeSpan.FromSeconds(60); /// /// A failing attempt's result is held back in case a later attempt supersedes it. If the From 28b8cb4bac34ffeb0a54e52b6f6f77e9a563d1e7 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 02:39:10 -0700 Subject: [PATCH 096/105] Say out loud that assembly fixtures nothing asks for are still created Nine test assemblies declare an assembly fixture, name it in no constructor anywhere, and depend entirely on what its constructor does: install the FHIR model info provider. This change removed the local implementation that used to guarantee that and now leans on the framework to do it. Nothing anywhere asserted the guarantee, so if it ever stopped holding the provider would go uninstalled and the failure would surface far from the cause. A scenario now declares such a fixture the same way and a test reads whether it was constructed before that test ran, which is precisely the property the nine assemblies rely on. Verified by removing the declaration: the scenario then fails, and passes again once it is restored. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AssemblyInfo.cs | 8 ++++ .../AssemblyFixtureLifecycleTests.cs | 27 +++++++++++++ .../AssemblyFixtureProbe.cs | 25 ++++++++++++ .../RecordingAssemblyFixture.cs | 23 +++++++++++ .../AssemblyFixtureLifecycleTests.cs | 40 +++++++++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureLifecycleTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureProbe.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/RecordingAssemblyFixture.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/AssemblyFixtureLifecycleTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs index e576b6fa90..9abb3a9860 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/AssemblyInfo.cs @@ -4,8 +4,16 @@ // ------------------------------------------------------------------------------------------------- using Microsoft.Health.Extensions.Xunit; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle; using Xunit; // Exercise the same custom framework the real test projects use, so these assets // cover the discoverer and executor shims and not just the retry test case. [assembly: TestFramework(typeof(CustomXunitTestFramework))] + +// Nine test assemblies declare an assembly fixture that no test class asks for and rely on its +// constructor alone to install the FHIR model info provider. Nothing they contain asserts that the +// fixture ran, so if the framework stopped creating unrequested fixtures those assemblies would fail +// in whatever way a missing provider happens to look like. Declaring one here the same way makes +// that assumption something a test can see. +[assembly: AssemblyFixture(typeof(RecordingAssemblyFixture))] diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureLifecycleTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureLifecycleTests.cs new file mode 100644 index 0000000000..a13e9f991d --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureLifecycleTests.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle +{ + /// + /// Observes whether an assembly fixture that nothing asks for was still constructed. + /// + public class AssemblyFixtureLifecycleTests + { + /// + /// The fixture is declared on the assembly and requested by no constructor anywhere, which is + /// exactly how the real test assemblies use one. + /// + [Fact] + public void GivenAnAssemblyFixtureNothingAsksFor_WhenATestRuns_ThenTheFixtureWasAlreadyConstructed() + { + Assert.True( + AssemblyFixtureProbe.Constructed, + "The assembly fixture had not been constructed by the time a test ran, so an assembly fixture used only for what its constructor does would have had no effect."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureProbe.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureProbe.cs new file mode 100644 index 0000000000..8c13cb896a --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/AssemblyFixtureProbe.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle +{ + /// + /// Records that the assembly fixture was constructed, so a test can tell whether that happened + /// before it ran. + /// + public static class AssemblyFixtureProbe + { + private static volatile bool _constructed; + + /// + /// Gets or sets a value indicating whether the assembly fixture has been constructed. + /// + public static bool Constructed + { + get => _constructed; + set => _constructed = value; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/RecordingAssemblyFixture.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/RecordingAssemblyFixture.cs new file mode 100644 index 0000000000..b33f302630 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/AssemblyFixtureLifecycle/RecordingAssemblyFixture.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle +{ + /// + /// An assembly fixture whose only effect is what its constructor does, which is how the real test + /// assemblies use one: no test class asks for it, so a framework that created assembly fixtures + /// only on demand would never construct it and the effect would simply not happen. + /// + public sealed class RecordingAssemblyFixture + { + /// + /// Initializes a new instance of the class. + /// + public RecordingAssemblyFixture() + { + AssemblyFixtureProbe.Constructed = true; + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssemblyFixtureLifecycleTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssemblyFixtureLifecycleTests.cs new file mode 100644 index 0000000000..625b46eb6b --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssemblyFixtureLifecycleTests.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Verifies that the custom framework still creates an assembly fixture that no test asks for. + /// + /// + /// This repository has nine test assemblies whose only use of an assembly fixture is the side + /// effect of its constructor - installing the FHIR model info provider - and none of them names + /// the fixture as a constructor argument anywhere. If unrequested fixtures were created lazily, + /// or not at all, the provider would never be installed and the failure would land somewhere far + /// from the cause. Nothing else in the repository states that expectation, so it is stated here. + /// + public class AssemblyFixtureLifecycleTests + { + /// + /// The fixture is declared on the assets assembly and requested by nothing, so a test seeing + /// it already constructed is the whole of what the nine assemblies depend on. + /// + [Fact] + public void GivenAnAssemblyFixtureNoTestAsksFor_WhenTheRunReachesATest_ThenTheFixtureHasBeenConstructed() + { + TestAssetRun run = TestAssetRunner.Run("AssemblyFixtureLifecycle"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + ["Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.AssemblyFixtureLifecycle.AssemblyFixtureLifecycleTests.GivenAnAssemblyFixtureNothingAsksFor_WhenATestRuns_ThenTheFixtureWasAlreadyConstructed"] = "Passed", + }); + } + } +} From 8eab51572451c14270b320aa40fd56d7fe387b25 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 03:04:02 -0700 Subject: [PATCH 097/105] Stop a failing test riding out of CI as a skip The retry extension every leg runs with treats an attempt that did not fail as a recovery. A test that fails and then skips - Assert.Skip, or a SkippableFact deciding at runtime that its environment is not there - is therefore reported as recovered: the final TRX records NotExecuted, the runner exits 0 and the leg is green. Reproduced here as a scenario: with a second retried test present to keep the run non-empty, an attempt of 2 failed followed by 1 passed + 1 skipped prints "Passed!" and exits 0. Nothing in the repository skips dynamically today, so this is a trap laid for the next person rather than a live hole, and it is invisible at review time because the skip and the failure are in different tests. Every leg now reconciles the attempts it kept: Assert-RetriedFailuresPassed reads the per-attempt TRX files the runner leaves under Retries/ and fails the step unless every test that failed an attempt is Passed in the final one. Results are keyed by (assembly, test name), because shared test files are compiled into several assemblies and the names alone collide. The rationale comment claiming the runner's exit code was the retry-aware verdict was rewritten in all seven blocks: it is that only for tests that end up failing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/build.yml | 28 +++- build/jobs/e2e-tests.yml | 15 ++- build/jobs/run-cosmos-tests.yml | 15 ++- build/jobs/run-export-tests.yml | 30 ++++- build/jobs/run-sql-tests.yml | 15 ++- .../scripts/Assert-RetriedFailuresPassed.ps1 | 125 ++++++++++++++++++ ....Health.Extensions.Xunit.TestAssets.csproj | 6 + .../FailThenSkipOnRetryTests.cs | 73 ++++++++++ .../RetriedFailureOutcomeTests.cs | 77 +++++++++++ .../TestAssetRunner.cs | 30 ++++- 10 files changed, 390 insertions(+), 24 deletions(-) create mode 100644 build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FailThenSkipOnRetry/FailThenSkipOnRetryTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetriedFailureOutcomeTests.cs diff --git a/build/jobs/build.yml b/build/jobs/build.yml index ff6555c7d7..e28db4bcb6 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -41,6 +41,13 @@ steps: publishTestResults: false env: platformOptions__resultDirectory: '$(Agent.TempDirectory)/unit-test-results' + - task: PowerShell@2 + displayName: 'Reconcile retried unit test failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/unit-test-results"' + condition: succeeded() - task: PublishTestResults@2 displayName: 'Publish unit test results' inputs: @@ -52,9 +59,10 @@ steps: # attempt under Retries///, all of which this glob matches. Attempt 1 therefore # keeps a Failed record for a test that later passed, and mergeTestResults does not # resolve the two, so failing the task on published results would cancel out every - # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, - # non-zero when they are exhausted -- and the step that runs the tests already fails on - # it, so the attempts are published for visibility only. + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step # succeeds silently and there is nothing to notice. @@ -73,6 +81,13 @@ steps: publishTestResults: false env: platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' + - task: PowerShell@2 + displayName: 'Reconcile retried unit test failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/coverage"' + condition: succeeded() - task: PublishTestResults@2 displayName: 'Publish unit test with coverage results' inputs: @@ -84,9 +99,10 @@ steps: # attempt under Retries///, all of which this glob matches. Attempt 1 therefore # keeps a Failed record for a test that later passed, and mergeTestResults does not # resolve the two, so failing the task on published results would cancel out every - # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, - # non-zero when they are exhausted -- and the step that runs the tests already fails on - # it, so the attempts are published for visibility only. + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step # succeeds silently and there is nothing to notice. diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml index 738445143a..84293e54fd 100644 --- a/build/jobs/e2e-tests.yml +++ b/build/jobs/e2e-tests.yml @@ -132,6 +132,14 @@ steps: 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults' + - task: PowerShell@2 + displayName: 'Reconcile retried E2E failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/testresults"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish test results' inputs: @@ -143,9 +151,10 @@ steps: # attempt under Retries///, all of which this glob matches. Attempt 1 therefore # keeps a Failed record for a test that later passed, and mergeTestResults does not # resolve the two, so failing the task on published results would cancel out every - # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, - # non-zero when they are exhausted -- and the step that runs the tests already fails on - # it, so the attempts are published for visibility only. + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step # succeeds silently and there is nothing to notice. diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index f0a9bab941..8205ddc2f6 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -143,6 +143,14 @@ jobs: 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)' 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) + - task: PowerShell@2 + displayName: 'Reconcile retried integration failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/coverage"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish integration test results' inputs: @@ -154,9 +162,10 @@ jobs: # attempt under Retries///, all of which this glob matches. Attempt 1 therefore # keeps a Failed record for a test that later passed, and mergeTestResults does not # resolve the two, so failing the task on published results would cancel out every - # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, - # non-zero when they are exhausted -- and the step that runs the tests already fails on - # it, so the attempts are published for visibility only. + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step # succeeds silently and there is nothing to notice. diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index 111aec7ed7..708a824bbf 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -165,6 +165,14 @@ jobs: 'app_globalWriterUserApp_secret': $(app_globalWriterUserApp_secret) platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults' + - task: PowerShell@2 + displayName: 'Reconcile retried export failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/testresults"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish test results' inputs: @@ -176,9 +184,10 @@ jobs: # attempt under Retries///, all of which this glob matches. Attempt 1 therefore # keeps a Failed record for a test that later passed, and mergeTestResults does not # resolve the two, so failing the task on published results would cancel out every - # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, - # non-zero when they are exhausted -- and the step that runs the tests already fails on - # it, so the attempts are published for visibility only. + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step # succeeds silently and there is nothing to notice. @@ -341,6 +350,14 @@ jobs: 'app_globalWriterUserApp_secret': $(app_globalWriterUserApp_secret) platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults' + - task: PowerShell@2 + displayName: 'Reconcile retried export failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/testresults"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish test results' inputs: @@ -352,9 +369,10 @@ jobs: # attempt under Retries///, all of which this glob matches. Attempt 1 therefore # keeps a Failed record for a test that later passed, and mergeTestResults does not # resolve the two, so failing the task on published results would cancel out every - # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, - # non-zero when they are exhausted -- and the step that runs the tests already fails on - # it, so the attempts are published for visibility only. + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step # succeeds silently and there is nothing to notice. diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 32ebaa6294..726cc52284 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -133,6 +133,14 @@ jobs: 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)' 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) + - task: PowerShell@2 + displayName: 'Reconcile retried integration failures' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1' + arguments: '-ResultsDirectory "$(Agent.TempDirectory)/coverage"' + condition: succeeded() + - task: PublishTestResults@2 displayName: 'Publish integration test results' inputs: @@ -144,9 +152,10 @@ jobs: # attempt under Retries///, all of which this glob matches. Attempt 1 therefore # keeps a Failed record for a test that later passed, and mergeTestResults does not # resolve the two, so failing the task on published results would cancel out every - # retry. The runner's exit code is the retry-aware verdict -- 0 once an attempt passes, - # non-zero when they are exhausted -- and the step that runs the tests already fails on - # it, so the attempts are published for visibility only. + # retry. The step that runs the tests fails on the runner's exit code, and the step that + # follows it reconciles the attempts against each other, because that exit code counts a + # test that failed and then skipped as recovered. The attempts are published here for + # visibility only. failTaskOnFailedTests: false # A missing TRX means the run reported nothing; without this the publish step # succeeds silently and there is nothing to notice. diff --git a/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 b/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 new file mode 100644 index 0000000000..5eb102648f --- /dev/null +++ b/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 @@ -0,0 +1,125 @@ +<# +.SYNOPSIS + Fails a test leg where a failing test was retried and never actually passed. + +.DESCRIPTION + The test legs let the runner's exit code be the verdict, because the results of every retry + attempt are published and failing on those would cancel out every retry. That reasoning holds + only while "the runner exited zero" means "every failure was cleared by a later attempt", and it + does not: a test that fails and then skips is not counted as a failure by the retry extension, so + a run where a real failure turned into a skip exits zero and the leg goes green. + + This reads what the attempts actually reported. Every test that failed in any attempt has to be + recorded as passed in the final attempt. A test that failed and then skipped, or that failed and + then was not run at all, fails the leg here instead of disappearing. + + Results are keyed by the assembly they came from as well as the test name, because this + repository compiles the same shared test files into several assemblies and the names alone + collide. + +.PARAMETER ResultsDirectory + The directory the runner was given for its results. Retry attempts are below it under Retries. +#> +param( + [Parameter(Mandatory = $true)] + [string] $ResultsDirectory +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $ResultsDirectory)) { + throw "The results directory '$ResultsDirectory' does not exist, so no test results can be read from it." +} + +$allReports = @(Get-ChildItem -LiteralPath $ResultsDirectory -Recurse -Filter *.trx -File) +$attemptReports = @($allReports | Where-Object { $_.FullName -match '[\\/]Retries[\\/]' }) + +if ($attemptReports.Count -eq 0) { + Write-Host "No retry attempts were recorded under '$ResultsDirectory', so there is nothing to reconcile." + exit 0 +} + +$finalReports = @($allReports | Where-Object { $_.FullName -notmatch '[\\/]Retries[\\/]' }) + +if ($finalReports.Count -eq 0) { + throw "Retry attempts were recorded under '$ResultsDirectory' but no final report was found beside them, so what the retried tests ended up doing cannot be read." +} + +function Read-Outcomes { + param([System.IO.FileInfo] $Report) + + [xml] $document = Get-Content -LiteralPath $Report.FullName -Raw + + $storageById = @{} + foreach ($definition in $document.TestRun.TestDefinitions.UnitTest) { + if ($null -ne $definition) { + $storageById[$definition.id] = ($definition.storage ?? '').ToLowerInvariant() + } + } + + foreach ($result in $document.TestRun.Results.UnitTestResult) { + if ($null -eq $result) { + continue + } + + $storage = '' + if ($storageById.ContainsKey($result.testId)) { + $storage = $storageById[$result.testId] + } + + [PSCustomObject]@{ + Key = "$storage|$($result.testName)" + Name = $result.testName + Outcome = $result.outcome + } + } +} + +$failedWhenRetried = @{} +foreach ($report in $attemptReports) { + foreach ($outcome in Read-Outcomes -Report $report) { + if ($outcome.Outcome -eq 'Failed') { + $failedWhenRetried[$outcome.Key] = $outcome.Name + } + } +} + +if ($failedWhenRetried.Count -eq 0) { + Write-Host "Retry attempts were recorded but none of them reported a failure, so there is nothing to reconcile." + exit 0 +} + +$finalOutcomes = @{} +foreach ($report in $finalReports) { + foreach ($outcome in Read-Outcomes -Report $report) { + # A test can be recorded more than once in a final report. Passed is what clears an earlier + # failure, so it wins over any other outcome recorded for the same test. + if ($outcome.Outcome -eq 'Passed' -or -not $finalOutcomes.ContainsKey($outcome.Key)) { + $finalOutcomes[$outcome.Key] = $outcome.Outcome + } + } +} + +$unresolved = @() +foreach ($key in $failedWhenRetried.Keys) { + $final = if ($finalOutcomes.ContainsKey($key)) { $finalOutcomes[$key] } else { 'not run at all' } + + if ($final -ne 'Passed') { + $unresolved += " $($failedWhenRetried[$key]) -> $final" + } +} + +if ($unresolved.Count -gt 0) { + $detail = $unresolved -join [Environment]::NewLine + throw @" +These tests failed and were retried, but the final attempt does not record them as passing: +$detail + +The runner treats anything other than a failure as retried successfully, so a test that fails and +then skips, or that fails and then does not run, exits the runner zero and would otherwise leave +this leg green with a real failure inside it. +"@ +} + +Write-Host "Every test that failed in a retry attempt passed in the final attempt ($($failedWhenRetried.Count) reconciled)." diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj index f8aee0e228..c93863a50d 100644 --- a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Microsoft.Health.Extensions.Xunit.TestAssets.csproj @@ -24,6 +24,12 @@ --> + + diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FailThenSkipOnRetry/FailThenSkipOnRetryTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FailThenSkipOnRetry/FailThenSkipOnRetryTests.cs new file mode 100644 index 0000000000..8898cecb27 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/FailThenSkipOnRetry/FailThenSkipOnRetryTests.cs @@ -0,0 +1,73 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.IO; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FailThenSkipOnRetry +{ + /// + /// Fails on its first attempt and skips on the next one, which is what a test guarded by a + /// run-time condition does when that condition changes between attempts. + /// + /// + /// The platform re-runs failed tests in a new process, so the attempt is counted through a file + /// named by the harness rather than in memory. + /// + public class FailThenSkipOnRetryTests + { + /// The environment variable naming the file that counts attempts. + public const string AttemptFileVariable = "XUNIT_EXT_ASSET_ATTEMPT_FILE"; + + /// + /// Fails once, then skips. + /// + [Fact] + public void FailsOnTheFirstAttemptAndSkipsOnTheNext() + { + string attemptFile = Environment.GetEnvironmentVariable(AttemptFileVariable); + + if (string.IsNullOrEmpty(attemptFile)) + { + throw new InvalidOperationException( + $"ASSET: {AttemptFileVariable} was not set, so this scenario cannot tell which attempt it is on."); + } + + if (File.Exists(attemptFile)) + { + Assert.Skip("ASSET: skipping on a later attempt, the way a run-time condition would."); + } + + File.WriteAllText(attemptFile, "attempted"); + Assert.Fail("ASSET: failing on the first attempt."); + } + + /// + /// Fails once, then passes, so that the retry attempt has a test that actually runs. + /// + [Fact] + public void FailsOnTheFirstAttemptAndPassesOnTheNext() + { + string attemptFile = Environment.GetEnvironmentVariable(AttemptFileVariable); + + if (string.IsNullOrEmpty(attemptFile)) + { + throw new InvalidOperationException( + $"ASSET: {AttemptFileVariable} was not set, so this scenario cannot tell which attempt it is on."); + } + + string companion = attemptFile + ".passing"; + + if (File.Exists(companion)) + { + return; + } + + File.WriteAllText(companion, "attempted"); + Assert.Fail("ASSET: failing on the first attempt."); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetriedFailureOutcomeTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetriedFailureOutcomeTests.cs new file mode 100644 index 0000000000..182823b7af --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetriedFailureOutcomeTests.cs @@ -0,0 +1,77 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Records what the runner reports when a retried test stops failing without passing. + /// + /// + /// Every CI leg runs with retries and treats the runner's exit code as the verdict, because the + /// results of the earlier attempts are published too and failing on those would cancel out every + /// retry. That is only sound while a zero exit code means every failure was cleared, and this is + /// the case where it does not: the retry counts a test that skipped as no longer failing, so the + /// run exits zero with a real failure inside it. + /// + /// The legs close that gap outside the runner, in + /// build/jobs/scripts/Assert-RetriedFailuresPassed.ps1, which requires every test that + /// failed in an attempt to be recorded as passing in the final one. This test is what would + /// notice if the runner ever started reporting this itself, at which point that script has + /// nothing left to do. + /// + public class RetriedFailureOutcomeTests + { + private const string ClassName = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FailThenSkipOnRetry.FailThenSkipOnRetryTests."; + private const string AttemptFileVariable = "XUNIT_EXT_ASSET_ATTEMPT_FILE"; + + /// + /// Runs two tests that both fail first: one then passes, one then skips. + /// + [Fact] + public void GivenARetriedTestThatSkipsInsteadOfPassing_WhenTheRunFinishes_ThenTheRunnerReportsSuccessAnyway() + { + string attemptFile = Path.Combine(Path.GetTempPath(), "xunit-ext-assets", $"attempt-{Guid.NewGuid():N}.txt"); + Directory.CreateDirectory(Path.GetDirectoryName(attemptFile)); + + try + { + TestAssetRun run = TestAssetRunner.Run( + "FailThenSkipOnRetry", + retryFailedTests: "2", + environment: new Dictionary { [AttemptFileVariable] = attemptFile }); + + // Both tests failed on the first attempt. The one that went on to skip is reported as + // NotExecuted, which is indistinguishable from a test that was always going to skip, + // and the run still exits zero: that is the whole of the exposure. + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ClassName + "FailsOnTheFirstAttemptAndSkipsOnTheNext"] = "NotExecuted", + [ClassName + "FailsOnTheFirstAttemptAndPassesOnTheNext"] = "Passed", + }); + } + finally + { + foreach (string path in new[] { attemptFile, attemptFile + ".passing" }) + { + try + { + File.Delete(path); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + // A leaked temp file is not worth failing an otherwise good test over. + } + } + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs index 745515b2ee..5d99aa805b 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/TestAssetRunner.cs @@ -69,8 +69,18 @@ public static class TestAssetRunner /// would have run the tests it stands for. The runner scopes the query to the scenario itself, /// because a query filter cannot be combined with the plain namespace filter used otherwise. /// + /// + /// How many attempts the runner may make at the tests that failed, in the form it takes on the + /// command line. Every CI leg runs with this on, and it changes what a run reports rather than + /// only how long it takes, so a scenario that describes a retried run has to set it. + /// + /// + /// Extra environment variables for the child process. A scenario whose behaviour differs + /// between attempts needs somewhere to record that it has already run, and the attempts are + /// separate processes. + /// /// The exit code, output and published results of the run. - public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null, string maxThreads = null, string filterNotTrait = null, string filterQueryTraits = null) + public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool preEnumerateTheories = true, string filterTrait = null, string maxThreads = null, string filterNotTrait = null, string filterQueryTraits = null, string retryFailedTests = null, IReadOnlyDictionary environment = null) { // The runner rejects a query filter outright once any plain filter has been added, so the // two forms cannot be mixed. Saying that here keeps a scenario that tries from failing as an @@ -149,7 +159,13 @@ public static TestAssetRun Run(string scenario, bool stopOnFail = false, bool pr arguments.Add(maxThreads); } - (int exitCode, string output, TimeSpan duration) = Execute(arguments); + if (!string.IsNullOrEmpty(retryFailedTests)) + { + arguments.Add("--retry-failed-tests"); + arguments.Add(retryFailedTests); + } + + (int exitCode, string output, TimeSpan duration) = Execute(arguments, environment); string trxPath = Path.Combine(resultsDirectory, trxFileName); if (!File.Exists(trxPath)) @@ -201,7 +217,7 @@ private static string ResolveAssetsAssembly() return path; } - private static (int ExitCode, string Output, TimeSpan Duration) Execute(IReadOnlyList arguments) + private static (int ExitCode, string Output, TimeSpan Duration) Execute(IReadOnlyList arguments, IReadOnlyDictionary environment) { var startInfo = new ProcessStartInfo("dotnet") { @@ -216,6 +232,14 @@ private static (int ExitCode, string Output, TimeSpan Duration) Execute(IReadOnl startInfo.ArgumentList.Add(argument); } + if (environment != null) + { + foreach (KeyValuePair variable in environment) + { + startInfo.Environment[variable.Key] = variable.Value; + } + } + // The pipeline sets this so its own test runs land in a known place. The child would // inherit it and could write its TRX somewhere other than --results-directory, which // would look exactly like a run that produced no report at all. From 6782a2e41ea00653bd66b18ef035d9f8d9d1a777 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 03:04:17 -0700 Subject: [PATCH 098/105] Refuse two fixture argument set dimensions of the same enum type Dimensions are declared by position but bound to a fixture's constructor by enum type, and xUnit's fixture cache is keyed by type as well, so two dimensions of one type cannot be told apart. Before this change the second was dropped: the method expanded to a single variant named for the whole combination, so a declaration reading as four pairings ran once, with the first value used for both arguments, and reported as though it had covered all of them. That is a suite claiming coverage it does not have. On origin/main the merge built its dimensions with Dictionary.Add, so this threw. The rewrite made it a silent overwrite; this restores the refusal at discovery, where it is reported per method as a failing stand-in under each combination the method would have claimed, leaving its siblings alone. Verified by removing the check: the scenario then expands to one variant named "(Sql, Cosmos)" and the three new harness tests go red. Both attributes in the repository - HttpIntegrationFixtureArgumentSets and FhirStorageTestsFixtureArgumentSets - use distinct enum types and are unaffected. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...DuplicateDimensionArgumentSetsAttribute.cs | 31 +++++++ .../DuplicateDimensionTests.cs | 61 +++++++++++++ .../DuplicateDimensionTests.cs | 86 +++++++++++++++++++ .../CustomXunitTestFrameworkDiscoverer.cs | 25 +++++- 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionArgumentSetsAttribute.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/DuplicateDimensionTests.cs diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionArgumentSetsAttribute.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionArgumentSetsAttribute.cs new file mode 100644 index 0000000000..5509e6285e --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionArgumentSetsAttribute.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DuplicateDimension +{ + /// + /// Declares two fixture argument set dimensions of the same enum type. + /// + /// + /// This reads as though it asks for every pairing of two data stores - a primary and a + /// secondary, say - which is exactly the kind of thing a test author would reach for. Fixture + /// arguments are matched to a fixture's constructor by their enum type, so the two cannot be + /// told apart, and what would actually run is the first value twice. + /// + public sealed class DuplicateDimensionArgumentSetsAttribute : FixtureArgumentSetsAttribute + { + /// + /// Initializes a new instance of the class. + /// + /// The first dimension. + /// The second dimension, of the same type as the first. + public DuplicateDimensionArgumentSetsAttribute(AssetDataStore first, AssetDataStore second) + : base(first, second) + { + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionTests.cs b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionTests.cs new file mode 100644 index 0000000000..14a157b4d0 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.TestAssets/Scenarios/DuplicateDimension/DuplicateDimensionTests.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.FixtureVariants; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DuplicateDimension +{ + /// + /// A class one of whose methods declares two fixture argument set dimensions of the same enum + /// type. + /// + /// + /// Dimensions are declared by position but bound to the fixture's constructor by type, so two of + /// the same type cannot be told apart: the second is dropped and the first value is used for + /// both. The combinations that were meant to differ would all be the same run, each reported + /// under a name claiming otherwise - a suite reporting coverage it does not have. The discoverer + /// refuses the declaration instead, so the mistake is a red test rather than a false green. + /// + [AssetArgumentSets(AssetDataStore.Sql | AssetDataStore.Cosmos)] + public class DuplicateDimensionTests : IClassFixture + { + private readonly AssetFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The variant fixture injected by the custom executor. + public DuplicateDimensionTests(AssetFixture fixture) + { + _fixture = fixture; + } + + /// + /// Declares in both of its dimensions, so discovery of this + /// method fails and a failing test case stands in for it. It never runs, which is why it can + /// assert nothing. + /// + [Fact] + [DuplicateDimensionArgumentSets(AssetDataStore.Sql, AssetDataStore.Cosmos)] + public void NeverRuns() + { + throw new InvalidOperationException("This method must never be executed: its discovery is expected to fail."); + } + + /// + /// Declares nothing of its own, so it expands normally. A fault on one method must not cost + /// its siblings their variants. + /// + [Fact] + public void SiblingStillRuns() + { + string displayName = TestContext.Current.Test.TestDisplayName; + + Assert.Contains($"({_fixture.DataStore})", displayName, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/DuplicateDimensionTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DuplicateDimensionTests.cs new file mode 100644 index 0000000000..378d9358dd --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/DuplicateDimensionTests.cs @@ -0,0 +1,86 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers an argument set attribute declaring two dimensions of the same enum type. + /// + /// + /// Dimensions are declared by position but bound to the fixture's constructor by type, so two of + /// the same type cannot be told apart: the second is dropped and every variant is built with the + /// first value, while still being named - and traited - for the combination it was supposed to + /// be. Every combination would report as having run, none of them would have, and the run would + /// be green. On origin/main the merge used Dictionary.Add and this threw; the + /// rewrite made it a silent overwrite, so the refusal below is what keeps it loud. + /// + public class DuplicateDimensionTests + { + private const string ScenarioClass = "Microsoft.Health.Extensions.Xunit.TestAssets.Scenarios.DuplicateDimension.DuplicateDimensionTests"; + + /// + /// The refusal is reported as a failure standing in for the offending method, under each of + /// the combinations it would have claimed, while the method that declared nothing unusual + /// runs normally. + /// + [Fact] + public void GivenAMethodDeclaringTwoDimensionsOfOneType_WhenItIsDiscovered_ThenItIsReportedAndItsSiblingStillRuns() + { + TestAssetRun run = TestAssetRunner.Run("DuplicateDimension"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Cosmos)"] = "Failed", + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql, Cosmos)"] = "Failed", + [ScenarioClass + ".SiblingStillRuns (Sql)"] = "Passed", + [ScenarioClass + ".SiblingStillRuns (Cosmos)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + + /// + /// The failure names the attribute and the type it repeated, because the declaration that + /// caused it is legal C# and reads as though it asks for every pairing of the two. + /// + [Fact] + public void GivenAMethodDeclaringTwoDimensionsOfOneType_WhenItIsDiscovered_ThenTheFailureNamesTheRepeatedType() + { + TestAssetRun run = TestAssetRunner.Run("DuplicateDimension"); + + Assert.Contains("DuplicateDimensionArgumentSetsAttribute", run.Output, StringComparison.Ordinal); + Assert.Contains("AssetDataStore", run.Output, StringComparison.Ordinal); + } + + /// + /// The leg's own filter shape. A failure the leg cannot see is no better than the silent + /// expansion the refusal replaced, so a leg excluding the other data store has to be left + /// with a failure of its own. The stand-in for the combination naming both stores carries + /// both traits and is excluded with them, which is why the per-value stand-ins matter. + /// + [Fact] + public void GivenAMethodDeclaringTwoDimensionsOfOneType_WhenALegExcludesTheOtherDataStore_ThenTheFailureIsStillReported() + { + TestAssetRun run = TestAssetRunner.Run("DuplicateDimension", filterNotTrait: "AssetDataStore=Cosmos"); + + TestAssetRunAssertions.PublishedExactly( + run, + new Dictionary + { + [ScenarioClass + ".NeverRuns (fixture argument set discovery: Sql)"] = "Failed", + [ScenarioClass + ".SiblingStillRuns (Sql)"] = "Passed", + }); + + Assert.NotEqual(0, run.ExitCode); + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs index afde5c640b..a1254e2f3c 100644 --- a/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs +++ b/src/Microsoft.Health.Extensions.Xunit/CustomXunitTestFrameworkDiscoverer.cs @@ -1273,7 +1273,30 @@ private static string BuildVariantClassKey(string variantKey, Type testClass) private static SingleFlag[][] ExpandEnumFlagsFromAttributeData(FixtureArgumentSetsAttribute attribute) { - return attribute.GetArgumentSets() + IReadOnlyList argumentSets = attribute.GetArgumentSets(); + + // Dimensions are declared by position but bound to the fixture's constructor by enum + // type, and xUnit's own fixture cache is keyed by type as well, so two dimensions of the + // same type cannot be told apart. The second one is dropped when the fixture is built, + // yet the variant is still named and traited for both: every combination would be + // reported as having run with values the fixture never received, and the combinations + // that were supposed to differ would all be the same run. That is a test suite reporting + // coverage it does not have, so the declaration is refused here. + Type duplicated = argumentSets + .Where(argument => argument != null) + .GroupBy(argument => argument.GetType()) + .FirstOrDefault(group => group.Count() > 1) + ?.Key; + + if (duplicated != null) + { + throw new InvalidOperationException( + $"'{attribute.GetType().Name}' declares more than one fixture argument set dimension of type '{duplicated}'. " + + "Fixture arguments are matched to a fixture constructor by their enum type, so dimensions sharing a type cannot be told apart: " + + "each variant would be built with the first value alone while being reported under every combination. Give each dimension its own enum type."); + } + + return argumentSets .Select(e => GetSingleValuedFlags(e).ToArray()) .ToArray(); } From c37482915d65fc68821008b152342ac3205ea6f6 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 03:11:29 -0700 Subject: [PATCH 099/105] Check leg reachability a method at a time, not a class at a time One method carrying fixture argument sets exempts its whole class from the check that every test class is selected by some leg, because that is enough for the framework to expand the class. It expands only that method: a sibling declaring nothing of its own, in a class declaring nothing of its own, is discovered as an ordinary test case with no DataStore trait, and every leg selects positively on that trait. So one decorated method can hide any number of siblings that run nowhere, from the very check written to find tests that run nowhere. No class is in that state today. Verified by putting one there: dropping the class-level attribute from BulkUpdateTests leaves its eleven undecorated methods unreachable, the new check names all eleven, and the class-level check stays green - which is the blind spot itself. Only the E2E copy runs this. The integration legs filter by excluding a store, and an exclusion still selects a test carrying no trait at all, so the same scan there would report tests that do run. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AssemblyChecker.cs | 52 ++++++++++++++++++- .../AssemblyValidationsTester.cs | 51 ++++++++++++++++++ .../AssemblyValidationsTests.cs | 20 +++++++ 3 files changed, 122 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs index e142e34837..5ddb607e8f 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyChecker.cs @@ -186,6 +186,46 @@ public static IReadOnlyList ScanForTestClassesNoStoreFilteredLegSelects(As .ToList(); } + /// + /// Looks for test methods that no data-store-filtered leg can select, in classes the class + /// level check exempts. + /// + /// + /// A class is exempt from when any + /// one of its methods carries argument sets, because that is enough for the framework to + /// expand that class. It expands only that method, though: a method carrying nothing of its + /// own, in a class declaring nothing of its own, is discovered as an ordinary test case with + /// no DataStore trait, so every leg filters it out. One decorated method is therefore + /// enough to hide all of its siblings from the class level check while they run nowhere and + /// every leg reports success. + /// + /// Assembly under analysis. + /// The trait the legs select on. + /// + /// The attribute the framework expands into per-store variants. + /// + /// + /// The unreachable methods, named as Namespace.Class.Method. Classes no leg selects at + /// all are left to the class level check rather than reported a method at a time. + /// + public static IReadOnlyList ScanForTestMethodsNoStoreFilteredLegSelects(Assembly assembly, string traitName, Type argumentSetAttributeType) + { + EnsureArg.IsNotNull(assembly, nameof(assembly)); + EnsureArg.IsNotNullOrWhiteSpace(traitName, nameof(traitName)); + EnsureArg.IsNotNull(argumentSetAttributeType, nameof(argumentSetAttributeType)); + + return assembly.GetTypes() + .Where(t => t.IsClass && IsTestClass(t)) + .Where(t => !CarriesArgumentSets(t.CustomAttributes, argumentSetAttributeType) && !DeclaresTrait(t.CustomAttributes, traitName)) + .Where(t => t.GetMethods().Any(m => CarriesArgumentSets(m.CustomAttributes, argumentSetAttributeType))) + .SelectMany(t => t.GetMethods() + .Where(IsTestMethod) + .Where(m => !CarriesArgumentSets(m.CustomAttributes, argumentSetAttributeType) && !DeclaresTrait(m.CustomAttributes, traitName)) + .Select(m => $"{t.FullName}.{m.Name}")) + .Distinct(StringComparer.Ordinal) + .ToList(); + } + private static bool IsExpandedIntoStoreVariants(Type type, Type argumentSetAttributeType) { // The attribute is declared on the class or on individual methods, and either is enough @@ -199,9 +239,19 @@ private static bool CarriesArgumentSets(IEnumerable attribu return attributes.Any(a => argumentSetAttributeType.IsAssignableFrom(a.AttributeType)); } + private static bool IsTestMethod(MethodInfo method) + { + return method.GetCustomAttributes().Any(a => a is FactAttribute || a is TheoryAttribute); + } + private static bool DeclaresTrait(Type type, string traitName) { - return type.CustomAttributes + return DeclaresTrait(type.CustomAttributes, traitName); + } + + private static bool DeclaresTrait(IEnumerable attributes, string traitName) + { + return attributes .Where(a => a.AttributeType == typeof(TraitAttribute) && a.ConstructorArguments.Count > 0) .Any(a => string.Equals(a.ConstructorArguments[0].Value as string, traitName, StringComparison.Ordinal)); } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs index 2599f4fb64..d5fb8f562a 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/AssemblyValidationsTester.cs @@ -133,6 +133,57 @@ public static void EnsureEveryTestClassIsSelectedBySomeLeg( } } + /// + /// Fails when a test method is invisible to every leg that selects on a data store, in a + /// class the class level check exempts. + /// + /// Assembly under analysis. + /// The trait the legs select on. + /// The attribute the framework expands into per-store variants. + /// + /// Methods already known to run in no leg, named as Namespace.Class.Method. + /// + /// + /// One decorated method exempts its whole class from the class level check, so without this + /// its undecorated siblings are hidden twice over: they carry no trait, so no leg selects + /// them, and the check that would have said so passes. + /// + public static void EnsureEveryTestMethodIsSelectedBySomeLeg( + Assembly assembly, + string traitName, + Type argumentSetAttributeType, + IReadOnlyCollection knownUnreachableMethods) + { + IReadOnlyList unreachable = AssemblyChecker.ScanForTestMethodsNoStoreFilteredLegSelects(assembly, traitName, argumentSetAttributeType); + + string[] unexpected = unreachable + .Where(name => !knownUnreachableMethods.Contains(name)) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + string[] staleEntries = knownUnreachableMethods + .Where(name => !unreachable.Contains(name, StringComparer.Ordinal)) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + if (unexpected.Length > 0) + { + Assert.Fail( + $"Assembly '{assembly}' has test methods that no leg selects. Their class carries neither a '{traitName}' trait nor fixture argument sets, and " + + $"neither do they - but a sibling method does, which is enough to exempt the class from the class-level check. Nothing skips or errors for these: " + + $"every leg filters them out, runs without them, and passes. Give the class argument sets, or give each method its own, or add it to the known list " + + $"with a reason.{Environment.NewLine}" + + string.Join(Environment.NewLine, unexpected)); + } + + if (staleEntries.Length > 0) + { + Assert.Fail( + $"Assembly '{assembly}' names methods as unreachable that a leg now selects. Remove them from the known list so it keeps describing what is " + + $"actually unrun.{Environment.NewLine}{string.Join(Environment.NewLine, staleEntries)}"); + } + } + private static void AssertTestClasses(Assembly currentAssembly, string traitName, IEnumerable types) { if (types == null || !types.Any()) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs index 311e244abf..acfaadbdce 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E.Common/AssemblyValidationsTests.cs @@ -57,6 +57,12 @@ public sealed class AssemblyValidationsTests "Microsoft.Health.Fhir.Tests.E2E.Rest.Search.SearchParameterInitializationTests", }; + /// + /// Methods in a class that only a sibling's argument sets make visible to the class-level + /// check. Empty, and meant to stay so: an entry here is a test nobody runs. + /// + private static readonly HashSet KnownUnreachableTestMethods = new HashSet(StringComparer.Ordinal); + [Fact] public void GivenCurrentAssembly_WhenScanned_EnsureAllTestsHaveCategoryTrait() { @@ -90,5 +96,19 @@ public void GivenCurrentAssembly_WhenScanned_EnsureEveryTestClassIsSelectedBySom typeof(FixtureArgumentSetsAttribute), KnownUnreachableTestClasses); } + + /// + /// A single decorated method exempts its whole class from the check above, so its + /// undecorated siblings would be selected by no leg and reported by nothing. + /// + [Fact] + public void GivenCurrentAssembly_WhenScanned_EnsureEveryTestMethodIsSelectedBySomeLeg() + { + AssemblyValidationsTester.EnsureEveryTestMethodIsSelectedBySomeLeg( + _currentAssembly, + nameof(DataStore), + typeof(FixtureArgumentSetsAttribute), + KnownUnreachableTestMethods); + } } } From d38b2b30506df74a0e6ffd4a85bb1d5ae547e204 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 03:15:36 -0700 Subject: [PATCH 100/105] Notice when a unit test project stops matching the leg's glob The unit test leg names no projects: it runs whatever '**/*UnitTests/*.csproj' matches. A project that leaves the glob - renamed, moved, or its directory suffix changed - is not reported anywhere. The other nineteen run, all of them pass, and the leg is green with an assembly's tests missing. The floor the other legs use does not fit this shape. They count what a filter discovers; here the loss is a whole project that was never run at all, which no count of the tests that did run can see. --minimum-expected-tests cannot either: it is applied per assembly, so it says nothing about an absent one, and it has to stay at 1 because the runner re-applies it to the smaller pass --retry-failed-tests starts. Twenty projects match today. Adding more is fine; losing one fails the leg before the tests run. Verified both ways against the working tree. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/build.yml | 12 +++++ .../Assert-UnitTestProjectsDiscovered.ps1 | 53 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 diff --git a/build/jobs/build.yml b/build/jobs/build.yml index e28db4bcb6..f77382f084 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -30,6 +30,12 @@ steps: workingDirectory: $(System.DefaultWorkingDirectory) - ${{ if eq(parameters.unitTest, 'true') }}: + - task: PowerShell@2 + displayName: 'Assert the unit test projects were found' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1' + arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -MinimumExpectedProjects 20' - task: DotNetCoreCLI@2 displayName: 'dotnet test' inputs: @@ -70,6 +76,12 @@ steps: condition: succeededOrFailed() - ${{ if eq(parameters.codeCoverage, 'true') }}: + - task: PowerShell@2 + displayName: 'Assert the unit test projects were found' + inputs: + pwsh: true + filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1' + arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -MinimumExpectedProjects 20' - task: DotNetCoreCLI@2 displayName: 'dotnet test with coverage' inputs: diff --git a/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 b/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 new file mode 100644 index 0000000000..00aa644978 --- /dev/null +++ b/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 @@ -0,0 +1,53 @@ +<# +.SYNOPSIS + Fails when the unit test leg's project glob matches fewer projects than expected. + +.DESCRIPTION + The unit test leg does not name its projects: it runs whatever '**/*UnitTests/*.csproj' + matches. Every other leg guards against running a fraction of its tests by counting the tests + a filter discovers, but that shape does not fit here, because this leg's failure is a whole + project leaving the glob - renamed, moved, or a directory suffix quietly changed - rather than + a filter selecting too little. Nothing notices: the remaining projects run, every one of them + passes, and the leg reports success with an entire assembly's tests missing. + + --minimum-expected-tests cannot cover this either. It is applied per assembly, so it says + nothing about an assembly that was never run, and it has to stay at 1 because the runner + re-applies it to the much smaller pass that --retry-failed-tests starts. + + Adding projects is fine; only losing them fails. + +.PARAMETER SourcesDirectory + The repository root to search. + +.PARAMETER MinimumExpectedProjects + The floor the match count must clear. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $SourcesDirectory, + + [Parameter(Mandatory = $true)] + [int] $MinimumExpectedProjects +) + +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $SourcesDirectory)) { + throw "Sources directory '$SourcesDirectory' does not exist." +} + +# Mirrors the 'projects' pattern of the leg's test task: any .csproj in a directory whose name +# ends in UnitTests. +$projects = Get-ChildItem -LiteralPath $SourcesDirectory -Recurse -Filter '*.csproj' -File | + Where-Object { $_.Directory.Name -like '*UnitTests' } | + Sort-Object -Property FullName + +Write-Host "Matched $($projects.Count) unit test project(s) (floor $MinimumExpectedProjects):" +foreach ($project in $projects) { + Write-Host " $($project.Directory.Name)" +} + +if ($projects.Count -lt $MinimumExpectedProjects) { + throw "The unit test project glob matched $($projects.Count) project(s), below this leg's floor of $MinimumExpectedProjects. The leg would have run a fraction of its unit tests and reported success." +} From 6b9a4d9c464f7cfe7fdfe8165e99cf208d3f05a9 Mon Sep 17 00:00:00 2001 From: mikaelweave Date: Sun, 23 Aug 2026 15:09:37 -0700 Subject: [PATCH 101/105] Refuse a retry outcome no one decided how to report The three helpers that turn a no-result run into something the runner can publish each fell through to doing nothing for an outcome they did not recognise, while the summary beside them counted that same outcome as a failure. The verdict comes from what is published, not from the summary, so another member of the enum would have been counted as failed and reported as nothing at all. Each helper now refuses an outcome it was not written for. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RetryTestCaseNoResultTests.cs | 38 +++++++++++++++++++ .../RetryTestCase.cs | 30 ++++++++++++--- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs index 034f28ec3f..b751dfba40 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryTestCaseNoResultTests.cs @@ -176,5 +176,43 @@ public void GivenNothingIsReplayed_WhenSelectingTheException_ThenNothingIsChosen Assert.Null(selected); } + + /// + /// An outcome none of the three helpers was taught to handle must stop the run rather than + /// be absorbed by a default branch. + /// + /// + /// The helpers used to disagree about an unknown value: the two that publish treated it as + /// "report nothing" while the one that counts treated anything but ReportNothing as a + /// failure. Since the runner's verdict comes from the messages and not the summary, an + /// outcome added later and missed in one switch would have counted a failure that was never + /// published - a lost failure, and a green run. The value below is deliberately not a + /// declared member, which is what a future addition looks like to code compiled against + /// today's enum. + /// + [Fact] + public void GivenAnOutcomeNoHelperHandles_WhenItIsAccountedFor_ThenEachHelperRefusesIt() + { + const NoResultOutcome unknown = (NoResultOutcome)int.MaxValue; + + Assert.Throws(() => RetryTestCase.CreateNoResultSummary(unknown)); + Assert.Throws(() => RetryTestCase.SelectNoResultException(unknown, null, null)); + Assert.Throws(() => RetryTestCase.ReportSingleResult(unknown, null, null)); + } + + /// + /// Every declared outcome is handled by all three helpers, so the refusal above cannot be + /// satisfied by a helper that refuses everything. + /// + [Fact] + public void GivenEveryDeclaredOutcome_WhenItIsAccountedFor_ThenNoHelperRefusesIt() + { + foreach (NoResultOutcome outcome in Enum.GetValues()) + { + RetryTestCase.CreateNoResultSummary(outcome); + RetryTestCase.SelectNoResultException(outcome, null, null); + RetryTestCase.ReportSingleResult(outcome, null, null); + } + } } } diff --git a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs index 3c8ff23ea1..95df5d1f41 100644 --- a/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs +++ b/src/Microsoft.Health.Extensions.Xunit/RetryTestCase.cs @@ -476,13 +476,21 @@ internal static bool ReportSingleResult( currentAttempt?.DiscardDeferredMessages(); return earlierAttempt?.ReplayDeferredMessages() ?? true; - default: + case NoResultOutcome.ReportNothing: // Nothing observed a failure, so there is no competition to resolve. Whatever the // current attempt is still holding - an abstention, typically - is the only // result the test has, and is left for its disposal to publish. earlierAttempt?.DiscardDeferredMessages(); currentAttempt?.PublishDeferredMessagesOnDispose(); return true; + + default: + // An outcome added later and not handled here would publish nothing while + // CreateNoResultSummary counted a failure, and the runner's verdict comes from + // the messages rather than the summary, so the failure would be lost and the run + // would pass. Refusing an outcome nobody taught this how to report is the only + // answer that cannot end in a green run hiding one. + throw new ArgumentOutOfRangeException(nameof(outcome), outcome, "No handling is defined for this outcome, so a failure it stands for would not reach the results."); } } @@ -513,8 +521,11 @@ internal static Exception SelectNoResultException(NoResultOutcome outcome, Excep case NoResultOutcome.ReplayEarlierAttempt: return earlierAttemptException; - default: + case NoResultOutcome.ReportNothing: return null; + + default: + throw new ArgumentOutOfRangeException(nameof(outcome), outcome, "No handling is defined for this outcome, so the exception reported with it would not describe the failure being replayed."); } } @@ -530,9 +541,18 @@ internal static Exception SelectNoResultException(NoResultOutcome outcome, Excep /// internal static RunSummary CreateNoResultSummary(NoResultOutcome outcome) { - return outcome == NoResultOutcome.ReportNothing - ? new RunSummary { Total = 0, Failed = 0 } - : new RunSummary { Total = 1, Failed = 1 }; + switch (outcome) + { + case NoResultOutcome.ReportNothing: + return new RunSummary { Total = 0, Failed = 0 }; + + case NoResultOutcome.ReplayCurrentAttempt: + case NoResultOutcome.ReplayEarlierAttempt: + return new RunSummary { Total = 1, Failed = 1 }; + + default: + throw new ArgumentOutOfRangeException(nameof(outcome), outcome, "No handling is defined for this outcome, so the totals it contributes would not match what was published."); + } } protected override void Serialize(IXunitSerializationInfo info) From 843bc75be488bba6b439023a9f07c93380ed0378 Mon Sep 17 00:00:00 2001 From: mikaelweave Date: Sun, 23 Aug 2026 15:09:54 -0700 Subject: [PATCH 102/105] Tell retried tests apart by identity, not by display name The check that every failed test passed on a later attempt keyed results by their display name, and display names are not identities: a theory whose rows differ only in the case of an argument produces two tests a case-insensitive lookup cannot separate, and this repository already has one. A row that passed then answered for a row that failed and skipped, which is the exact failure this check was written to catch, and the leg went green. Results are now keyed by the identifier the report gives each test, which the retry attempts reuse, together with the assembly it came from. A result with no identifier fails the leg rather than being dropped. The script is covered by tests for the first time, running the real script over reports shaped like the runner's own. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../scripts/Assert-RetriedFailuresPassed.ps1 | 58 +++- ...t.Health.Extensions.Xunit.UnitTests.csproj | 24 ++ .../RetryReconciliationScriptTests.cs | 325 ++++++++++++++++++ .../ScriptRun.cs | 14 + .../ScriptRunner.cs | 185 ++++++++++ 5 files changed, 592 insertions(+), 14 deletions(-) create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryReconciliationScriptTests.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRun.cs create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRunner.cs diff --git a/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 b/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 index 5eb102648f..3e864e2c74 100644 --- a/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 +++ b/build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 @@ -13,13 +13,18 @@ recorded as passed in the final attempt. A test that failed and then skipped, or that failed and then was not run at all, fails the leg here instead of disappearing. - Results are keyed by the assembly they came from as well as the test name, because this - repository compiles the same shared test files into several assemblies and the names alone - collide. + Results are keyed by the identifier the report gives each test, together with the assembly it + came from. Display names are not identities: a theory whose rows differ only in the case of an + argument produces two tests with names PowerShell's hashtables cannot tell apart, so keying by + name lets one test's pass clear a different test's failure. The assembly stays in the key + because this repository compiles the same shared test files into several assemblies, which can + give the same test the same identifier in each of them. .PARAMETER ResultsDirectory The directory the runner was given for its results. Retry attempts are below it under Retries. #> +#requires -Version 7.0 + param( [Parameter(Mandatory = $true)] [string] $ResultsDirectory @@ -51,32 +56,57 @@ function Read-Outcomes { [xml] $document = Get-Content -LiteralPath $Report.FullName -Raw - $storageById = @{} - foreach ($definition in $document.TestRun.TestDefinitions.UnitTest) { + # An empty or is what a report that recorded nothing looks like, + # and the XML adapter surfaces an empty element as a string, so reaching through it for child + # elements fails. That would fail the leg with a PowerShell error in place of the verdict below, + # which is the one message that explains a report holding no results at all. Selecting from the + # document instead is unaffected by how any one element happens to be surfaced, and matching on + # local names keeps it working whether or not the TRX carries its namespace. + $definitions = @($document.SelectNodes('//*[local-name()="UnitTest"]')) + + $storageById = [System.Collections.Hashtable]::new([System.StringComparer]::Ordinal) + foreach ($definition in $definitions) { if ($null -ne $definition) { - $storageById[$definition.id] = ($definition.storage ?? '').ToLowerInvariant() + # Attributes are read with GetAttribute rather than as properties: strict mode turns a + # missing property into an error about PowerShell, which would replace the explanations + # below with noise for the one kind of report that most needs explaining. + $storageById[$definition.GetAttribute('id').ToLowerInvariant()] = $definition.GetAttribute('storage').ToLowerInvariant() } } - foreach ($result in $document.TestRun.Results.UnitTestResult) { + $results = @($document.SelectNodes('//*[local-name()="UnitTestResult"]')) + + foreach ($result in $results) { if ($null -eq $result) { continue } + $testId = $result.GetAttribute('testId') + $testName = $result.GetAttribute('testName') + + # Without an identifier there is nothing to reconcile this result against, and guessing from + # the name is what this key exists to avoid. Refusing here fails the leg, which is the safe + # direction: the alternative is dropping a result that may be the failure being looked for. + if ([string]::IsNullOrWhiteSpace($testId)) { + throw "'$($Report.FullName)' records a result for '$testName' with no test id, so it cannot be matched to the same test in another attempt." + } + + $testId = $testId.ToLowerInvariant() + $storage = '' - if ($storageById.ContainsKey($result.testId)) { - $storage = $storageById[$result.testId] + if ($storageById.ContainsKey($testId)) { + $storage = $storageById[$testId] } [PSCustomObject]@{ - Key = "$storage|$($result.testName)" - Name = $result.testName - Outcome = $result.outcome + Key = "$storage|$testId" + Name = $testName + Outcome = $result.GetAttribute('outcome') } } } -$failedWhenRetried = @{} +$failedWhenRetried = [System.Collections.Hashtable]::new([System.StringComparer]::Ordinal) foreach ($report in $attemptReports) { foreach ($outcome in Read-Outcomes -Report $report) { if ($outcome.Outcome -eq 'Failed') { @@ -90,7 +120,7 @@ if ($failedWhenRetried.Count -eq 0) { exit 0 } -$finalOutcomes = @{} +$finalOutcomes = [System.Collections.Hashtable]::new([System.StringComparer]::Ordinal) foreach ($report in $finalReports) { foreach ($outcome in Read-Outcomes -Report $report) { # A test can be recorded more than once in a final report. Passed is what clears an earlier diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj b/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj index 7f5cc418ea..282a88096c 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj @@ -8,6 +8,18 @@ once per scenario, with the runner flags that scenario needs. --> $(MSBuildThisFileDirectory)..\Microsoft.Health.Extensions.Xunit.TestAssets\bin\$(Configuration)\$(TargetFramework)\Microsoft.Health.Extensions.Xunit.TestAssets.dll + + $(MSBuildThisFileDirectory)..\..\build\jobs\scripts\Assert-RetriedFailuresPassed.ps1 + + $(MSBuildThisFileDirectory)..\..\build\jobs\scripts\Assert-UnitTestProjectsDiscovered.ps1 + $(MSBuildThisFileDirectory)..\..\build\jobs\scripts\UnitTestProjects.txt @@ -35,6 +47,18 @@ <_Parameter1>TestAssetsAssembly <_Parameter2>$(TestAssetsAssembly) + + <_Parameter1>RetryReconciliationScript + <_Parameter2>$(RetryReconciliationScript) + + + <_Parameter1>UnitTestProjectsScript + <_Parameter2>$(UnitTestProjectsScript) + + + <_Parameter1>UnitTestProjectManifest + <_Parameter2>$(UnitTestProjectManifest) + diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryReconciliationScriptTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryReconciliationScriptTests.cs new file mode 100644 index 0000000000..fd94e4cd99 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/RetryReconciliationScriptTests.cs @@ -0,0 +1,325 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Xml.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers the script the CI legs use to decide whether a retried failure really cleared. + /// + /// + /// Every test leg runs with retries and lets the runner's exit code be the verdict, and the + /// runner exits zero when a failing test stops failing without passing. The script in + /// build/jobs/scripts/Assert-RetriedFailuresPassed.ps1 is the only thing that turns that + /// back into a red leg, so a mistake in it puts a real failure back on the green path it was + /// written to close. These tests run the real script over reports shaped like the ones the + /// runner writes. + /// + public class RetryReconciliationScriptTests + { + private const string TrxNamespace = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + private const string DefaultStorage = @"c:\bin\assembly.dll"; + + private static readonly TimeSpan RunTimeout = TimeSpan.FromMinutes(2); + + /// + /// Two tests whose names differ only in case, where the failing one never passed. + /// + /// + /// Display names are not identities. A theory whose rows differ only in the case of an + /// argument produces exactly this shape, and the repository already has one: + /// VersioningConfigurationTests passes both "Versioned" and the constant whose + /// value is "versioned" to the same theory. Keyed by name in a comparison that + /// ignores case, the row that passed answers for the row that did not, and the failure the + /// script exists to catch leaves through the door it was meant to close. + /// + [Fact] + public void GivenTwoTestsNamedAlikeApartFromCase_WhenOnlyOneOfThemPassed_ThenTheOtherIsStillReportedAsUnresolved() + { + var results = new ResultsDirectory(); + + results.WriteAttempt( + Result("11111111-1111-1111-1111-111111111111", "T.Row(input: \"Versioned\")", "Failed")); + + results.WriteFinal( + Result("11111111-1111-1111-1111-111111111111", "T.Row(input: \"Versioned\")", "NotExecuted"), + Result("22222222-2222-2222-2222-222222222222", "T.Row(input: \"versioned\")", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("T.Row(input: \"Versioned\") -> NotExecuted", run.Output, StringComparison.Ordinal); + } + + /// + /// Two tests sharing one display name exactly, both of which passed in the end. + /// + /// + /// This is the other half of telling tests apart by identity: distinguishing them must not + /// come at the cost of reporting a test that did pass, or the legs learn to ignore this + /// script. + /// + [Fact] + public void GivenTwoTestsSharingADisplayName_WhenBothOfThemPassed_ThenNothingIsReported() + { + var results = new ResultsDirectory(); + + results.WriteAttempt( + Result("11111111-1111-1111-1111-111111111111", "T.Row", "Failed"), + Result("22222222-2222-2222-2222-222222222222", "T.Row", "Failed")); + + results.WriteFinal( + Result("11111111-1111-1111-1111-111111111111", "T.Row", "Passed"), + Result("22222222-2222-2222-2222-222222222222", "T.Row", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("(2 reconciled)", run.Output, StringComparison.Ordinal); + } + + /// + /// A test that failed and then passed, which is what a retry is for. + /// + [Fact] + public void GivenAFailureThatPassedOnTheNextAttempt_WhenReconciling_ThenTheLegIsLeftGreen() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "Failed")); + results.WriteFinal(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("(1 reconciled)", run.Output, StringComparison.Ordinal); + } + + /// + /// A test that failed and then skipped, which the runner reports as a success. + /// + [Fact] + public void GivenAFailureThatSkippedOnTheNextAttempt_WhenReconciling_ThenTheLegIsFailed() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "Failed")); + results.WriteFinal(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "NotExecuted")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("T.Flaky -> NotExecuted", run.Output, StringComparison.Ordinal); + } + + /// + /// A test that failed and then was not recorded in the final attempt at all. + /// + [Fact] + public void GivenAFailureAbsentFromTheFinalAttempt_WhenReconciling_ThenTheLegIsFailed() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result("11111111-1111-1111-1111-111111111111", "T.Vanished", "Failed")); + results.WriteFinal(Result("22222222-2222-2222-2222-222222222222", "T.Other", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("T.Vanished -> not run at all", run.Output, StringComparison.Ordinal); + } + + /// + /// The same test identity reported by two different assemblies. + /// + /// + /// This repository compiles the same shared test files into several assemblies, so the same + /// fully qualified name is a real test in each of them and the reports can give them the + /// same identifier. A pass in one assembly must not answer for a failure in another. + /// + [Fact] + public void GivenOneTestIdentityInTwoAssemblies_WhenOnlyOneOfThemPassed_ThenTheOtherIsStillReportedAsUnresolved() + { + var results = new ResultsDirectory(); + + results.WriteAttempt( + Result("11111111-1111-1111-1111-111111111111", "Shared.T.Case", "Failed", storage: @"c:\bin\r4.dll")); + + // Each assembly reports its own results, so the two reports below are what the same test + // identity coming from two assemblies actually looks like on disk. + results.WriteFinal( + Result("11111111-1111-1111-1111-111111111111", "Shared.T.Case", "NotExecuted", storage: @"c:\bin\r4.dll")); + + results.WriteFinal( + Result("11111111-1111-1111-1111-111111111111", "Shared.T.Case", "Passed", storage: @"c:\bin\r5.dll")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("Shared.T.Case -> NotExecuted", run.Output, StringComparison.Ordinal); + } + + /// + /// A result the report gives no identifier, which nothing can be matched against. + /// + [Fact] + public void GivenAResultWithNoTestId_WhenReconciling_ThenTheLegIsFailedWithAnExplanation() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result(testId: null, "T.Anonymous", "Failed")); + results.WriteFinal(Result("11111111-1111-1111-1111-111111111111", "T.Anonymous", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("with no test id", run.Output, StringComparison.Ordinal); + } + + /// + /// Reports that recorded nothing at all, which is what an empty run writes. + /// + /// + /// The verdict for a leg that ran nothing belongs to the discovery floor, which says so in + /// as many words. This script only has to leave that message intact instead of replacing it + /// with an error about reading XML. + /// + [Fact] + public void GivenReportsThatRecordedNothing_WhenReconciling_ThenTheLegIsLeftGreenWithoutAnError() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(); + results.WriteFinal(); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("nothing to reconcile", run.Output, StringComparison.Ordinal); + } + + /// + /// A run with no retries below it, which is every run where nothing failed. + /// + [Fact] + public void GivenNoRetryAttempts_WhenReconciling_ThenTheLegIsLeftGreen() + { + var results = new ResultsDirectory(); + + results.WriteFinal(Result("11111111-1111-1111-1111-111111111111", "T.Steady", "Passed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("nothing to reconcile", run.Output, StringComparison.Ordinal); + } + + /// + /// Retry attempts with no final report beside them, so nothing says how they ended. + /// + [Fact] + public void GivenRetryAttemptsWithNoFinalReport_WhenReconciling_ThenTheLegIsFailed() + { + var results = new ResultsDirectory(); + + results.WriteAttempt(Result("11111111-1111-1111-1111-111111111111", "T.Flaky", "Failed")); + + ScriptRun run = results.Reconcile(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("no final report", run.Output, StringComparison.Ordinal); + } + + private static TestResult Result(string testId, string testName, string outcome, string storage = DefaultStorage) + => new TestResult(testId, testName, outcome, storage); + + private sealed record TestResult(string TestId, string TestName, string Outcome, string Storage); + + /// + /// A results directory shaped the way the retry extension leaves one behind. + /// + private sealed class ResultsDirectory + { + private readonly string _root = Path.Combine(Path.GetTempPath(), "xunit-ext-reconcile", Guid.NewGuid().ToString("N")); + private int _attempts; + private int _reports; + + public void WriteAttempt(params TestResult[] results) + { + _attempts++; + Write(Path.Combine(_root, "Retries", "run", _attempts.ToString(CultureInfo.InvariantCulture), "report.trx"), results); + } + + /// + /// Writes one final report. Each call writes a separate report, which is how results + /// from more than one assembly reach the same results directory. + /// + public void WriteFinal(params TestResult[] results) + { + _reports++; + Write(Path.Combine(_root, FormattableString.Invariant($"report{_reports}.trx")), results); + } + + public ScriptRun Reconcile() + { + try + { + return ScriptRunner.Run( + ScriptRunner.Resolve("RetryReconciliationScript"), + new Dictionary { ["ResultsDirectory"] = _root }); + } + finally + { + try + { + Directory.Delete(_root, recursive: true); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + // A leaked temp directory is not worth failing an otherwise good test over. + } + } + } + + private static void Write(string path, IReadOnlyList results) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)); + + XNamespace ns = TrxNamespace; + + var document = new XDocument( + new XElement( + ns + "TestRun", + new XAttribute("id", Guid.NewGuid().ToString()), + new XElement( + ns + "Results", + results.Select(r => new XElement( + ns + "UnitTestResult", + r.TestId == null ? null : new XAttribute("testId", r.TestId), + new XAttribute("testName", r.TestName), + new XAttribute("outcome", r.Outcome)))), + new XElement( + ns + "TestDefinitions", + results.Where(r => r.TestId != null).Select(r => new XElement( + ns + "UnitTest", + new XAttribute("id", r.TestId), + new XAttribute("name", r.TestName), + new XAttribute("storage", r.Storage)))))); + + document.Save(path); + } + } + } +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRun.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRun.cs new file mode 100644 index 0000000000..765a2c7ae7 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRun.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// What a script reported. + /// + /// The exit code the leg reads to decide whether to fail. + /// Everything the script wrote, on either stream. + internal sealed record ScriptRun(int ExitCode, string Output); +} diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRunner.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRunner.cs new file mode 100644 index 0000000000..09f419ed14 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ScriptRunner.cs @@ -0,0 +1,185 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Runs one of the repository's CI guard scripts the way the pipeline runs it. + /// + /// + /// Two of the guards that decide whether a test leg is allowed to report success are PowerShell + /// scripts rather than tests, so the only way to assert on what they do is to run them. They are + /// run from their place in the repository rather than from a copy, so what these tests cover is + /// what the pipeline invokes. + /// + internal static class ScriptRunner + { + private static readonly TimeSpan RunTimeout = TimeSpan.FromMinutes(2); + + /// + /// Runs a script and returns its exit code together with everything it wrote. + /// + /// The script to run. + /// The parameters to pass, by name without the leading dash. + public static ScriptRun Run(string scriptPath, IReadOnlyDictionary parameters) + { + var startInfo = new ProcessStartInfo("pwsh") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + // The pipeline runs these scripts with pwsh set on every task that calls them, and they + // say so themselves with a #requires line. Windows PowerShell cannot parse them, so + // running whatever "powershell" happens to mean here would test something no leg runs. + // + // They are invoked through -Command rather than -File so that a failure message can be + // read back as the script wrote it: PowerShell's default error view draws a box around a + // terminating error and rewraps the text inside it to the console width, which would + // leave assertions matching on how wide a machine's console happens to be. The exit code + // is unaffected by the view, and is what the leg actually reads. + var command = new StringBuilder("$ErrorView='NormalView'; & '").Append(scriptPath).Append('\''); + + foreach (KeyValuePair parameter in parameters) + { + command.Append(" -").Append(parameter.Key).Append(" '").Append(parameter.Value).Append('\''); + } + + command.Append("; exit $LASTEXITCODE"); + + startInfo.ArgumentList.Add("-NoProfile"); + startInfo.ArgumentList.Add("-NonInteractive"); + startInfo.ArgumentList.Add("-Command"); + startInfo.ArgumentList.Add(command.ToString()); + + using var process = new Process { StartInfo = startInfo }; + using var outputComplete = new ManualResetEventSlim(false); + using var errorComplete = new ManualResetEventSlim(false); + + var output = new StringBuilder(); + + // Both streams are drained as they arrive rather than one after the other. Reading one + // stream to the end first deadlocks as soon as the script writes more to the other than + // its pipe holds, which is not a hypothetical: these scripts report what went wrong by + // naming every test or project involved, on the error stream, while the same run is + // writing what it matched to the output stream. The wait below cannot rescue that, + // because it is only reached once the reads have finished. + void Append(DataReceivedEventArgs e, ManualResetEventSlim complete) + { + if (e.Data == null) + { + try + { + complete.Set(); + } + catch (ObjectDisposedException) + { + // The only way this event is disposed while a callback is still in flight is + // that this method has already given up waiting and is unwinding. Letting the + // exception escape would take down the test host from a thread pool thread. + } + } + else + { + lock (output) + { + output.AppendLine(e.Data); + } + } + } + + process.OutputDataReceived += (_, e) => Append(e, outputComplete); + process.ErrorDataReceived += (_, e) => Append(e, errorComplete); + + try + { + process.Start(); + } + catch (System.ComponentModel.Win32Exception e) + { + throw new InvalidOperationException( + "PowerShell 7 ('pwsh') was not found on this machine. The CI legs run this script with it, so these tests need it to run the same thing the legs do.", + e); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + bool exited = process.WaitForExit((int)RunTimeout.TotalMilliseconds); + + // Waiting for the process to exit is not the same as having been handed everything it + // wrote: the timed overload of WaitForExit returns as soon as the process is gone, while + // the handlers above are still being called on other threads. Reading the output without + // waiting for them yields whatever happened to have arrived, which on a loaded machine + // is nothing at all, and turns a passing script into an assertion about an empty string. + if (exited) + { + outputComplete.Wait(RunTimeout); + errorComplete.Wait(RunTimeout); + } + else + { + try + { + process.Kill(entireProcessTree: true); + } + catch (Exception e) when (e is InvalidOperationException or System.ComponentModel.Win32Exception) + { + // The child can exit on its own between the wait giving up and the kill landing. + } + + lock (output) + { + throw new InvalidOperationException($"'{scriptPath}' did not finish within {RunTimeout}. It reported:{Environment.NewLine}{output}"); + } + } + + lock (output) + { + return new ScriptRun(process.ExitCode, output.ToString()); + } + } + + /// + /// Resolves a path the project file recorded in assembly metadata. + /// + /// The metadata key holding the path. + public static string Resolve(string metadataKey) + { + string path = typeof(ScriptRunner).Assembly + .GetCustomAttributes() + .FirstOrDefault(a => a.Key == metadataKey) + ?.Value; + + if (string.IsNullOrEmpty(path)) + { + throw new InvalidOperationException( + $"The {metadataKey} metadata attribute is missing. It is injected by the project file and identifies a file these tests run or read."); + } + + // The project file writes this path with the separator MSBuild used on the machine that + // built it, and a backslash is an ordinary filename character everywhere except Windows. + path = Path.GetFullPath(path.Replace('\\', Path.DirectorySeparatorChar)); + + if (!File.Exists(path)) + { + throw new InvalidOperationException($"The file recorded as {metadataKey} was not found at '{path}'."); + } + + return path; + } + } +} From a62c0e62bec680c9041984aadbc6b5a7b5307af0 Mon Sep 17 00:00:00 2001 From: mikaelweave Date: Sun, 23 Aug 2026 15:10:11 -0700 Subject: [PATCH 103/105] Name the projects the unit test leg is expected to run The leg's guard counted the projects its glob matched and failed when the count dropped. A count only notices projects leaving while none arrive: renaming one project and adding another in the same change leaves the count where it was, and the renamed project stops being tested with nothing to show for it. The expected projects are now listed by path and each one is checked, so a project that leaves the glob is named whatever else changed alongside it. Projects that are not listed still run and are reported, since adding tests should not fail a leg. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/build.yml | 4 +- .../Assert-UnitTestProjectsDiscovered.ps1 | 73 +++++- build/jobs/scripts/UnitTestProjects.txt | 30 +++ .../UnitTestProjectManifestTests.cs | 214 ++++++++++++++++++ 4 files changed, 307 insertions(+), 14 deletions(-) create mode 100644 build/jobs/scripts/UnitTestProjects.txt create mode 100644 src/Microsoft.Health.Extensions.Xunit.UnitTests/UnitTestProjectManifestTests.cs diff --git a/build/jobs/build.yml b/build/jobs/build.yml index f77382f084..72249ce33f 100644 --- a/build/jobs/build.yml +++ b/build/jobs/build.yml @@ -35,7 +35,7 @@ steps: inputs: pwsh: true filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1' - arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -MinimumExpectedProjects 20' + arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -ManifestPath "$(Build.SourcesDirectory)/build/jobs/scripts/UnitTestProjects.txt"' - task: DotNetCoreCLI@2 displayName: 'dotnet test' inputs: @@ -81,7 +81,7 @@ steps: inputs: pwsh: true filePath: '$(Build.SourcesDirectory)/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1' - arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -MinimumExpectedProjects 20' + arguments: '-SourcesDirectory "$(Build.SourcesDirectory)" -ManifestPath "$(Build.SourcesDirectory)/build/jobs/scripts/UnitTestProjects.txt"' - task: DotNetCoreCLI@2 displayName: 'dotnet test with coverage' inputs: diff --git a/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 b/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 index 00aa644978..12338d25fa 100644 --- a/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 +++ b/build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 @@ -14,21 +14,27 @@ nothing about an assembly that was never run, and it has to stay at 1 because the runner re-applies it to the much smaller pass that --retry-failed-tests starts. - Adding projects is fine; only losing them fails. + Each expected project is checked by path rather than by counting them, because a count only + notices projects leaving while none arrive: renaming one project and adding another in the same + change leaves the count where it was, and the renamed project stops being tested silently. + + Adding projects is fine; only losing one that is listed fails. .PARAMETER SourcesDirectory The repository root to search. -.PARAMETER MinimumExpectedProjects - The floor the match count must clear. +.PARAMETER ManifestPath + The file listing the repository-relative path of every project this leg is expected to run. #> +#requires -Version 7.0 + [CmdletBinding()] param( [Parameter(Mandatory = $true)] [string] $SourcesDirectory, [Parameter(Mandatory = $true)] - [int] $MinimumExpectedProjects + [string] $ManifestPath ) $ErrorActionPreference = 'Stop' @@ -37,17 +43,60 @@ if (-not (Test-Path -LiteralPath $SourcesDirectory)) { throw "Sources directory '$SourcesDirectory' does not exist." } +if (-not (Test-Path -LiteralPath $ManifestPath)) { + throw "The expected project list '$ManifestPath' does not exist, so there is nothing to check the leg's projects against." +} + +$expected = @( + Get-Content -LiteralPath $ManifestPath | + ForEach-Object { $_.Trim() } | + Where-Object { $_ -and -not $_.StartsWith('#') }) + +if ($expected.Count -eq 0) { + throw "The expected project list '$ManifestPath' names no projects, so it would accept a leg that ran nothing." +} + +$root = (Resolve-Path -LiteralPath $SourcesDirectory).Path.TrimEnd([System.IO.Path]::DirectorySeparatorChar) + # Mirrors the 'projects' pattern of the leg's test task: any .csproj in a directory whose name # ends in UnitTests. -$projects = Get-ChildItem -LiteralPath $SourcesDirectory -Recurse -Filter '*.csproj' -File | - Where-Object { $_.Directory.Name -like '*UnitTests' } | - Sort-Object -Property FullName +$matched = @( + Get-ChildItem -LiteralPath $root -Recurse -Filter '*.csproj' -File | + Where-Object { $_.Directory.Name -like '*UnitTests' } | + ForEach-Object { $_.FullName.Substring($root.Length + 1).Replace([System.IO.Path]::DirectorySeparatorChar, '/') } | + Sort-Object) + +# Paths are compared without regard to case because the repository is developed on Windows and +# built on Linux, and a case-only difference is not a project leaving the leg. +$comparer = [System.StringComparer]::OrdinalIgnoreCase +$matchedSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$matched, $comparer) +$expectedSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$expected, $comparer) -Write-Host "Matched $($projects.Count) unit test project(s) (floor $MinimumExpectedProjects):" -foreach ($project in $projects) { - Write-Host " $($project.Directory.Name)" +Write-Host "Matched $($matched.Count) unit test project(s) against $($expected.Count) expected:" +foreach ($project in $matched) { + Write-Host " $project" } -if ($projects.Count -lt $MinimumExpectedProjects) { - throw "The unit test project glob matched $($projects.Count) project(s), below this leg's floor of $MinimumExpectedProjects. The leg would have run a fraction of its unit tests and reported success." +$missing = @($expected | Where-Object { -not $matchedSet.Contains($_) }) + +if ($missing.Count -gt 0) { + $detail = ($missing | ForEach-Object { " $_" }) -join [Environment]::NewLine + throw @" +These projects are expected to run in the unit test leg but no longer match its project glob: +$detail + +The leg runs whatever the glob matches, so it would have run without them and reported success. If +a project was renamed or moved on purpose, update '$ManifestPath' to say so. +"@ +} + +$unlisted = @($matched | Where-Object { -not $expectedSet.Contains($_) }) + +if ($unlisted.Count -gt 0) { + # An unlisted project still runs. Naming it here is what keeps the list from drifting so far + # behind that a later loss is hard to tell from a rename. + Write-Host "These projects run in this leg but are not listed in '$ManifestPath':" + foreach ($project in $unlisted) { + Write-Host " $project" + } } diff --git a/build/jobs/scripts/UnitTestProjects.txt b/build/jobs/scripts/UnitTestProjects.txt new file mode 100644 index 0000000000..b98dbac05a --- /dev/null +++ b/build/jobs/scripts/UnitTestProjects.txt @@ -0,0 +1,30 @@ +# The unit test leg runs whatever '**/*UnitTests/*.csproj' matches, so a project that leaves the +# glob - renamed, moved, or its directory suffix quietly changed - stops being tested without +# anything failing. This is the list of projects that leg is expected to run, checked against what +# the glob actually matches before the tests start. +# +# Paths are relative to the repository root and use forward slashes. Adding a project here is not +# required for it to run, but a project listed here that no longer matches fails the leg. +# +# Microsoft.Health.Fhir.Api.UnitTests holds a project named for R4; the directory and the project +# names differ, which is why it does not look like the rest. +src/Microsoft.Health.Extensions.Xunit.UnitTests/Microsoft.Health.Extensions.Xunit.UnitTests.csproj +src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj +src/Microsoft.Health.Fhir.Azure.UnitTests/Microsoft.Health.Fhir.Azure.UnitTests.csproj +src/Microsoft.Health.Fhir.Core.UnitTests/Microsoft.Health.Fhir.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj +src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj +src/Microsoft.Health.Fhir.R4.Core.UnitTests/Microsoft.Health.Fhir.R4.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.R4.Web.UnitTests/Microsoft.Health.Fhir.R4.Web.UnitTests.csproj +src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj +src/Microsoft.Health.Fhir.R4B.Core.UnitTests/Microsoft.Health.Fhir.R4B.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.R4B.Web.UnitTests/Microsoft.Health.Fhir.R4B.Web.UnitTests.csproj +src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj +src/Microsoft.Health.Fhir.R5.Core.UnitTests/Microsoft.Health.Fhir.R5.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.R5.Web.UnitTests/Microsoft.Health.Fhir.R5.Web.UnitTests.csproj +src/Microsoft.Health.Fhir.SchemaManager.UnitTests/Microsoft.Health.Fhir.SchemaManager.UnitTests.csproj +src/Microsoft.Health.Fhir.SqlServer.UnitTests/Microsoft.Health.Fhir.SqlServer.UnitTests.csproj +src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj +src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Microsoft.Health.Fhir.Stu3.Core.UnitTests.csproj +src/Microsoft.Health.Fhir.Stu3.Web.UnitTests/Microsoft.Health.Fhir.Stu3.Web.UnitTests.csproj +src/Microsoft.Health.TaskManagement.UnitTests/Microsoft.Health.TaskManagement.UnitTests.csproj diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/UnitTestProjectManifestTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/UnitTestProjectManifestTests.cs new file mode 100644 index 0000000000..f9da2100a2 --- /dev/null +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/UnitTestProjectManifestTests.cs @@ -0,0 +1,214 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit; + +namespace Microsoft.Health.Extensions.Xunit.UnitTests +{ + /// + /// Covers the guard that checks the unit test leg still runs the projects it is meant to. + /// + /// + /// The unit test leg names no projects: it runs whatever **/*UnitTests/*.csproj matches. + /// A project that leaves the glob is never run, every project that stayed passes, and the leg + /// reports success. The script in + /// build/jobs/scripts/Assert-UnitTestProjectsDiscovered.ps1 is what turns that into a red + /// leg, by checking the projects that matched against a list of the ones expected. + /// + public class UnitTestProjectManifestTests + { + /// + /// Every project the list names still exists where it says. + /// + /// + /// The leg's own copy of this check runs against the repository the pipeline cloned. This + /// one runs against the working tree, so a rename that was not carried into the list is + /// reported by the tests rather than only by CI. + /// + [Fact] + public void GivenTheExpectedProjectList_WhenReadAgainstTheRepository_ThenEveryProjectListedExists() + { + string manifest = ScriptRunner.Resolve("UnitTestProjectManifest"); + string root = Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(manifest)))); + + string[] missing = ReadManifest(manifest) + .Where(relative => !File.Exists(Path.Combine(root, relative.Replace('/', Path.DirectorySeparatorChar)))) + .ToArray(); + + Assert.Empty(missing); + } + + /// + /// The list is not empty, which is the one shape that would accept a leg running nothing. + /// + [Fact] + public void GivenTheExpectedProjectList_WhenRead_ThenItNamesProjects() + { + Assert.NotEmpty(ReadManifest(ScriptRunner.Resolve("UnitTestProjectManifest"))); + } + + /// + /// A tree holding every expected project. + /// + [Fact] + public void GivenEveryExpectedProject_WhenChecked_ThenTheLegIsLeftGreen() + { + using var tree = new ProjectTree(); + + ScriptRun run = tree.Check(); + + Assert.Equal(0, run.ExitCode); + } + + /// + /// One project renamed and another added, which leaves the number of projects unchanged. + /// + /// + /// This is the case a count cannot see, and it is not a hypothetical one: renaming a project + /// and adding a project are both ordinary changes, and they only have to land in the same + /// pull request. The renamed project stops being tested and the count still adds up. + /// + [Fact] + public void GivenAProjectRenamedWhileAnotherIsAdded_WhenChecked_ThenTheRenamedProjectIsReported() + { + using var tree = new ProjectTree(); + + string renamed = tree.Remove(0); + tree.Add("src/Some.Brand.New.UnitTests/Some.Brand.New.UnitTests.csproj"); + + ScriptRun run = tree.Check(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains(renamed, run.Output, StringComparison.Ordinal); + } + + /// + /// A project added without being listed, which still runs and must not fail the leg. + /// + [Fact] + public void GivenAnUnlistedProject_WhenChecked_ThenTheLegIsLeftGreenAndTheProjectIsNamed() + { + using var tree = new ProjectTree(); + + tree.Add("src/Some.Brand.New.UnitTests/Some.Brand.New.UnitTests.csproj"); + + ScriptRun run = tree.Check(); + + Assert.Equal(0, run.ExitCode); + Assert.Contains("Some.Brand.New.UnitTests", run.Output, StringComparison.Ordinal); + } + + /// + /// A project whose directory suffix changed, so the glob no longer matches it. + /// + [Fact] + public void GivenAProjectDirectoryNoLongerEndingInUnitTests_WhenChecked_ThenItIsReported() + { + using var tree = new ProjectTree(); + + string moved = tree.Remove(0); + tree.Add(moved.Replace("UnitTests/", "Tests/", StringComparison.Ordinal)); + + ScriptRun run = tree.Check(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains(moved, run.Output, StringComparison.Ordinal); + } + + /// + /// A list naming nothing, which would otherwise accept any tree at all. + /// + [Fact] + public void GivenAnEmptyExpectedProjectList_WhenChecked_ThenTheLegIsFailed() + { + using var tree = new ProjectTree(); + + tree.OverwriteManifest("# every line here is a comment"); + + ScriptRun run = tree.Check(); + + Assert.Equal(1, run.ExitCode); + Assert.Contains("names no projects", run.Output, StringComparison.Ordinal); + } + + private static IReadOnlyList ReadManifest(string path) + => File.ReadAllLines(path) + .Select(line => line.Trim()) + .Where(line => line.Length > 0 && !line.StartsWith('#')) + .ToArray(); + + /// + /// A directory holding a project file for each entry of the real expected project list. + /// + private sealed class ProjectTree : IDisposable + { + private readonly string _root = Path.Combine(Path.GetTempPath(), "xunit-ext-projects", Guid.NewGuid().ToString("N")); + private readonly List _projects; + private string _manifest; + + public ProjectTree() + { + _manifest = ScriptRunner.Resolve("UnitTestProjectManifest"); + _projects = ReadManifest(_manifest).ToList(); + + foreach (string project in _projects) + { + Add(project); + } + } + + public void Add(string relativePath) + { + string full = Path.Combine(_root, relativePath.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(full)); + File.WriteAllText(full, ""); + } + + /// + /// Deletes one of the expected projects and returns the path it was listed under. + /// + public string Remove(int index) + { + string relativePath = _projects[index]; + Directory.Delete(Path.GetDirectoryName(Path.Combine(_root, relativePath.Replace('/', Path.DirectorySeparatorChar))), recursive: true); + return relativePath; + } + + /// + /// Replaces the list the check reads with one written for a single test. + /// + public void OverwriteManifest(string contents) + { + _manifest = Path.Combine(_root, "expected-projects.txt"); + File.WriteAllText(_manifest, contents); + } + + public ScriptRun Check() + => ScriptRunner.Run( + ScriptRunner.Resolve("UnitTestProjectsScript"), + new Dictionary + { + ["SourcesDirectory"] = _root, + ["ManifestPath"] = _manifest, + }); + + public void Dispose() + { + try + { + Directory.Delete(_root, recursive: true); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + // A leaked temp directory is not worth failing an otherwise good test over. + } + } + } + } +} From 44d01d50e43c4e78b7467febe0969c436f29c24c Mon Sep 17 00:00:00 2001 From: mikaelweave Date: Sun, 23 Aug 2026 15:10:12 -0700 Subject: [PATCH 104/105] Write each integration leg's store filter down once The discovery floor that notices a leg selecting nothing repeated the store filter that the test run used, so an edit to one copy would leave the floor guarding a selection no longer run. Each leg now defines its filter once and both steps read it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/jobs/run-cosmos-tests.yml | 16 ++++++++++++++-- build/jobs/run-sql-tests.yml | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml index 8205ddc2f6..b4dfc71745 100644 --- a/build/jobs/run-cosmos-tests.yml +++ b/build/jobs/run-cosmos-tests.yml @@ -21,6 +21,13 @@ jobs: - ImageOverride -equals $(InternalLinuxImage) variables: AllowPtrToDetectTestRunRetryFiles: true + # The one place this leg's selection is written down. The discovery floor below is the only + # thing that notices a regression silently emptying this leg, and it can only say that about + # the selection it counts, so it has to count the same one the tests run under. Two copies of + # this text would let an edit to one of them leave the floor guarding a selection that is no + # longer used. The values carry no spaces, which is what lets the floor step split this back + # into arguments. + storeFilterArguments: '--filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer"' steps: - checkout: self fetchDepth: 1 @@ -107,7 +114,12 @@ jobs: throw "Test assembly not found for $($project.FullName) at $assemblyPath" } - $listOutput = & dotnet $assemblyPath --list-tests --filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer" + $storeFilter = @('$(storeFilterArguments)' -split '\s+' | Where-Object { $_ } | ForEach-Object { $_.Trim('"') }) + if ($storeFilter.Count -lt 2) { + throw "The store filter '$(storeFilterArguments)' did not split into arguments, so the floor below would count a selection this leg does not run." + } + + $listOutput = & dotnet $assemblyPath --list-tests @storeFilter if ($LASTEXITCODE -ne 0) { throw "Listing tests for $($project.BaseName) failed with exit code $LASTEXITCODE" } @@ -129,7 +141,7 @@ jobs: inputs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx --minimum-expected-tests 1' + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- $(storeFilterArguments) --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx --minimum-expected-tests 1' # Disable the task's built-in result publishing so the PublishTestResults@2 task below, # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml index 726cc52284..9be2f922fb 100644 --- a/build/jobs/run-sql-tests.yml +++ b/build/jobs/run-sql-tests.yml @@ -26,6 +26,13 @@ jobs: - ImageOverride -equals $(InternalLinuxImage) variables: AllowPtrToDetectTestRunRetryFiles: true + # The one place this leg's selection is written down. The discovery floor below is the only + # thing that notices a regression silently emptying this leg, and it can only say that about + # the selection it counts, so it has to count the same one the tests run under. Two copies of + # this text would let an edit to one of them leave the floor guarding a selection that is no + # longer used. The values carry no spaces, which is what lets the floor step split this back + # into arguments. + storeFilterArguments: '--filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb"' steps: - checkout: self fetchDepth: 1 @@ -99,7 +106,12 @@ jobs: throw "Test assembly not found for $($project.FullName) at $assemblyPath" } - $listOutput = & dotnet $assemblyPath --list-tests --filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb" + $storeFilter = @('$(storeFilterArguments)' -split '\s+' | Where-Object { $_ } | ForEach-Object { $_.Trim('"') }) + if ($storeFilter.Count -lt 2) { + throw "The store filter '$(storeFilterArguments)' did not split into arguments, so the floor below would count a selection this leg does not run." + } + + $listOutput = & dotnet $assemblyPath --list-tests @storeFilter if ($LASTEXITCODE -ne 0) { throw "Listing tests for $($project.BaseName) failed with exit code $LASTEXITCODE" } @@ -121,7 +133,7 @@ jobs: inputs: command: test projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx --minimum-expected-tests 1' + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- $(storeFilterArguments) --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx --minimum-expected-tests 1' # Disable the task's built-in result publishing so the PublishTestResults@2 task below, # which controls the glob and how failures are treated, is the sole publisher. publishTestResults: false From b7513423c5e40ab9dda4e338d66dea41a6c8cfda Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sun, 23 Aug 2026 15:44:45 -0700 Subject: [PATCH 105/105] Stop two harness assertions from passing on nothing An assertion over a collection says nothing when the collection is empty, and both of these could be emptied by the very regression they exist to catch: a fault reported as a test that ran, and options never read at all. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AssertionFormattingEnvironmentVariableTests.cs | 3 +++ .../ClassAttributeFaultTests.cs | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs index 9b2417e5c4..e789adc0dc 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/AssertionFormattingEnvironmentVariableTests.cs @@ -98,6 +98,9 @@ public void GivenNoAssertionFormattingOptions_WhenTheyAreCopiedToTheEnvironment_ IReadOnlyList> environment = CustomXunitTestFrameworkExecutor.BuildAssertionFormattingEnvironment(new RecordingExecutionOptions()); + // Asserting only that every value is null would hold just as well for a list that named no + // variables at all, which is the one result that would mean the options are never read. + Assert.NotEmpty(environment); Assert.All(environment, pair => Assert.Null(pair.Value)); } diff --git a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs index d5fc233afb..54dfd10ee7 100644 --- a/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs +++ b/src/Microsoft.Health.Extensions.Xunit.UnitTests/ClassAttributeFaultTests.cs @@ -112,6 +112,14 @@ public void GivenAClassLevelFault_WhenTestsAreSelectedByACompoundQuery_ThenTheMa /// failures are carrying traits pooled across methods - which is what makes a leg excluding /// one method's trait drop the failures standing in for all the others. /// + /// + /// A query selecting nothing would prove nothing if the runner had not understood the query + /// at all. What rules that out is + /// , + /// which selects a failure with this same query shape. Anchoring it again here would mean a + /// second whole-assembly run - query filtering is not scoped to one scenario's namespace - + /// and that added load makes the sibling query runs flaky. + /// [Fact] public void GivenAClassLevelFault_WhenAQuerySelectsACombinationNoMethodDeclared_ThenNothingMatches() { @@ -227,6 +235,10 @@ public void GivenTwoOverloadsThatBothFailDiscovery_WhenTheyAreDiscovered_ThenBot result.Name.Contains("OverloadedFaultTests.NeverRuns", StringComparison.Ordinal))); Assert.NotEqual(0, run.ExitCode); + + // The class holds nothing but the two lost overloads, so any passing result would mean a + // fault was reported as a test that ran - which counting only failures cannot see. + Assert.DoesNotContain(run.Results, result => string.Equals(result.Outcome, "Passed", StringComparison.Ordinal)); } } }