Skip to content

Add non-serialization strategy for data unfolding #4467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 36 additions & 19 deletions src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs
Original file line number Diff line number Diff line change
@@ -86,10 +86,11 @@ internal ICollection<UnitTestElement> EnumerateAssembly(
DataRowAttribute.TestIdGenerationStrategy = testIdGenerationStrategy;
DynamicDataAttribute.TestIdGenerationStrategy = testIdGenerationStrategy;

TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy = ReflectHelper.GetTestDataSourceOptions(assembly)?.UnfoldingStrategy switch
TestDataSourceOptionsAttribute? testDataSourceOptionsAttribute = ReflectHelper.GetTestDataSourceOptions(assembly);
TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy = testDataSourceOptionsAttribute?.UnfoldingStrategy switch
{
// When strategy is auto we want to unfold
TestDataSourceUnfoldingStrategy.Auto => TestDataSourceUnfoldingStrategy.Unfold,
TestDataSourceUnfoldingStrategy.Auto => TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer,
// When strategy is set, let's use it
{ } value => value,
// When the attribute is not set, let's look at the legacy attribute
@@ -103,7 +104,7 @@ internal ICollection<UnitTestElement> EnumerateAssembly(
// as the ID generator will ignore it.
(null, TestIdGenerationStrategy.Legacy) => TestDataSourceUnfoldingStrategy.Fold,
#pragma warning restore CS0618 // Type or member is obsolete
_ => TestDataSourceUnfoldingStrategy.Unfold,
_ => TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer,
},
};

@@ -115,8 +116,7 @@ internal ICollection<UnitTestElement> EnumerateAssembly(
continue;
}

List<UnitTestElement> testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals,
dataSourcesUnfoldingStrategy, testIdGenerationStrategy, fixturesTests);
List<UnitTestElement> testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals, dataSourcesUnfoldingStrategy, testIdGenerationStrategy, fixturesTests);
tests.AddRange(testsInType);
}

@@ -367,10 +367,12 @@ private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodIn
try
{
bool isDataDriven = false;
int dataSourceIndex = -1;
foreach (ITestDataSource dataSource in testDataSources)
{
dataSourceIndex++;
isDataDriven = true;
if (!TryUnfoldITestDataSource(dataSource, dataSourcesUnfoldingStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests))
if (!TryUnfoldITestDataSource(dataSource, dataSourceIndex, dataSourcesUnfoldingStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests))
{
// TODO: Improve multi-source design!
// Ideally we would want to consider each data source separately but when one source cannot be expanded,
@@ -400,7 +402,7 @@ private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodIn
}
}

private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List<UnitTestElement> tests)
private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, int dataSourceIndex, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List<UnitTestElement> tests)
{
var unfoldingCapability = dataSource as ITestDataSourceUnfoldingCapability;

@@ -455,6 +457,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat
// If strategy is DisplayName and we have a duplicate test name don't expand the test, bail out.
#pragma warning disable CS0618 // Type or member is obsolete
if (test.TestMethod.TestIdGenerationStrategy == TestIdGenerationStrategy.DisplayName
#pragma warning restore CS0618 // Type or member is obsolete
&& testDisplayNameFirstSeen.TryGetValue(discoveredTest.DisplayName!, out int firstIndexSeen))
{
string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_DuplicateDisplayName, firstIndexSeen, index, discoveredTest.DisplayName);
@@ -464,24 +467,38 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat
// Duplicated display name so bail out. Caller will handle adding the original test.
return false;
}
#pragma warning restore CS0618 // Type or member is obsolete

try
if (unfoldingCapability?.UnfoldingStrategy == TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex
|| (unfoldingCapability?.UnfoldingStrategy is null or TestDataSourceUnfoldingStrategy.Auto
&& dataSourcesUnfoldingStrategy == TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex))
{
discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d);
discoveredTest.TestMethod.TestDataSourceIgnoreMessage = testDataSourceIgnoreMessage;
discoveredTest.TestMethod.SerializedData = new string[3]
{
TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex.ToString(),
dataSourceIndex.ToString(CultureInfo.InvariantCulture),
index.ToString(CultureInfo.InvariantCulture),
};
discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource;
}
catch (SerializationException ex)
else
{
string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_CannotSerialize, index, discoveredTest.DisplayName);
warning += Environment.NewLine;
warning += ex.ToString();
warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning);
PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}");
try
{
discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d);
discoveredTest.TestMethod.TestDataSourceIgnoreMessage = testDataSourceIgnoreMessage;
discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource;
}
catch (SerializationException ex)
{
string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_CannotSerialize, index, discoveredTest.DisplayName);
warning += Environment.NewLine;
warning += ex.ToString();
warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning);
PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}");

// Serialization failed for the type, bail out. Caller will handle adding the original test.
return false;
// Serialization failed for the type, bail out. Caller will handle adding the original test.
return false;
}
}

discoveredTests.Add(discoveredTest);
55 changes: 41 additions & 14 deletions src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs
Original file line number Diff line number Diff line change
@@ -169,22 +169,11 @@ internal List<TestResult> RunTestMethod()

bool isDataDriven = false;
var parentStopwatch = Stopwatch.StartNew();
if (_test.DataType == DynamicDataType.ITestDataSource)
if (TryExecuteITestDataSource(results))
{
if (_test.TestDataSourceIgnoreMessage is not null)
{
return [new() { Outcome = UTF.UnitTestOutcome.Ignored, IgnoreReason = _test.TestDataSourceIgnoreMessage }];
}

object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData);
TestResult[] testResults = ExecuteTestWithDataSource(null, data);
results.AddRange(testResults);
}
else if (TryExecuteDataSourceBasedTests(results))
{
isDataDriven = true;
}
else if (TryExecuteFoldedDataDrivenTests(results))
else if (TryExecuteDataSourceBasedTests(results)
|| TryExecuteFoldedDataDrivenTests(results))
{
isDataDriven = true;
}
@@ -526,4 +515,42 @@ private static List<TestResult> UpdateResultsWithParentInfo(

return updatedResults;
}

private bool TryExecuteITestDataSource(List<TestResult> results)
{
if (_test.DataType != DynamicDataType.ITestDataSource)
{
return false;
}

UTF.ITestDataSource? dataSource;
object?[]? data;
if (_test.SerializedData?.Length == 3)
{
if (!Enum.TryParse(_test.SerializedData[0], out TestDataSourceUnfoldingStrategy _)
|| !int.TryParse(_test.SerializedData[1], out int dataSourceIndex)
|| !int.TryParse(_test.SerializedData[2], out int dataIndex))
{
throw ApplicationStateGuard.Unreachable();
}

dataSource = _testMethodInfo.GetAttributes<Attribute>(false)?.OfType<UTF.ITestDataSource>().Skip(dataSourceIndex).FirstOrDefault();
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Runtime team just told me not to rely on guaranteed reflection order. I'll try to see what other option we have. Given that this is currently always working fine, I'd vote for adding a comment and moving on with the PR.

if (dataSource is null)
{
throw ApplicationStateGuard.Unreachable();
}

data = dataSource.GetData(_testMethodInfo.MethodInfo).Skip(dataIndex).FirstOrDefault();
}
else
{
dataSource = null;
data = DataSerializationHelper.Deserialize(_test.SerializedData);
}

TestResult[] testResults = ExecuteTestWithDataSource(dataSource, data);
results.AddRange(testResults);

return true;
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.ComponentModel;

namespace Microsoft.VisualStudio.TestTools.UnitTesting;

/// <summary>
@@ -22,17 +24,33 @@ public enum TestDataSourceUnfoldingStrategy : byte
/// <summary>
/// MSTest will decide whether to unfold the parameterized test based on value from the assembly level attribute
/// <see cref="TestDataSourceOptionsAttribute" />. If no assembly level attribute is specified, then the default
/// configuration is to unfold.
/// configuration is to unfold using <see cref="System.Runtime.Serialization.Json.DataContractJsonSerializer"/>.
/// </summary>
Auto,

/// <summary>
/// Each data row is treated as a separate test case.
/// </summary>
/// <inheritdoc cref="UnfoldUsingDataContractJsonSerializer"/>
[EditorBrowsable(EditorBrowsableState.Never)]
[Obsolete("Use 'UnfoldUsingDataContractJsonSerializer' instead")]
Unfold,

/// <summary>
/// The parameterized test is not unfolded; all data rows are treated as a single test case.
/// </summary>
Fold,

/// <summary>
/// Each data row is treated as a separate test case, and the data is unfolded using
/// <see cref="System.Runtime.Serialization.Json.DataContractJsonSerializer"/>.
/// </summary>
UnfoldUsingDataContractJsonSerializer,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we make this the same numeric value as Unfold?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's probably not important, is it?


/// <summary>
/// Each data row is treated as a separate test case, and the data is unfolded using the data
/// source index and data index.
/// </summary>
/// <remarks>
/// Using this strategy will alter the test ID if the data source is reordered, as it depends
/// on the index of the data. This may affect the ability to track test cases over time.
/// </remarks>
UnfoldUsingDataIndex,
}
Original file line number Diff line number Diff line change
@@ -204,6 +204,8 @@ Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.Ign
Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.IgnoreMessage.set -> void
Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.IgnoreMessage.get -> string?
Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.IgnoreMessage.set -> void
Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer = 3 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex = 4 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.IgnoreMessage.get -> string?
Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.IgnoreMessage.set -> void
Microsoft.VisualStudio.TestTools.UnitTesting.UnitTestOutcome.Ignored = 10 -> Microsoft.VisualStudio.TestTools.UnitTesting.UnitTestOutcome
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.Immutable;

using Microsoft.MSTestV2.CLIAutomation;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;

namespace MSTest.IntegrationTests;

@@ -15,8 +18,8 @@ public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRow
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -34,8 +37,8 @@ public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_Simp
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -50,8 +53,8 @@ public void DataRowsExecuteWithRequiredAndOptionalParameters()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -67,8 +70,8 @@ public void DataRowsExecuteWithParamsArrayParameter()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -85,8 +88,8 @@ public void DataRowsFailWhenInvalidArgumentsProvided()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular&TestCategory~DataRowOptionalInvalidArguments");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular&TestCategory~DataRowOptionalInvalidArguments");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsFailed(
@@ -102,8 +105,8 @@ public void DataRowsShouldSerializeDoublesProperly()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular.DataRowTestDouble");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular.DataRowTestDouble");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -118,8 +121,8 @@ public void DataRowsShouldSerializeMixedTypesProperly()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowTestMixed");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowTestMixed");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -133,8 +136,8 @@ public void DataRowsShouldSerializeEnumsProperly()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowEnums");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowEnums");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -151,8 +154,8 @@ public void DataRowsShouldHandleNonSerializableValues()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowNonSerializable");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowNonSerializable");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsDiscovered(
@@ -172,8 +175,8 @@ public void ExecuteDataRowTests_Enums()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Enums");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Enums");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -247,8 +250,8 @@ public void ExecuteDataRowTests_NonSerializablePaths()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_NonSerializablePaths");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_NonSerializablePaths");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -265,8 +268,8 @@ public void ExecuteDataRowTests_Regular()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
@@ -326,8 +329,8 @@ public void GetDisplayName_AfterOverriding_GetsTheNewDisplayName()
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenGetDisplayName");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenGetDisplayName");
ImmutableArray<TestResult> testResults = RunTests(testCases);

VerifyE2E.TestsPassed(
testResults,
@@ -340,12 +343,30 @@ public void ParameterizedTestsWithTestMethodSettingDisplayName_DataIsPrefixWithD
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenTestMethodDisplayNameForParameterizedTest");
System.Collections.Immutable.ImmutableArray<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult> testResults = RunTests(testCases);
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenTestMethodDisplayNameForParameterizedTest");
ImmutableArray<TestResult> testResults = RunTests(testCases);

VerifyE2E.TestsPassed(
testResults,
"SomeCustomDisplayName2 (\"SomeData\")",
"SomeCustomDisplayName3 (\"SomeData\")");
}

public void RunTestsFailingWhenUsingSerialization()
{
// Arrange
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, "ClassName~DataRowTests_Index");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
testResults,
"NestedInputTypesInvalid (0,System.Object[])",
"NestedInputTypes (0,System.Object[])",
"NestedInputTypes (0,System.Object[])",
"NestedInputTypes (0,System.Object[])");
}
}
Original file line number Diff line number Diff line change
@@ -132,4 +132,51 @@ public void ExecuteNonExpandableDynamicDataTests()
"TestMethodSourceOnCurrentType (2,b)");
VerifyE2E.FailedTestCount(testResults, 0);
}

public void ExecuteTestsFailingWhenUsingSerialization()
{
// Arrange
string assemblyPath = GetAssetFullPath(TestAssetName);

// Act
ImmutableArray<TestCase> testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~IndexBasedDataTests");
ImmutableArray<TestResult> testResults = RunTests(testCases);

// Assert
VerifyE2E.TestsPassed(
testResults,
"Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])",
"Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])",
"Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])",
"Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])",
"TestReadonlyCollectionData (,DataRowTestProject.IndexBasedDataTests+MyData)",
"TestUnlimitedNatural (0,*,False)",
"TestUnlimitedNatural (0,0,True)",
"ValidateExMessage (DataRowTestProject.IndexBasedDataTests+InvalidUpdateException: Test exception message)",
"Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -0.150 / 2.030,Vector2D: 4.230 / 6.812,13.1935961)",
"Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -1.000 / -2.000,Vector2D: -3.400 / 2.750,-2.1)",
"Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -22.723 / -78.298,Vector2D: -17.433 / -8.196,1037.82079593)",
"Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000,0)",
"Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 0.000 / 0.000,Vector2D: 2.000 / 3.000,0)",
"Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 1.000 / 2.000,Vector2D: 0.000 / 0.000,0)",
"Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 1.000 / 3.000,Vector2D: 1.000 / 2.000,7)",
"Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 3.355 / -2.211,Vector2D: 12.430 / -2.754,47.791744)",
"Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: -17.433 / -8.196,-0.45,Vector2D: 7.845 / 3.688)",
"Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: -3.400 / 2.750,22.415,Vector2D: -76.211 / 61.641)",
"Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 0.000 / 0.000,-3.5,Vector2D: 0.000 / 0.000)",
"Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 0.000 / 0.000,0,Vector2D: 0.000 / 0.000)",
"Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 1.000 / 2.000,-0.73,Vector2D: -0.730 / -1.460)",
"Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 12.430 / -2.754,1023.56,Vector2D: 12722.851 / -2818.884)",
"Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 2.000 / 3.000,2.1,Vector2D: 4.200 / 6.300)",
"Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 4.230 / 6.812,-13.25,Vector2D: -56.048 / -90.257)",
"Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-0.45,Vector2D: -17.433 / -8.196,Vector2D: 7.845 / 3.688)",
"Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-0.73,Vector2D: 1.000 / 2.000,Vector2D: -0.730 / -1.460)",
"Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-13.25,Vector2D: 4.230 / 6.812,Vector2D: -56.048 / -90.257)",
"Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-3.5,Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000)",
"Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (0,Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000)",
"Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (1023.56,Vector2D: 12.430 / -2.754,Vector2D: 12722.851 / -2818.884)",
"Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (2.1,Vector2D: 2.000 / 3.000,Vector2D: 4.200 / 6.300)",
"Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (22.415,Vector2D: -3.400 / 2.750,Vector2D: -76.211 / 61.641)");
VerifyE2E.FailedTestCount(testResults, 0);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace DataRowTestProject;

[TestClass]
public class DataRowTests_Index
{
#region // https://github.com/microsoft/testfx/issues/2390

[TestMethod]
[DataRow((byte)0, new object[] { (byte)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
[DataRow((short)0, new object[] { (short)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
[DataRow(0L, new object[] { 0L }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
public void NestedInputTypes(object org, object nested)
=> Assert.IsTrue(
org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal),
string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name));

[TestMethod]
[DataRow(0, new object[] { (byte)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
public void NestedInputTypesInvalid(object org, object nested)
=> Assert.IsFalse(
org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal),
string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name));

#endregion
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,252 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.ObjectModel;
using System.Runtime.Serialization;

using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace DataRowTestProject;

[TestClass]
public class IndexBasedDataTests
{
#region https://github.com/microsoft/testfx/issues/908

[TestMethod]
[DynamicData(nameof(AddTestCases), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
public void Add_ShouldAddTheExpectedValues(Collection<string> systemUnderTest, IEnumerable<string> itemsToAdd, Collection<string> expected)
{
// The actual tested method is irrelevant. Executing this empty tests provokes the error
}

public static IEnumerable<object[]> AddTestCases
{
get
{
var sut = new Collection<string>();
var expected = new Collection<string>();
yield return new object[] { sut, Enumerable.Empty<string>(), expected };

sut = new Collection<string>() { "1" };
expected = new Collection<string>() { "1" };
yield return new object[] { sut, Enumerable.Empty<string>(), expected };

sut = new Collection<string>();
expected = new Collection<string>() { "1" };
yield return new object[] { sut, new[] { "1" }, expected };

sut = new Collection<string>() { "1", "a", "b" };
expected = new Collection<string>() { "1", "a", "b", "z", "j" };
yield return new object[] { sut, new[] { "z", "j" }, expected };
}
}

#endregion

#region https://github.com/microsoft/testfx/issues/1022

[TestMethod]
[DynamicData(nameof(UnlimitedNaturalData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
public void TestUnlimitedNatural(UnlimitedNatural testObject, UnlimitedNatural other, bool expected)
{
bool actual = testObject.Equals(other);
Assert.AreEqual(expected, actual);
}

public static IEnumerable<object[]> UnlimitedNaturalData
{
get
{
yield return new object[] { UnlimitedNatural.Zero, UnlimitedNatural.Infinite, false };
yield return new object[] { UnlimitedNatural.Zero, UnlimitedNatural.Zero, true };
}
}

#pragma warning disable CS0659 // Type overrides Object.Equals(object o) but does not override Object.GetHashCode()
#pragma warning disable CA2231 // Overload operator equals on overriding value type Equals
public readonly struct UnlimitedNatural : IEquatable<UnlimitedNatural>
#pragma warning restore CA2231 // Overload operator equals on overriding value type Equals
#pragma warning restore CS0659 // Type overrides Object.Equals(object o) but does not override Object.GetHashCode()
{
public static readonly UnlimitedNatural Infinite;
public static readonly UnlimitedNatural One = new(1);
public static readonly UnlimitedNatural Zero = new(0);

public UnlimitedNatural(uint? value)
=> Value = value;

public uint? Value { get; }

public override string ToString()
=> Value?.ToString(CultureInfo.InvariantCulture) ?? "*";

public override bool Equals(object obj)
=> obj is UnlimitedNatural other && Equals(other);

public bool Equals(UnlimitedNatural other)
=> Value == other.Value;
}

#endregion

#region https://github.com/microsoft/testfx/issues/1037

[DynamicData(nameof(TestData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
[TestMethod]
public void ValidateExMessage(Exception ex)
=> Assert.AreEqual(ex?.Message, "Test exception message");

private static IEnumerable<object[]> TestData()
=> new List<object[]>()
{
new object[] { new InvalidUpdateException("Test exception message") },
};

[Serializable]
public class InvalidUpdateException : Exception
{
public InvalidUpdateException(string message)
: base(message)
{
}

public InvalidUpdateException()
{
}

[ExcludeFromCodeCoverage]
protected InvalidUpdateException(SerializationInfo serializationInfo, StreamingContext streamingContext)
{
}
}

#endregion

#region https://github.com/microsoft/testfx/issues/1094

[TestMethod]
[DynamicData(nameof(Create_test_cases_for_multiplication_of_vector_and_real), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
public void Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector(Vector2D v, double d, Vector2D expected)
{
Vector2D actual = v * d;
double delta = 0.00001;

Assert.AreEqual(expected.U, actual.U, delta, "The U-component of the vector hasn't been computed correctly.");
Assert.AreEqual(expected.V, actual.V, delta, "The V-component of the vector hasn't been computed correctly.");
}

[TestMethod]
[DynamicData(nameof(Create_test_cases_for_multiplication_of_real_and_vector), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
public void Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector(double d, Vector2D v, Vector2D expected)
{
Vector2D actual = d * v;
double delta = 0.00001;

Assert.AreEqual(expected.U, actual.U, delta, "The U-component of the vector hasn't been computed correctly.");
Assert.AreEqual(expected.V, actual.V, delta, "The V-component of the vector hasn't been computed correctly.");
}

[TestMethod]
[DynamicData(nameof(Create_test_cases_for_scalar_product_of_two_vectors), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
public void Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product(Vector2D v, Vector2D w, double expected)
{
double actual = v * w;
double delta = 0.00001;

Assert.AreEqual(expected, actual, delta, "The scalar product hasn't been computed correctly.");
}

private static IEnumerable<object[]> Create_test_cases_for_multiplication_of_vector_and_real()
=> new[]
{
new object[] { new Vector2D(0.0, 0.0), 0.0, new Vector2D(0.0, 0.0) },
new object[] { new Vector2D(0.0, 0.0), -3.5, new Vector2D(0.0, 0.0) },
new object[] { new Vector2D(2.0, 3.0), 2.1, new Vector2D(4.2, 6.3) },
new object[] { new Vector2D(1.0, 2.0), -0.73, new Vector2D(-0.73, -1.46) },
new object[] { new Vector2D(-3.4, 2.75), 22.415, new Vector2D(-76.211, 61.64125) },
new object[] { new Vector2D(12.43, -2.754), 1023.56, new Vector2D(12722.8508, -2818.88424) },
new object[] { new Vector2D(4.23, 6.81187), -13.25, new Vector2D(-56.0475, -90.2572775) },
new object[] { new Vector2D(-17.4327, -8.1956), -0.45, new Vector2D(7.844715, 3.68802) },
};

private static IEnumerable<object[]> Create_test_cases_for_multiplication_of_real_and_vector()
=> new[]
{
new object[] { 0.0, new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0) },
new object[] { -3.5, new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0) },
new object[] { 2.1, new Vector2D(2.0, 3.0), new Vector2D(4.2, 6.3) },
new object[] { -0.73, new Vector2D(1.0, 2.0), new Vector2D(-0.73, -1.46) },
new object[] { 22.415, new Vector2D(-3.4, 2.75), new Vector2D(-76.211, 61.64125) },
new object[] { 1023.56, new Vector2D(12.43, -2.754), new Vector2D(12722.8508, -2818.88424) },
new object[] { -13.25, new Vector2D(4.23, 6.81187), new Vector2D(-56.0475, -90.2572775) },
new object[] { -0.45, new Vector2D(-17.4327, -8.1956), new Vector2D(7.844715, 3.68802) },
};

private static IEnumerable<object[]> Create_test_cases_for_scalar_product_of_two_vectors()
=> new[]
{
new object[] { new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0), 0.0 },
new object[] { new Vector2D(1.0, 2.0), new Vector2D(0.0, 0.0), 0.0 },
new object[] { new Vector2D(0.0, 0.0), new Vector2D(2.0, 3.0), 0.0 },
new object[] { new Vector2D(1.0, 3.0), new Vector2D(1.0, 2.0), 7.0 },
new object[] { new Vector2D(-1.0, -2.0), new Vector2D(-3.4, 2.75), -2.1 },
new object[] { new Vector2D(3.355, -2.211), new Vector2D(12.43, -2.754), 47.791744 },
new object[] { new Vector2D(-0.15, 2.03), new Vector2D(4.23, 6.81187), 13.1935961 },
new object[] { new Vector2D(-22.7231, -78.2976), new Vector2D(-17.4327, -8.1956), 1037.82079593 },
};

public readonly struct Vector2D
{
public Vector2D(double u, double v)
{
U = u;
V = v;
}

public double U { get; }

public double V { get; }

public override string ToString()
=> FormattableString.Invariant($"Vector2D: {U:F3} / {V:F3}");

public static Vector2D operator *(Vector2D v, double d)
=> new(v.U * d, v.V * d);

public static Vector2D operator *(double d, Vector2D v)
=> new(d * v.U, d * v.V);

public static double operator *(Vector2D v, Vector2D w)
=> (v.U * w.U) + (v.V * w.V);
}

#endregion

#region https://github.com/microsoft/testfx/issues/1588

[TestMethod]
[DynamicData(nameof(GetTestData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)]
public void TestReadonlyCollectionData(string someString, MyData foo)
{
}

private static IEnumerable<object[]> GetTestData()
{
yield return new object[]
{
string.Empty, new MyData(),
};
}

public class MyData
{
private readonly SortedSet<int> _values;

public MyData() => _values = new SortedSet<int>();

public IEnumerable<int> Values => _values;
}

#endregion
}