-
Notifications
You must be signed in to change notification settings - Fork 280
Add non-serialization strategy for data unfolding #4467
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -169,22 +169,11 @@ internal List<TestResult> RunTestMethod() | |
|
||
bool isDataDriven = false; | ||
var parentStopwatch = Stopwatch.StartNew(); | ||
if (_test.DataType == DynamicDataType.ITestDataSource) | ||
if (TryExecuteITestDataSource(results)) | ||
{ | ||
if (_test.TestDataSourceIgnoreMessage is not null) | ||
{ | ||
return [new() { Outcome = UTF.UnitTestOutcome.Ignored, IgnoreReason = _test.TestDataSourceIgnoreMessage }]; | ||
} | ||
|
||
object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData); | ||
TestResult[] testResults = ExecuteTestWithDataSource(null, data); | ||
results.AddRange(testResults); | ||
} | ||
else if (TryExecuteDataSourceBasedTests(results)) | ||
{ | ||
isDataDriven = true; | ||
} | ||
else if (TryExecuteFoldedDataDrivenTests(results)) | ||
else if (TryExecuteDataSourceBasedTests(results) | ||
|| TryExecuteFoldedDataDrivenTests(results)) | ||
{ | ||
isDataDriven = true; | ||
} | ||
|
@@ -526,4 +515,42 @@ private static List<TestResult> UpdateResultsWithParentInfo( | |
|
||
return updatedResults; | ||
} | ||
|
||
private bool TryExecuteITestDataSource(List<TestResult> results) | ||
{ | ||
if (_test.DataType != DynamicDataType.ITestDataSource) | ||
{ | ||
return false; | ||
} | ||
|
||
UTF.ITestDataSource? dataSource; | ||
object?[]? data; | ||
if (_test.SerializedData?.Length == 3) | ||
{ | ||
if (!Enum.TryParse(_test.SerializedData[0], out TestDataSourceUnfoldingStrategy _) | ||
|| !int.TryParse(_test.SerializedData[1], out int dataSourceIndex) | ||
|| !int.TryParse(_test.SerializedData[2], out int dataIndex)) | ||
{ | ||
throw ApplicationStateGuard.Unreachable(); | ||
} | ||
|
||
dataSource = _testMethodInfo.GetAttributes<Attribute>(false)?.OfType<UTF.ITestDataSource>().Skip(dataSourceIndex).FirstOrDefault(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Runtime team just told me not to rely on guaranteed reflection order. I'll try to see what other option we have. Given that this is currently always working fine, I'd vote for adding a comment and moving on with the PR. |
||
if (dataSource is null) | ||
{ | ||
throw ApplicationStateGuard.Unreachable(); | ||
} | ||
|
||
data = dataSource.GetData(_testMethodInfo.MethodInfo).Skip(dataIndex).FirstOrDefault(); | ||
} | ||
else | ||
{ | ||
dataSource = null; | ||
data = DataSerializationHelper.Deserialize(_test.SerializedData); | ||
} | ||
|
||
TestResult[] testResults = ExecuteTestWithDataSource(dataSource, data); | ||
results.AddRange(testResults); | ||
|
||
return true; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,8 @@ | ||
// Copyright (c) Microsoft Corporation. All rights reserved. | ||
// Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
||
using System.ComponentModel; | ||
|
||
namespace Microsoft.VisualStudio.TestTools.UnitTesting; | ||
|
||
/// <summary> | ||
|
@@ -22,17 +24,33 @@ public enum TestDataSourceUnfoldingStrategy : byte | |
/// <summary> | ||
/// MSTest will decide whether to unfold the parameterized test based on value from the assembly level attribute | ||
/// <see cref="TestDataSourceOptionsAttribute" />. If no assembly level attribute is specified, then the default | ||
/// configuration is to unfold. | ||
/// configuration is to unfold using <see cref="System.Runtime.Serialization.Json.DataContractJsonSerializer"/>. | ||
/// </summary> | ||
Auto, | ||
|
||
/// <summary> | ||
/// Each data row is treated as a separate test case. | ||
/// </summary> | ||
/// <inheritdoc cref="UnfoldUsingDataContractJsonSerializer"/> | ||
[EditorBrowsable(EditorBrowsableState.Never)] | ||
[Obsolete("Use 'UnfoldUsingDataContractJsonSerializer' instead")] | ||
Evangelink marked this conversation as resolved.
Show resolved
Hide resolved
|
||
Unfold, | ||
|
||
/// <summary> | ||
/// The parameterized test is not unfolded; all data rows are treated as a single test case. | ||
/// </summary> | ||
Fold, | ||
|
||
/// <summary> | ||
/// Each data row is treated as a separate test case, and the data is unfolded using | ||
/// <see cref="System.Runtime.Serialization.Json.DataContractJsonSerializer"/>. | ||
/// </summary> | ||
UnfoldUsingDataContractJsonSerializer, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we make this the same numeric value as There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's probably not important, is it? |
||
|
||
/// <summary> | ||
/// Each data row is treated as a separate test case, and the data is unfolded using the data | ||
/// source index and data index. | ||
/// </summary> | ||
/// <remarks> | ||
/// Using this strategy will alter the test ID if the data source is reordered, as it depends | ||
/// on the index of the data. This may affect the ability to track test cases over time. | ||
/// </remarks> | ||
UnfoldUsingDataIndex, | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
// Copyright (c) Microsoft Corporation. All rights reserved. | ||
// Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
||
using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
|
||
namespace DataRowTestProject; | ||
|
||
[TestClass] | ||
public class DataRowTests_Index | ||
{ | ||
#region // https://github.com/microsoft/testfx/issues/2390 | ||
|
||
[TestMethod] | ||
[DataRow((byte)0, new object[] { (byte)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
[DataRow((short)0, new object[] { (short)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
[DataRow(0L, new object[] { 0L }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
public void NestedInputTypes(object org, object nested) | ||
=> Assert.IsTrue( | ||
org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal), | ||
string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name)); | ||
|
||
[TestMethod] | ||
[DataRow(0, new object[] { (byte)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
public void NestedInputTypesInvalid(object org, object nested) | ||
=> Assert.IsFalse( | ||
org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal), | ||
string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name)); | ||
|
||
#endregion | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,252 @@ | ||
// Copyright (c) Microsoft Corporation. All rights reserved. | ||
// Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
||
using System.Collections.ObjectModel; | ||
using System.Runtime.Serialization; | ||
|
||
using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
|
||
namespace DataRowTestProject; | ||
|
||
[TestClass] | ||
public class IndexBasedDataTests | ||
{ | ||
#region https://github.com/microsoft/testfx/issues/908 | ||
|
||
[TestMethod] | ||
[DynamicData(nameof(AddTestCases), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
public void Add_ShouldAddTheExpectedValues(Collection<string> systemUnderTest, IEnumerable<string> itemsToAdd, Collection<string> expected) | ||
{ | ||
// The actual tested method is irrelevant. Executing this empty tests provokes the error | ||
} | ||
|
||
public static IEnumerable<object[]> AddTestCases | ||
{ | ||
get | ||
{ | ||
var sut = new Collection<string>(); | ||
var expected = new Collection<string>(); | ||
yield return new object[] { sut, Enumerable.Empty<string>(), expected }; | ||
|
||
sut = new Collection<string>() { "1" }; | ||
expected = new Collection<string>() { "1" }; | ||
yield return new object[] { sut, Enumerable.Empty<string>(), expected }; | ||
|
||
sut = new Collection<string>(); | ||
expected = new Collection<string>() { "1" }; | ||
yield return new object[] { sut, new[] { "1" }, expected }; | ||
|
||
sut = new Collection<string>() { "1", "a", "b" }; | ||
expected = new Collection<string>() { "1", "a", "b", "z", "j" }; | ||
yield return new object[] { sut, new[] { "z", "j" }, expected }; | ||
} | ||
} | ||
|
||
#endregion | ||
|
||
#region https://github.com/microsoft/testfx/issues/1022 | ||
|
||
[TestMethod] | ||
[DynamicData(nameof(UnlimitedNaturalData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
public void TestUnlimitedNatural(UnlimitedNatural testObject, UnlimitedNatural other, bool expected) | ||
{ | ||
bool actual = testObject.Equals(other); | ||
Assert.AreEqual(expected, actual); | ||
} | ||
|
||
public static IEnumerable<object[]> UnlimitedNaturalData | ||
{ | ||
get | ||
{ | ||
yield return new object[] { UnlimitedNatural.Zero, UnlimitedNatural.Infinite, false }; | ||
yield return new object[] { UnlimitedNatural.Zero, UnlimitedNatural.Zero, true }; | ||
} | ||
} | ||
|
||
#pragma warning disable CS0659 // Type overrides Object.Equals(object o) but does not override Object.GetHashCode() | ||
#pragma warning disable CA2231 // Overload operator equals on overriding value type Equals | ||
public readonly struct UnlimitedNatural : IEquatable<UnlimitedNatural> | ||
#pragma warning restore CA2231 // Overload operator equals on overriding value type Equals | ||
#pragma warning restore CS0659 // Type overrides Object.Equals(object o) but does not override Object.GetHashCode() | ||
{ | ||
public static readonly UnlimitedNatural Infinite; | ||
public static readonly UnlimitedNatural One = new(1); | ||
public static readonly UnlimitedNatural Zero = new(0); | ||
|
||
public UnlimitedNatural(uint? value) | ||
=> Value = value; | ||
|
||
public uint? Value { get; } | ||
|
||
public override string ToString() | ||
=> Value?.ToString(CultureInfo.InvariantCulture) ?? "*"; | ||
|
||
public override bool Equals(object obj) | ||
=> obj is UnlimitedNatural other && Equals(other); | ||
|
||
public bool Equals(UnlimitedNatural other) | ||
=> Value == other.Value; | ||
} | ||
|
||
#endregion | ||
|
||
#region https://github.com/microsoft/testfx/issues/1037 | ||
|
||
[DynamicData(nameof(TestData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
[TestMethod] | ||
public void ValidateExMessage(Exception ex) | ||
=> Assert.AreEqual(ex?.Message, "Test exception message"); | ||
|
||
private static IEnumerable<object[]> TestData() | ||
=> new List<object[]>() | ||
{ | ||
new object[] { new InvalidUpdateException("Test exception message") }, | ||
}; | ||
|
||
[Serializable] | ||
public class InvalidUpdateException : Exception | ||
{ | ||
public InvalidUpdateException(string message) | ||
: base(message) | ||
{ | ||
} | ||
|
||
public InvalidUpdateException() | ||
{ | ||
} | ||
|
||
[ExcludeFromCodeCoverage] | ||
protected InvalidUpdateException(SerializationInfo serializationInfo, StreamingContext streamingContext) | ||
{ | ||
} | ||
} | ||
|
||
#endregion | ||
|
||
#region https://github.com/microsoft/testfx/issues/1094 | ||
|
||
[TestMethod] | ||
[DynamicData(nameof(Create_test_cases_for_multiplication_of_vector_and_real), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
public void Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector(Vector2D v, double d, Vector2D expected) | ||
{ | ||
Vector2D actual = v * d; | ||
double delta = 0.00001; | ||
|
||
Assert.AreEqual(expected.U, actual.U, delta, "The U-component of the vector hasn't been computed correctly."); | ||
Assert.AreEqual(expected.V, actual.V, delta, "The V-component of the vector hasn't been computed correctly."); | ||
} | ||
|
||
[TestMethod] | ||
[DynamicData(nameof(Create_test_cases_for_multiplication_of_real_and_vector), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
public void Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector(double d, Vector2D v, Vector2D expected) | ||
{ | ||
Vector2D actual = d * v; | ||
double delta = 0.00001; | ||
|
||
Assert.AreEqual(expected.U, actual.U, delta, "The U-component of the vector hasn't been computed correctly."); | ||
Assert.AreEqual(expected.V, actual.V, delta, "The V-component of the vector hasn't been computed correctly."); | ||
} | ||
|
||
[TestMethod] | ||
[DynamicData(nameof(Create_test_cases_for_scalar_product_of_two_vectors), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
public void Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product(Vector2D v, Vector2D w, double expected) | ||
{ | ||
double actual = v * w; | ||
double delta = 0.00001; | ||
|
||
Assert.AreEqual(expected, actual, delta, "The scalar product hasn't been computed correctly."); | ||
} | ||
|
||
private static IEnumerable<object[]> Create_test_cases_for_multiplication_of_vector_and_real() | ||
=> new[] | ||
{ | ||
new object[] { new Vector2D(0.0, 0.0), 0.0, new Vector2D(0.0, 0.0) }, | ||
new object[] { new Vector2D(0.0, 0.0), -3.5, new Vector2D(0.0, 0.0) }, | ||
new object[] { new Vector2D(2.0, 3.0), 2.1, new Vector2D(4.2, 6.3) }, | ||
new object[] { new Vector2D(1.0, 2.0), -0.73, new Vector2D(-0.73, -1.46) }, | ||
new object[] { new Vector2D(-3.4, 2.75), 22.415, new Vector2D(-76.211, 61.64125) }, | ||
new object[] { new Vector2D(12.43, -2.754), 1023.56, new Vector2D(12722.8508, -2818.88424) }, | ||
new object[] { new Vector2D(4.23, 6.81187), -13.25, new Vector2D(-56.0475, -90.2572775) }, | ||
new object[] { new Vector2D(-17.4327, -8.1956), -0.45, new Vector2D(7.844715, 3.68802) }, | ||
}; | ||
|
||
private static IEnumerable<object[]> Create_test_cases_for_multiplication_of_real_and_vector() | ||
=> new[] | ||
{ | ||
new object[] { 0.0, new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0) }, | ||
new object[] { -3.5, new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0) }, | ||
new object[] { 2.1, new Vector2D(2.0, 3.0), new Vector2D(4.2, 6.3) }, | ||
new object[] { -0.73, new Vector2D(1.0, 2.0), new Vector2D(-0.73, -1.46) }, | ||
new object[] { 22.415, new Vector2D(-3.4, 2.75), new Vector2D(-76.211, 61.64125) }, | ||
new object[] { 1023.56, new Vector2D(12.43, -2.754), new Vector2D(12722.8508, -2818.88424) }, | ||
new object[] { -13.25, new Vector2D(4.23, 6.81187), new Vector2D(-56.0475, -90.2572775) }, | ||
new object[] { -0.45, new Vector2D(-17.4327, -8.1956), new Vector2D(7.844715, 3.68802) }, | ||
}; | ||
|
||
private static IEnumerable<object[]> Create_test_cases_for_scalar_product_of_two_vectors() | ||
=> new[] | ||
{ | ||
new object[] { new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0), 0.0 }, | ||
new object[] { new Vector2D(1.0, 2.0), new Vector2D(0.0, 0.0), 0.0 }, | ||
new object[] { new Vector2D(0.0, 0.0), new Vector2D(2.0, 3.0), 0.0 }, | ||
new object[] { new Vector2D(1.0, 3.0), new Vector2D(1.0, 2.0), 7.0 }, | ||
new object[] { new Vector2D(-1.0, -2.0), new Vector2D(-3.4, 2.75), -2.1 }, | ||
new object[] { new Vector2D(3.355, -2.211), new Vector2D(12.43, -2.754), 47.791744 }, | ||
new object[] { new Vector2D(-0.15, 2.03), new Vector2D(4.23, 6.81187), 13.1935961 }, | ||
new object[] { new Vector2D(-22.7231, -78.2976), new Vector2D(-17.4327, -8.1956), 1037.82079593 }, | ||
}; | ||
|
||
public readonly struct Vector2D | ||
{ | ||
public Vector2D(double u, double v) | ||
{ | ||
U = u; | ||
V = v; | ||
} | ||
|
||
public double U { get; } | ||
|
||
public double V { get; } | ||
|
||
public override string ToString() | ||
=> FormattableString.Invariant($"Vector2D: {U:F3} / {V:F3}"); | ||
|
||
public static Vector2D operator *(Vector2D v, double d) | ||
=> new(v.U * d, v.V * d); | ||
|
||
public static Vector2D operator *(double d, Vector2D v) | ||
=> new(d * v.U, d * v.V); | ||
|
||
public static double operator *(Vector2D v, Vector2D w) | ||
=> (v.U * w.U) + (v.V * w.V); | ||
} | ||
|
||
#endregion | ||
|
||
#region https://github.com/microsoft/testfx/issues/1588 | ||
|
||
[TestMethod] | ||
[DynamicData(nameof(GetTestData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] | ||
public void TestReadonlyCollectionData(string someString, MyData foo) | ||
{ | ||
} | ||
|
||
private static IEnumerable<object[]> GetTestData() | ||
{ | ||
yield return new object[] | ||
{ | ||
string.Empty, new MyData(), | ||
}; | ||
} | ||
|
||
public class MyData | ||
{ | ||
private readonly SortedSet<int> _values; | ||
|
||
public MyData() => _values = new SortedSet<int>(); | ||
|
||
public IEnumerable<int> Values => _values; | ||
} | ||
|
||
#endregion | ||
} |
Uh oh!
There was an error while loading. Please reload this page.