Merge pull request #502 from manfred-brands/issue442_TestCaseSourcePa…

…rameterCheck

Issue442 test case source parameter check

documentation/NUnit1029.md

@@ -0,0 +1,73 @@
+# NUnit1029
+
+## The number of parameters provided by the TestCaseSource does not match the number of parameters in the Test method
+
+| Topic    | Value
+| :--      | :--
+| Id       | NUnit1029
+| Severity | Error
+| Enabled  | True
+| Category | Structure
+| Code     | [TestCaseSourceUsesStringAnalyzer](https://github.com/nunit/nunit.analyzers/blob/master/src/nunit.analyzers/TestCaseSourceUsage/TestCaseSourceUsesStringAnalyzer.cs)
+
+## Description
+
+The number of parameters provided by the TestCaseSource must match the number of parameters in the Test method.
+
+Note that the current implementation only works for single parameters.
+
+## Motivation
+
+A `TestCaseSourceAttribute` is used to pass parameters to a test method, but the test method does not expect any or more parameters than supplied.
+
+```charp
+private static readonly IEnumerable<string> NUnitNameSpaces = new[] { ".NUnit", ".NUnitExtensions" };
+
+[TestCaseSource(nameof(NUnitNameSpaces))]
+public void IsNUnit()
+{
+}
+```
+
+## How to fix violations
+
+Match the number of parameters between the test data and the test method.
+
+<!-- start generated config severity -->
+## Configure severity
+
+### Via ruleset file
+
+Configure the severity per project, for more info see [MSDN](https://msdn.microsoft.com/en-us/library/dd264949.aspx).
+
+### Via .editorconfig file
+
+```ini
+# NUnit1029: The number of parameters provided by the TestCaseSource does not match the number of parameters in the Test method
+dotnet_diagnostic.NUnit1029.severity = chosenSeverity
+```
+
+where `chosenSeverity` can be one of `none`, `silent`, `suggestion`, `warning`, or `error`.
+
+### Via #pragma directive
+
+```csharp
+#pragma warning disable NUnit1029 // The number of parameters provided by the TestCaseSource does not match the number of parameters in the Test method
+Code violating the rule here
+#pragma warning restore NUnit1029 // The number of parameters provided by the TestCaseSource does not match the number of parameters in the Test method
+```
+
+Or put this at the top of the file to disable all instances.
+
+```csharp
+#pragma warning disable NUnit1029 // The number of parameters provided by the TestCaseSource does not match the number of parameters in the Test method
+```
+
+### Via attribute `[SuppressMessage]`
+
+```csharp
+[System.Diagnostics.CodeAnalysis.SuppressMessage("Structure",
+    "NUnit1029:The number of parameters provided by the TestCaseSource does not match the number of parameters in the Test method",
+    Justification = "Reason...")]
+```
+<!-- end generated config severity -->

documentation/NUnit1030.md

@@ -0,0 +1,74 @@
+# NUnit1030
+
+## The type of parameter provided by the TestCaseSource does not match the type of the parameter in the Test method
+
+| Topic    | Value
+| :--      | :--
+| Id       | NUnit1030
+| Severity | Error
+| Enabled  | True
+| Category | Structure
+| Code     | [TestCaseSourceUsesStringAnalyzer](https://github.com/nunit/nunit.analyzers/blob/master/src/nunit.analyzers/TestCaseSourceUsage/TestCaseSourceUsesStringAnalyzer.cs)
+
+## Description
+
+The type of parameters provided by the TestCaseSource must match the type of parameters in the Test method.
+
+Note that the current implementation only works for single parameters.
+
+## Motivation
+
+A `TestCaseSourceAttribute` is used to pass parameters to a test method, but the test method expects a different type of parameter.
+
+```charp
+private static readonly IEnumerable<string> NUnitNameSpaces = new[] { ".NUnit", ".NUnitExtensions" };
+
+[TestCaseSource(nameof(NUnitNameSpaces))]
+public void IsNUnit(int n)
+{
+}
+```
+
+## How to fix violations
+
+Match the type of parameters between the test data and the test method.
+
+
+<!-- start generated config severity -->
+## Configure severity
+
+### Via ruleset file
+
+Configure the severity per project, for more info see [MSDN](https://msdn.microsoft.com/en-us/library/dd264949.aspx).
+
+### Via .editorconfig file
+
+```ini
+# NUnit1030: The type of parameter provided by the TestCaseSource does not match the type of the parameter in the Test method
+dotnet_diagnostic.NUnit1030.severity = chosenSeverity
+```
+
+where `chosenSeverity` can be one of `none`, `silent`, `suggestion`, `warning`, or `error`.
+
+### Via #pragma directive
+
+```csharp
+#pragma warning disable NUnit1030 // The type of parameter provided by the TestCaseSource does not match the type of the parameter in the Test method
+Code violating the rule here
+#pragma warning restore NUnit1030 // The type of parameter provided by the TestCaseSource does not match the type of the parameter in the Test method
+```
+
+Or put this at the top of the file to disable all instances.
+
+```csharp
+#pragma warning disable NUnit1030 // The type of parameter provided by the TestCaseSource does not match the type of the parameter in the Test method
+```
+
+### Via attribute `[SuppressMessage]`
+
+```csharp
+[System.Diagnostics.CodeAnalysis.SuppressMessage("Structure",
+    "NUnit1030:The type of parameter provided by the TestCaseSource does not match the type of the parameter in the Test method",
+    Justification = "Reason...")]
+```
+<!-- end generated config severity -->

documentation/index.md

@@ -47,6 +47,8 @@ Rules which enforce structural requirements on the test code.
 | [NUnit1026](https://github.com/nunit/nunit.analyzers/tree/master/documentation/NUnit1026.md) | The test or setup/teardown method is not public | :white_check_mark: | :exclamation: | :white_check_mark: |
 | [NUnit1027](https://github.com/nunit/nunit.analyzers/tree/master/documentation/NUnit1027.md) | The test method has parameters, but no arguments are supplied by attributes | :white_check_mark: | :exclamation: | :x: |
 | [NUnit1028](https://github.com/nunit/nunit.analyzers/tree/master/documentation/NUnit1028.md) | The non-test method is public | :white_check_mark: | :information_source: | :white_check_mark: |
+| [NUnit1029](https://github.com/nunit/nunit.analyzers/tree/master/documentation/NUnit1029.md) | The number of parameters provided by the TestCaseSource does not match the number of parameters in the Test method | :white_check_mark: | :exclamation: | :x: |
+| [NUnit1030](https://github.com/nunit/nunit.analyzers/tree/master/documentation/NUnit1030.md) | The type of parameter provided by the TestCaseSource does not match the type of the parameter in the Test method | :white_check_mark: | :exclamation: | :x: |
 
 ### Assertion Rules (NUnit2001 - )
 

src/nunit.analyzers.sln

@@ -9,7 +9,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "nunit.analyzers.vsix", "nun
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "nunit.analyzers.tests", "nunit.analyzers.tests\nunit.analyzers.tests.csproj", "{070974CB-B483-4347-BA5A-53ED977E639C}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{2F501E14-35D4-432A-9F93-810A0AAEA128}"
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "documentation", "documentation", "{2F501E14-35D4-432A-9F93-810A0AAEA128}"
 	ProjectSection(SolutionItems) = preProject
 		..\documentation\index.md = ..\documentation\index.md
 		..\documentation\NUnit1001.md = ..\documentation\NUnit1001.md
@@ -40,6 +40,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{2F501E14-3
 		..\documentation\NUnit1026.md = ..\documentation\NUnit1026.md
 		..\documentation\NUnit1027.md = ..\documentation\NUnit1027.md
 		..\documentation\NUnit1028.md = ..\documentation\NUnit1028.md
+		..\documentation\NUnit1029.md = ..\documentation\NUnit1029.md
+		..\documentation\NUnit1030.md = ..\documentation\NUnit1030.md
 		..\documentation\NUnit2001.md = ..\documentation\NUnit2001.md
 		..\documentation\NUnit2002.md = ..\documentation\NUnit2002.md
 		..\documentation\NUnit2003.md = ..\documentation\NUnit2003.md

src/nunit.analyzers.tests/Constants/NUnitFrameworkConstantsTests.cs

@@ -139,6 +139,7 @@ public sealed class NUnitFrameworkConstantsTests
             (nameof(NUnitFrameworkConstants.FullNameOfTypeITestBuilder), typeof(ITestBuilder)),
             (nameof(NUnitFrameworkConstants.FullNameOfTypeISimpleTestBuilder), typeof(ISimpleTestBuilder)),
             (nameof(NUnitFrameworkConstants.FullNameOfTypeIParameterDataSource), typeof(IParameterDataSource)),
+            (nameof(NUnitFrameworkConstants.FullNameOfTypeTestCaseData), typeof(TestCaseData)),
 
             (nameof(NUnitFrameworkConstants.FullNameOfTypeOneTimeSetUpAttribute), typeof(OneTimeSetUpAttribute)),
             (nameof(NUnitFrameworkConstants.FullNameOfTypeOneTimeTearDownAttribute), typeof(OneTimeTearDownAttribute)),

src/nunit.analyzers.tests/TestCaseSourceUsage/TestCaseSourceUsesStringAnalyzerTests.cs

@@ -1,4 +1,3 @@
-using System.Linq;
 using Gu.Roslyn.Asserts;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CodeFixes;
@@ -26,7 +25,7 @@ public class AnalyzeWhenNameOfSameClass
         static string[] Tests = new[] { ""Data"" };
 
         [TestCaseSource(nameof(Tests))]
-        public void Test()
+        public void Test(string data)
         {
         }
     }");
@@ -285,7 +284,7 @@ public class AnalyzeWhenSourceDoesProvideIEnumerable
         {testCaseMember}
 
         [TestCaseSource(nameof(TestCases))]
-        public void Test()
+        public void Test(object data)
         {{
         }}
     }}");
@@ -547,5 +546,158 @@ public class InnerClass
             var message = "Consider using nameof(AnotherClass.InnerClass.TestCases) instead of \"TestCases\"";
             RoslynAssert.CodeFix(analyzer, fix, expectedDiagnostic.WithMessage(message), testCode, fixedCode);
         }
+
+        [Test]
+        public void AnalyzeWhenNumberOfParametersOfTestIsLessThanProvidedByTestCaseSource()
+        {
+            var testCode = TestUtility.WrapClassInNamespaceAndAddUsing(@"
+    [TestFixture]
+    public class AnalyzeWhenNumberOfParametersOfTestIsLessThanProvidedByTestCaseSource
+    {
+        [TestCaseSource(↓nameof(TestData), new object[] { 3 })]
+        public void ShortName()
+        {
+            Assert.That(3, Is.GreaterThanOrEqualTo(0));
+        }
+
+        static IEnumerable<int> TestData(int maximum)
+        {
+            for (int i = 1; i <= maximum; i++)
+            {
+                yield return i;
+            }
+        }
+    }", additionalUsings: "using System.Collections.Generic;");
+
+            var expectedDiagnostic = ExpectedDiagnostic
+                .Create(AnalyzerIdentifiers.TestCaseSourceMismatchInNumberOfTestMethodParameters)
+                .WithMessage("The TestCaseSource provides '1' parameter(s), but the Test method expects '0' parameter(s)");
+            RoslynAssert.Diagnostics(analyzer, expectedDiagnostic, testCode);
+        }
+
+        [Test]
+        public void AnalyzeWhenNumberOfParametersOfTestIsMoreThanProvidedByTestCaseSource()
+        {
+            var testCode = TestUtility.WrapClassInNamespaceAndAddUsing(@"
+    [TestFixture]
+    public class AnalyzeWhenNumberOfParametersOfTestIsMoreThanProvidedByTestCaseSource
+    {
+        [TestCaseSource(↓nameof(TestData))]
+        public void ShortName(int x, int y)
+        {
+            Assert.That(x + y, Is.GreaterThanOrEqualTo(0));
+        }
+
+        static IEnumerable<int> TestData()
+        {
+            yield return 1;
+            yield return 2;
+            yield return 3;
+        }
+    }", additionalUsings: "using System.Collections.Generic;");
+
+            var expectedDiagnostic = ExpectedDiagnostic
+                .Create(AnalyzerIdentifiers.TestCaseSourceMismatchInNumberOfTestMethodParameters)
+                .WithMessage("The TestCaseSource provides '1' parameter(s), but the Test method expects '2' parameter(s)");
+            RoslynAssert.Diagnostics(analyzer, expectedDiagnostic, testCode);
+        }
+
+        [Test]
+        public void AnalyzeWhenParameterTypeOfTestDiffersFromTestCaseSource()
+        {
+            var testCode = TestUtility.WrapClassInNamespaceAndAddUsing(@"
+    [TestFixture]
+    public class AnalyzeWhenParameterTypeOfTestDiffersFromTestCaseSource
+    {
+        [TestCaseSource(↓nameof(TestData))]
+        public void ShortName(string message)
+        {
+            Assert.That(message.Length, Is.GreaterThanOrEqualTo(0));
+        }
+
+        static IEnumerable<int> TestData()
+        {
+            yield return 1;
+            yield return 2;
+            yield return 3;
+        }
+    }", additionalUsings: "using System.Collections.Generic;");
+
+            var expectedDiagnostic = ExpectedDiagnostic
+                .Create(AnalyzerIdentifiers.TestCaseSourceMismatchWithTestMethodParameterType)
+                .WithMessage("The TestCaseSource provides type 'int', but the Test method expects type 'string' for parameter 'message'");
+            RoslynAssert.Diagnostics(analyzer, expectedDiagnostic, testCode);
+        }
+
+        [Test]
+        public void AnalyzeWhenNumberOfParametersOfTestIsNotEvidentFromTestSource()
+        {
+            var testCode = TestUtility.WrapClassInNamespaceAndAddUsing(@"
+    [TestFixture]
+    public class AnalyzeWhenNumberOfParametersOfTestIsNotEvidentFromTestSource
+    {
+        [Explicit(""The code is wrong, but it is too complext for the analyzer to detect this."")]
+        [TestCaseSource(nameof(TestData))]
+        public void ShortName(int n)
+        {
+            Assert.That(n, Is.GreaterThanOrEqualTo(0));
+        }
+
+        static IEnumerable<object> TestData()
+        {
+            yield return new object[] { 1, 2 };
+            yield return new object[] { 2, 3 };
+        }
+    }", additionalUsings: "using System.Collections.Generic;");
+
+            // The TestCaseSource provides '>0' parameter(s), but the Test method expects '1' parameter(s)
+            // Analyzing the actual code inside the TestCaseSource method is beyond the scope of the analyzer.
+            RoslynAssert.Valid(analyzer, testCode);
+        }
+
+        [TestCase("IEnumerable", "object", "System.Collections")]
+        [TestCase("IEnumerable<object>", "object", "System.Collections.Generic")]
+        [TestCase("IEnumerable<TestCaseData>", "TestCaseData", "System.Collections.Generic")]
+        [TestCase("IEnumerable<int>", "int", "System.Collections.Generic")]
+        public void NoIssueIsRaisedWhenOneParameterIsExpectedAndTypeCannotBeDetermined(string enumerableType, string testCaseType, string collections)
+        {
+            var testCode = TestUtility.WrapClassInNamespaceAndAddUsing($@"
+        using {collections};
+
+        public class NoIssueIsRaisedWhenOneParameterIsExpectedAndTypeCannotBeDetermined
+        {{
+            [TestCaseSource(nameof(TestData))]
+            public void ShortName(int n)
+            {{
+                Assert.That(n, Is.GreaterThanOrEqualTo(0));
+            }}
+
+            public static {enumerableType} TestData() => Array.Empty<{testCaseType}>();
+        }}");
+
+            RoslynAssert.Valid(analyzer, testCode);
+        }
+
+        [TestCase("IEnumerable", "object", "System.Collections")]
+        [TestCase("IEnumerable<object>", "object", "System.Collections.Generic")]
+        [TestCase("IEnumerable<TestCaseData>", "TestCaseData", "System.Collections.Generic")]
+        public void NoIssueIsRaisedWhenMultipleParameterAreExpectedAndTypeCannotBeDetermined(string enumerableType, string testCaseType, string collections)
+        {
+            var testCode = TestUtility.WrapClassInNamespaceAndAddUsing($@"
+        using {collections};
+
+        public class NoIssueIsRaisedWhenMultipleParameterAreExpectedAndTypeCannotBeDetermined
+        {{
+            [TestCaseSource(nameof(TestData))]
+            public void ShortName(int first, int second)
+            {{
+                Assert.That(first, Is.LessThan(second));
+            }}
+
+            public static {enumerableType} TestData() => Array.Empty<{testCaseType}>();
+        }}");
+
+            RoslynAssert.Valid(analyzer, testCode);
+        }
     }
 }

src/nunit.analyzers/Constants/AnalyzerIdentifiers.cs

@@ -32,6 +32,8 @@ internal static class AnalyzerIdentifiers
         internal const string TestMethodIsNotPublic = "NUnit1026";
         internal const string SimpleTestMethodHasParameters = "NUnit1027";
         internal const string NonTestMethodIsPublic = "NUnit1028";
+        internal const string TestCaseSourceMismatchInNumberOfTestMethodParameters = "NUnit1029";
+        internal const string TestCaseSourceMismatchWithTestMethodParameterType = "NUnit1030";
 
         #endregion Structure
 

src/nunit.analyzers/Constants/NUnitFrameworkConstants.cs

@@ -107,6 +107,7 @@ public static class NUnitFrameworkConstants
         public const string FullNameOfTypeISimpleTestBuilder = "NUnit.Framework.Interfaces.ISimpleTestBuilder";
         public const string FullNameOfTypeValueSourceAttribute = "NUnit.Framework.ValueSourceAttribute";
         public const string FullNameOfTypeIParameterDataSource = "NUnit.Framework.Interfaces.IParameterDataSource";
+        public const string FullNameOfTypeTestCaseData = "NUnit.Framework.TestCaseData";
 
         public const string FullNameOfTypeOneTimeSetUpAttribute = "NUnit.Framework.OneTimeSetUpAttribute";
         public const string FullNameOfTypeOneTimeTearDownAttribute = "NUnit.Framework.OneTimeTearDownAttribute";

src/nunit.analyzers/TestCaseSourceUsage/TestCaseSourceUsageConstants.cs

@@ -29,5 +29,13 @@ internal static class TestCaseSourceUsageConstants
         internal const string TestCaseSourceSuppliesParametersTitle = "The TestCaseSource provides parameters to a source - field or property - that expects no parameters";
         internal const string TestCaseSourceSuppliesParametersMessage = "The TestCaseSource provides '{0}' parameter(s), but {1} cannot take parameters";
         internal const string TestCaseSourceSuppliesParametersDescription = "The TestCaseSource must not provide any parameters when the source is a field or a property.";
+
+        internal const string MismatchInNumberOfTestMethodParametersTitle = "The number of parameters provided by the TestCaseSource does not match the number of parameters in the Test method";
+        internal const string MismatchInNumberOfTestMethodParametersMessage = "The TestCaseSource provides '{0}' parameter(s), but the Test method expects '{1}' parameter(s)";
+        internal const string MismatchInNumberOfTestMethodParametersDescription = "The number of parameters provided by the TestCaseSource must match the number of parameters in the Test method.";
+
+        internal const string MismatchWithTestMethodParameterTypeTitle = "The type of parameter provided by the TestCaseSource does not match the type of the parameter in the Test method";
+        internal const string MismatchWithTestMethodParameterTypeMessage = "The TestCaseSource provides type '{0}', but the Test method expects type '{1}' for parameter '{2}'";
+        internal const string MismatchWithTestMethodParameterTypeDescription = "The type of parameters provided by the TestCaseSource must match the type of parameters in the Test method.";
     }
 }