Add integration tests for deterministic build support

.gitignore

@@ -301,3 +301,6 @@ __pycache__/
 
 # DeterministicSourcePaths workaround generated file DeterministicBuild.targets
 .AssemblyAttributes
+DeterministicTest.props
+test/coverlet.integration.determisticbuild/*.txt
+test/coverlet.integration.determisticbuild/runsettings

eng/build.yml

@@ -15,7 +15,7 @@ steps:
 - script: dotnet build -c $(BuildConfiguration) --no-restore
  displayName: Build
 
-- script: dotnet pack -c $(BuildConfiguration)
+- script: dotnet pack -c $(BuildConfiguration) --no-restore
  displayName: Pack
 
 - task: DotNetCoreCLI@2

test/coverlet.core.tests/Coverage/InstrumenterHelper.Assertions.cs

@@ -275,19 +275,13 @@ public static Document AssertNonInstrumentedLines(this Document document, BuildC
 
  private static BuildConfiguration GetAssemblyBuildConfiguration()
  {
- var configurationAttribute = Assembly.GetExecutingAssembly().GetCustomAttribute<AssemblyConfigurationAttribute>();
- if (configurationAttribute.Configuration.Equals("Debug", StringComparison.InvariantCultureIgnoreCase))
- {
- return Tests.BuildConfiguration.Debug;
- }
- else if (configurationAttribute.Configuration.Equals("Release", StringComparison.InvariantCultureIgnoreCase))
- {
- return Tests.BuildConfiguration.Release;
- }
- else
- {
- throw new NotSupportedException($"Build configuration '{configurationAttribute.Configuration}' not supported");
- }
+#if DEBUG
+ return BuildConfiguration.Debug;
+#endif
+#if RELEASE
+ return BuildConfiguration.Release;
+#endif
+ throw new NotSupportedException($"Build configuration not supported");
  }
  }
 }

test/coverlet.integration.determisticbuild/DeepThought.cs

@@ -0,0 +1,10 @@
+namespace Coverlet.Integration.DeterministicBuild
+{
+ public class DeepThought
+ {
+ public int AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()
+ {
+ return 42;
+ }
+ }
+}

test/coverlet.integration.determisticbuild/TemplateTest.cs

@@ -0,0 +1,14 @@
+using Xunit;
+
+namespace Coverlet.Integration.DeterministicBuild
+{
+ public class TemplateTest
+ {
+ [Fact]
+ public void Answer()
+ {
+ DeepThought dt = new DeepThought();
+ Assert.Equal(42, dt.AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything());
+ }
+ }
+}

test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+ <!-- Import coverlet version-->
+ <Import Project="$(MSBuildThisFileDirectory)\DeterministicTest.props" />
+
+ <PropertyGroup>
+ <TargetFramework>netcoreapp3.1</TargetFramework>
+ <IsPackable>false</IsPackable>
+ <AssemblyName>coverletsample.integration.determisticbuild</AssemblyName>
+ <RestoreSources>
+ https://api.nuget.org/v3/index.json;
+ $(RepoRoot)bin\$(Configuration)\Packages
+ </RestoreSources>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="coverlet.msbuild" Version="$(coverletMsbuilVersion)">
+ <PrivateAssets>all</PrivateAssets>
+ <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+ </PackageReference>
+ <PackageReference Include="coverlet.collector" Version="$(coverletCollectorsVersion)">
+ <PrivateAssets>all</PrivateAssets>
+ <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+ </PackageReference>
+ <PackageReference Include="xunit" />
+ <PackageReference Include="xunit.runner.visualstudio" />
+ <PackageReference Include="Microsoft.NET.Test.Sdk" />
+ </ItemGroup>
+</Project>

test/coverlet.integration.template/coverlet.integration.template.csproj

@@ -4,6 +4,7 @@
  <TargetFramework>netcoreapp3.1</TargetFramework>
  <IsPackable>false</IsPackable>
  <AssemblyName>coverletsamplelib.integration.template</AssemblyName>
+ <IsTestProject>false</IsTestProject>
  </PropertyGroup>
 
  <ItemGroup>

test/coverlet.integration.tests/BaseTest.cs

@@ -15,7 +15,7 @@
 namespace Coverlet.Integration.Tests
 {
  [Flags]
- enum BuildConfiguration
+ public enum BuildConfiguration
  {
  Debug = 1,
  Release = 2
@@ -24,27 +24,16 @@ enum BuildConfiguration
  public abstract class BaseTest
  {
  private static int _folderSuffix = 0;
- private BuildConfiguration GetAssemblyBuildConfiguration()
- {
- var configurationAttribute = Assembly.GetExecutingAssembly().GetCustomAttribute<AssemblyConfigurationAttribute>();
 
- if (configurationAttribute is null)
- {
- throw new ArgumentNullException("AssemblyConfigurationAttribute not found");
- }
-
- if (configurationAttribute.Configuration.Equals("Debug", StringComparison.InvariantCultureIgnoreCase))
- {
- return BuildConfiguration.Debug;
- }
- else if (configurationAttribute.Configuration.Equals("Release", StringComparison.InvariantCultureIgnoreCase))
- {
- return BuildConfiguration.Release;
- }
- else
- {
- throw new NotSupportedException($"Build configuration '{configurationAttribute.Configuration}' not supported");
- }
+ protected BuildConfiguration GetAssemblyBuildConfiguration()
+ {
+#if DEBUG
+ return BuildConfiguration.Debug;
+#endif
+#if RELEASE
+ return BuildConfiguration.Release;
+#endif
+ throw new NotSupportedException($"Build configuration not supported");
  }
 
  private protected string GetPackageVersion(string filter)
@@ -63,7 +52,7 @@ private protected string GetPackageVersion(string filter)
 
  private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispose = true, string testSDKVersion = "16.5.0")
  {
- DirectoryInfo finalRoot = Directory.CreateDirectory($"template{Interlocked.Increment(ref _folderSuffix)}");
+ DirectoryInfo finalRoot = Directory.CreateDirectory($"{Guid.NewGuid().ToString("N").Substring(0, 6)}{Interlocked.Increment(ref _folderSuffix)}");
  foreach (string file in (Directory.GetFiles($"../../../../coverlet.integration.template", "*.cs")
  .Union(Directory.GetFiles($"../../../../coverlet.integration.template", "*.csproj")
  .Union(Directory.GetFiles($"../../../../coverlet.integration.template", "nuget.config")))))
@@ -84,6 +73,8 @@ private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispo
 
  AddMicrosoftNETTestSdkRef(finalRoot.FullName, testSDKVersion);
 
+ SetIsTestProjectTrue(finalRoot.FullName);
+
  return new ClonedTemplateProject(finalRoot.FullName, cleanupOnDispose);
  }
 
@@ -131,6 +122,26 @@ private protected void UpdateNugeConfigtWithLocalPackageFolder(string projectPat
  xml.Save(nugetFile);
  }
 
+ private void SetIsTestProjectTrue(string projectPath)
+ {
+ string csproj = Path.Combine(projectPath, "coverlet.integration.template.csproj");
+ if (!File.Exists(csproj))
+ {
+ throw new FileNotFoundException("coverlet.integration.template.csproj not found", "coverlet.integration.template.csproj");
+ }
+ XDocument xml;
+ using (var csprojStream = File.OpenRead(csproj))
+ {
+ xml = XDocument.Load(csprojStream);
+ }
+
+ xml.Element("Project")
+ .Element("PropertyGroup")
+ .Element("IsTestProject").Value = "true";
+
+ xml.Save(csproj);
+ }
+
  private protected void AddMicrosoftNETTestSdkRef(string projectPath, string version)
  {
  string csproj = Path.Combine(projectPath, "coverlet.integration.template.csproj");
@@ -189,17 +200,17 @@ private protected void AddCoverletCollectosRef(string projectPath)
  xml.Save(csproj);
  }
 
- private protected string AddCollectorRunsettingsFile(string projectPath)
+ private protected string AddCollectorRunsettingsFile(string projectPath, string includeFilter = "[coverletsamplelib.integration.template]*DeepThought")
  {
  string runSettings =
-@"<?xml version=""1.0"" encoding=""utf-8"" ?>
+$@"<?xml version=""1.0"" encoding=""utf-8"" ?>
  <RunSettings>
  <DataCollectionRunSettings> 
  <DataCollectors> 
  <DataCollector friendlyName=""XPlat code coverage"" >
  <Configuration>
  <Format>json,cobertura</Format>
- <Include>[coverletsamplelib.integration.template]*DeepThought</Include>
+ <Include>{includeFilter}</Include>
  <!-- We need to include test assembly because test and code to cover are in same template project -->
  <IncludeTestAssembly>true</IncludeTestAssembly>
  </Configuration>
@@ -215,18 +226,21 @@ private protected string AddCollectorRunsettingsFile(string projectPath)
 
  private protected void AssertCoverage(ClonedTemplateProject clonedTemplateProject, string filter = "coverage.json", string standardOutput = "")
  {
- bool coverageChecked = false;
- foreach (string coverageFile in clonedTemplateProject.GetFiles(filter))
+ if (GetAssemblyBuildConfiguration() == BuildConfiguration.Debug)
  {
- JsonConvert.DeserializeObject<Modules>(File.ReadAllText(coverageFile))
- .Document("DeepThought.cs")
- .Class("Coverlet.Integration.Template.DeepThought")
- .Method("System.Int32 Coverlet.Integration.Template.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()")
- .AssertLinesCovered((6, 1), (7, 1), (8, 1));
- coverageChecked = true;
- }
+ bool coverageChecked = false;
+ foreach (string coverageFile in clonedTemplateProject.GetFiles(filter))
+ {
+ JsonConvert.DeserializeObject<Modules>(File.ReadAllText(coverageFile))
+ .Document("DeepThought.cs")
+ .Class("Coverlet.Integration.Template.DeepThought")
+ .Method("System.Int32 Coverlet.Integration.Template.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()")
+ .AssertLinesCovered((6, 1), (7, 1), (8, 1));
+ coverageChecked = true;
+ }
 
- Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}");
+ Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}");
+ }
  }
 
  private protected void UpdateProjectTargetFramework(ClonedTemplateProject project, params string[] targetFrameworks)

test/coverlet.integration.tests/Collectors.cs

@@ -42,6 +42,13 @@ public TestSDK_Preview()
 
  public abstract class Collectors : BaseTest
  {
+ private string _buildConfiguration;
+
+ public Collectors()
+ {
+ _buildConfiguration = GetAssemblyBuildConfiguration().ToString();
+ }
+
  protected string? TestSDKVersion { get; set; }
 
  private ClonedTemplateProject PrepareTemplateProject()
@@ -68,8 +75,7 @@ private protected virtual void AssertCollectorsInjection(ClonedTemplateProject c
  public void TestVsTest_Test()
  {
  using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
- string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!);
- Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput);
+ Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput);
  // We don't have any result to check because tests and code to instrument are in same assembly so we need to pass
  // IncludeTestAssembly=true we do it in other test
  Assert.Contains("Test Run Successful.", standardOutput);
@@ -81,7 +87,7 @@ public void TestVsTest_Test_Settings()
  {
  using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
  string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!);
- Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError), standardOutput);
+ Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError), standardOutput);
  Assert.Contains("Test Run Successful.", standardOutput);
  AssertCoverage(clonedTemplateProject);
  AssertCollectorsInjection(clonedTemplateProject);
@@ -92,7 +98,7 @@ public void TestVsTest_VsTest()
  {
  using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
  string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!);
- Assert.True(DotnetCli($"publish {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError), standardOutput);
+ Assert.True(DotnetCli($"publish -c {_buildConfiguration} {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError), standardOutput);
  string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => f.Contains("publish"));
  Assert.NotNull(publishedTestFile);
  Assert.True(DotnetCli($"vstest \"{publishedTestFile}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out standardOutput, out standardError), standardOutput);
@@ -107,7 +113,7 @@ public void TestVsTest_VsTest_Settings()
  {
  using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
  string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!);
- Assert.True(DotnetCli($"publish \"{clonedTemplateProject.ProjectRootPath}\"", out string standardOutput, out string standardError), standardOutput);
+ Assert.True(DotnetCli($"publish -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\"", out string standardOutput, out string standardError), standardOutput);
  string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => f.Contains("publish"));
  Assert.NotNull(publishedTestFile);
  Assert.True(DotnetCli($"vstest \"{publishedTestFile}\" --collect:\"XPlat Code Coverage\" --ResultsDirectory:\"{clonedTemplateProject.ProjectRootPath}\" /settings:\"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out standardOutput, out standardError), standardOutput);