Establish Code Coverage for WinForms testing (#514)

* include coverlet.msbuild 2.5.1 * add targets and coverage group * mock dotnet/machinelearning#2843 * Hook coverlet up to the 'RunTests' target * Upload data to codecov.io * Add code coverage badge to the README * Add clarifying comments for code coverage configuration * Move coverage upload after pick/sign/publish
dotnet · Mar 15, 2019 · 1f0aead · 1f0aead
1 parent cd90051
commit 1f0aead
Show file tree

Hide file tree

Showing 13 changed files with 168 additions and 10 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -13,10 +13,28 @@
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
   </PropertyGroup>
 
-  <!-- Make sure the pdb is a separate file, and not embedded inside the dll (which is the ArcadeSDK default behavior) -->
-  <!-- Note that this MUST COME AFTER the import of Sdk.props -->
-  <PropertyGroup>
-    <DebugType>Full</DebugType>
+  <!-- For the purposes of generating code coverage as part of the build -->
+  <PropertyGroup Condition="'$(Coverage)' == 'true'">
+      <!-- Coverlet assumes PDB files exist on disk https://github.com/tonerdo/coverlet/issues/362 -->
+      <DebugType Condition="'$(DebugType)' == 'embedded'">portable</DebugType>
+      <!-- Coverlet's PDB check cannot handle deterministic source paths https://github.com/tonerdo/coverlet/issues/363 -->
+      <DeterministicSourcePaths>false</DeterministicSourcePaths>
+
+      <!-- Note: CoverletOutput references $(TargetDir) so it is set in Directory.Build.targets -->
+
+      <CollectCoverage>true</CollectCoverage>
+      <SingleHit>true</SingleHit> <!--see https://github.com/dotnet/machinelearning/pull/2843/files#r262544802-->
+      <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies> <!-- required due to https://github.com/tonerdo/coverlet/issues/72 -->
+      <CoverletOutputFormat>opencover</CoverletOutputFormat>
+      <Include>[System.Windows.*]*</Include>
+      <!-- Exclude only Obsolete and tagged with ExcludeFromCodeCoverage !!Avoid using this!! -->
+      <Exclude />
+      <ExcludeByAttribute>Obsolete,ExcludeFromCodeCoverage</ExcludeByAttribute>
+      <ExcludeByFile />
   </PropertyGroup>
 
+  <!-- Make sure coverage targets are defined even in projects where coverlet is not installed -->
+  <Target Name="InstrumentModulesAfterBuild" />
+  <Target Name="GenerateCoverageResult" />
+
 </Project>
diff --git a/Directory.Build.targets b/Directory.Build.targets
@@ -8,5 +8,18 @@
   <ItemGroup>
     <PackageReference Update="Microsoft.NETCore.Platforms" Version="$(MicrosoftNETCorePlatformsPackageVersion)" />
   </ItemGroup>
-</Project>
 
+  <!-- Set code coverage properties that reference properties not available in Directory.Build.props -->
+  <PropertyGroup Condition="'$(Coverage)' == 'true'">
+    <CoverletOutput>$(TargetDir)coverage\$(MSBuildProjectName).coverage</CoverletOutput>
+  </PropertyGroup>
+
+  <Target Name="InstrumentModulesNoBuildBeforeTest" Condition="'$(CollectCoverage)' == 'true'" BeforeTargets="RunTests">
+    <CallTarget Targets="InstrumentModulesAfterBuild" />
+  </Target>
+
+  <Target Name="GenerateCoverageResultAfterTest" Condition="'$(CollectCoverage)' == 'true'" AfterTargets="RunTests">
+    <CallTarget Targets="GenerateCoverageResult" />
+  </Target>
+
+</Project>
diff --git a/README.md b/README.md
@@ -1,6 +1,7 @@
 # Windows Forms
 
-[![Build Status](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/winforms/dotnet-winforms%20CI)](https://dnceng.visualstudio.com/public/_build/latest?definitionId=267&branch=master) 
+[![Build Status](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/winforms/dotnet-winforms%20CI)](https://dnceng.visualstudio.com/public/_build/latest?definitionId=267&branch=master)
+[![codecov](https://codecov.io/gh/dotnet/winforms/branch/master/graph/badge.svg?flag=production)](https://codecov.io/gh/dotnet/winforms)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/dotnet/winforms/blob/master/LICENSE.TXT)
 
 Windows Forms (WinForms) is a UI framework for building Windows desktop applications. It is a .NET wrapper over Windows user interface libraries, such as User32 and GDI+. It also offers controls and other functionality that is unique to Windows Forms.

diff --git a/codecov.yml b/codecov.yml
@@ -0,0 +1,34 @@
+# https://docs.codecov.io/docs/codecov-yaml
+# https://github.com/codecov/support/wiki/Codecov-Yaml
+
+coverage:
+  status:
+    project:
+      default: false
+    patch:
+      default: false
+  fixes:
+    - "eng/::/"
+
+comment:
+  layout: "diff, flags"
+
+flags:
+  production:
+    paths:
+      - src/Common/src/
+      - src/System.Design/src/
+      - src/System.Drawing/src/
+      - src/System.Drawing.Design/src/
+      - src/System.Windows.Forms/src/
+      - src/System.Windows.Forms.Design/src/
+      - src/System.Windows.Forms.Design.Editors/src/
+  test:
+    paths:
+      - src/Common/tests/
+      - src/System.Design/tests/
+      - src/System.Drawing/tests/
+      - src/System.Drawing.Design/tests/
+      - src/System.Windows.Forms/tests/
+      - src/System.Windows.Forms.Design/tests/
+      - src/System.Windows.Forms.Design.Editors/tests/
diff --git a/eng/CodeCoverage.proj b/eng/CodeCoverage.proj
@@ -0,0 +1,61 @@
+<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Codecov">
+
+  <PropertyGroup>
+    <!-- We need to specify a framework in order for the Restore target to work -->
+    <TargetFramework>netcoreapp3.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+      <PackageReference Include="Codecov" Version="$(CodecovVersion)" />
+      <PackageReference Include="ReportGenerator" Version="$(ReportGeneratorVersion)" />
+  </ItemGroup>
+  <Target Name="Codecov" DependsOnTargets="Restore">
+    <PropertyGroup>
+      <_CodecovPath>$(NuGetPackageRoot)codecov\$(CodecovVersion)\tools\Codecov.exe</_CodecovPath>
+      <_ReportGeneratorPath>$(NuGetPackageRoot)reportgenerator\$(ReportGeneratorVersion)\tools\net47\ReportGenerator.exe</_ReportGeneratorPath>
+
+      <!-- The name of the source branch of the pull request. Ideally this would include the fork name, such as
+        'sharwell:coverage-cleanup', but only the branch name itself is available, such as 'coverage-cleanup'. -->
+      <_BranchName Condition="'$(_BranchName)' == ''">$(SYSTEM_PULLREQUEST_SOURCEBRANCH)</_BranchName>
+      <_BranchName Condition="'$(_BranchName)' == ''">$(BUILD_SOURCEBRANCHNAME)</_BranchName>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <!-- [...\artifacts\bin\][project]\[configuration]\[framework]\coverage\[project].coverage-->
+      <_CoverageReports Include="$(ArtifactsBinDir)*\$(Configuration)\*\coverage\*.coverage" />
+    </ItemGroup>
+
+    <!-- Merge multiple coverlet reports into a single Cobertura report before uploading to codecov.io, in order to
+      reduce upload size and load on the codecov.io processing servers. -->
+    <Message Importance="high" Text="&quot;$(_ReportGeneratorPath)&quot; &quot;-reports:@(_CoverageReports)&quot; -targetdir:$(BaseOutputPath)coverage -reporttypes:Cobertura" />
+    <Exec Command="&quot;$(_ReportGeneratorPath)&quot; &quot;-reports:@(_CoverageReports)&quot; -targetdir:$(BaseOutputPath)coverage -reporttypes:Cobertura" />
+
+    <ItemGroup>
+      <!-- These are the best known interpretation of options defined in
+        https://github.com/codecov/codecov-exe/blob/master/Source/Codecov/Program/CommandLineOptions.cs -->
+      <_CodecovArgs Include="-f;$(BaseOutputPath)coverage\Cobertura.xml" />
+      <_CodecovArgs Include="-r;$(BUILD_REPOSITORY_NAME)" Condition="'$(BUILD_REPOSITORY_NAME)' != ''" />
+      <_CodecovArgs Include="--pr;$(SYSTEM_PULLREQUEST_PULLREQUESTNUMBER)" Condition="'$(SYSTEM_PULLREQUEST_PULLREQUESTNUMBER)' != ''" />
+      <_CodecovArgs Include="-b;$(BUILD_BUILDNUMBER)" Condition="'$(BUILD_BUILDNUMBER)' != ''" />
+      <_CodecovArgs Include="--branch;$(_BranchName)" Condition="'$(_BranchName)' != ''" />
+      <_CodecovArgs Include="-c;$(BUILD_SOURCEVERSION)" Condition="'$(BUILD_SOURCEVERSION)' != ''" />
+      <_CodecovArgs Include="-n;$(BUILD_DEFINITIONNAME)" Condition="'$(BUILD_DEFINITIONNAME)' != ''" />
+      <_CodecovArgs Include="-t;$(CodeCovToken)" Condition="'$(CodeCovToken)' != ''" />
+
+      <_CodecovFlags Include="$(Configuration)" Condition="'$(Configuration)' != ''" />
+      <_CodecovProductionFlags Include="@(_CodecovFlags)" />
+      <_CodecovProductionFlags Include="production" />
+      <_CodecovTestFlags Include="@(_CodecovFlags)" />
+      <_CodecovTestFlags Include="test" />
+    </ItemGroup>
+
+    <!-- Upload the coverage file with a 'production' flag, which will be filtered by codecov.io to production code -->
+    <Message Importance="high" Text="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ') --flag @(_CodecovProductionFlags, ',')" />
+    <Exec Command="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ') --flag @(_CodecovProductionFlags, ',')" />
+
+    <!-- Upload the coverage file with a 'test' flag, which will be filtered by codecov.io to test code -->
+    <Message Importance="high" Text="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ') --flag @(_CodecovTestFlags, ',')" />
+    <Exec Command="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ') --flag @(_CodecovTestFlags, ',')" />
+  </Target>
+
+</Project>
diff --git a/eng/FacadeAssemblies.props b/eng/FacadeAssemblies.props
@@ -9,6 +9,9 @@
     <IsPartialFacadeAssembly>true</IsPartialFacadeAssembly>
     <GenFacadesIgnoreMissingTypes>true</GenFacadesIgnoreMissingTypes>
 
+    <!-- Produce a full PDB for facade assemblies -->
+    <DebugType>full</DebugType>
+
     <!-- allow desktop references -->
     <AssetTargetFallback>net472</AssetTargetFallback>
     <NoWarn>$(NoWarn);NU1701</NoWarn>

diff --git a/eng/Versions.props b/eng/Versions.props
@@ -33,6 +33,12 @@
     <XUnitExtensibilityExecutionVersion>$(XUnitVersion)</XUnitExtensibilityExecutionVersion>
     <XUnitStaFactPackageVersion>0.3.5</XUnitStaFactPackageVersion>
   </PropertyGroup>
+  <!-- Code Coverage -->
+  <PropertyGroup>
+    <CoverletMSBuildPackageVersion>2.6.0</CoverletMSBuildPackageVersion>
+    <CodecovVersion>1.1.1</CodecovVersion>
+    <ReportGeneratorVersion>4.0.9</ReportGeneratorVersion>
+  </PropertyGroup>
   <!-- Additional unchanging dependencies -->
   <PropertyGroup>
     <AccessibilityPackageVersion>4.6.0-alpha-27122-5</AccessibilityPackageVersion>

diff --git a/eng/ci.yml b/eng/ci.yml
@@ -109,8 +109,10 @@ jobs:
             _PublishType: none
             _SignType: test
             _DotNetPublishToBlobFeed : false
+            _Coverage: true
           Build_Release:
             _BuildConfig: Release
+            _Coverage: false
 
       steps:
       - checkout: self
@@ -121,22 +123,29 @@ jobs:
         displayName: Pre-Build - Set VSO Variables
 
       # Build and rename binlog
+      # Note: The /p:Coverage argument is passed here since some build properties change to accommodate running with
+      #       coverage. This is part of the workarounds for https://github.com/tonerdo/coverlet/issues/362 and
+      #       https://github.com/tonerdo/coverlet/issues/363.
       - script: eng\cibuild.cmd
           -prepareMachine
           -configuration $(_BuildConfig)
           $(_OfficialBuildIdArgs)
           -build
+          /p:Coverage=$(_Coverage)
           /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Build.binlog          
         displayName: Build
 
     # Run Unit Tests
         # this should no longer be necessary once Helix is ready to report back code coverage
+        # Tests are run with /m:1 to work around https://github.com/tonerdo/coverlet/issues/364
       - script: eng\cibuild.cmd
           -prepareMachine
           -configuration $(_BuildConfig)
           $(_OfficialBuildIdArgs)
           -test
-          /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Test.binlog #SendToHelix.binlog for helix version        
+          /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Test.binlog
+          /p:Coverage=$(_Coverage)
+          /m:1
         displayName: Run Unit Tests on Build Machine (Debug only)
         condition: succeeded() # ,  eq(variables['_BuildConfig'], 'Debug')) # on build machine (with test signing)
 
@@ -172,6 +181,15 @@ jobs:
           /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\PackSignPublish.binlog
         displayName: Pack, Sign, and Publish
 
+      # Upload code coverage data
+      - script: $(Build.SourcesDirectory)/.dotnet/dotnet msbuild
+          eng/CodeCoverage.proj
+          /p:Configuration=$(_BuildConfig)
+          /p:CodeCovToken=$(CODECOV_TOKEN)
+          /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\CodeCoverage.binlog
+        displayName: Upload coverage to codecov.io
+        condition: and(succeeded(), eq(variables._Coverage, 'true'))
+
       # Perform some internal-build-only tasks
       - ${{ if eq(parameters.runAsPublic, 'false') }}:
 
@@ -187,4 +205,4 @@ jobs:
         # Run component governance detection (only for release; only for internal)
         - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
           displayName: Component Governance Detection
-          condition: and(succeeded(), eq(variables['_BuildConfig'], 'Release'))
+          condition: and(succeeded(), eq(variables['_BuildConfig'], 'Release'))
diff --git a/eng/packageContent.targets b/eng/packageContent.targets
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <RefPackagePath Condition="'$(RefPackagePath)' == ''">ref/$(TargetFramework)</RefPackagePath>
-    <IncludePdbInPackage Condition="'$(IncludePdbInPackage)' == '' AND '$(IsFacadeAssembly)' != 'true'">true</IncludePdbInPackage>
+    <IncludePdbInPackage Condition="'$(IncludePdbInPackage)' == '' AND '$(IsFacadeAssembly)' != 'true' AND '$(DebugType)' != 'embedded'">true</IncludePdbInPackage>
     <IncludeResourcesInPackage>true</IncludeResourcesInPackage>
   </PropertyGroup>
 

diff --git a/...ows.Forms.Design.Editors/tests/UnitTests/System.Windows.Forms.Design.Editors.Tests.csproj b/...ows.Forms.Design.Editors/tests/UnitTests/System.Windows.Forms.Design.Editors.Tests.csproj
@@ -8,6 +8,7 @@
   <ItemGroup>    
     <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
     <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
+    <PackageReference Include="coverlet.msbuild" Version="$(CoverletMSBuildPackageVersion)" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/System.Windows.Forms.Design/tests/UnitTests/System.Windows.Forms.Design.Tests.csproj b/src/System.Windows.Forms.Design/tests/UnitTests/System.Windows.Forms.Design.Tests.csproj
@@ -9,12 +9,13 @@
   <ItemGroup>
     <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
     <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
+    <PackageReference Include="coverlet.msbuild" Version="$(CoverletMSBuildPackageVersion)" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\src\System.Windows.Forms.Design.csproj" />
     <Compile Include="..\..\..\Common\tests\CommonTestHelper.cs" Link="Common\CommonTestHelper.cs" />
     <ProjectReference Include="..\..\..\Common\tests\InternalUtilitiesForTests\InternalUtilitiesForTests.csproj" />
   </ItemGroup>
-  
+
 </Project>
diff --git a/...nTests/System.Windows.Forms.IntegrationTests/System.Windows.Forms.IntegrationTests.csproj b/...nTests/System.Windows.Forms.IntegrationTests/System.Windows.Forms.IntegrationTests.csproj
@@ -9,6 +9,7 @@
   <ItemGroup>
     <PackageReference Include="$(MoqPackage)" Version="$(MoqPackageVersion)" />
     <PackageReference Include="$(XUnitStaFactPackage)" Version="$(XUnitStaFactPackageVersion)" />
+    <PackageReference Include="coverlet.msbuild" Version="$(CoverletMSBuildPackageVersion)" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/System.Windows.Forms/tests/UnitTests/System.Windows.Forms.Tests.csproj b/src/System.Windows.Forms/tests/UnitTests/System.Windows.Forms.Tests.csproj
@@ -9,6 +9,7 @@
   <ItemGroup>
     <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
     <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
+    <PackageReference Include="coverlet.msbuild" Version="$(CoverletMSBuildPackageVersion)" />
   </ItemGroup>
 
   <ItemGroup>