Add hangdump and crash dump capabilities and options (#2434)

Separate how hang and crash dumps are done and add more options.

azure-pipelines.yml

@@ -63,13 +63,15 @@ jobs:
       testResultsFormat: VSTest
       testResultsFiles: '**\*.trx'
     condition: succeededOrFailed()
-
-- job: Linux
-  pool:
-    vmImage: 'ubuntu-16.04'
-  variables:
-      buildConfiguration: 'Release'
-  steps:
-  - script: ./build.sh -c $(buildConfiguration)
-    displayName: './build.sh -c $(buildConfiguration)'
+
+# Linux build does not work when we mix runtimes and 
+# we don't use the results to do anything, skipping it for now  
+# - job: Linux
+#   pool:
+#     vmImage: 'ubuntu-16.04'
+#   variables:
+#       buildConfiguration: 'Release'
+#   steps:
+#   - script: ./build.sh -c $(buildConfiguration)
+#     displayName: './build.sh -c $(buildConfiguration)'
 

scripts/build.ps1

@@ -94,6 +94,7 @@ Write-Verbose "Setup build configuration."
 $TPB_Solution = "TestPlatform.sln"
 $TPB_TestAssets_Solution = Join-Path $env:TP_ROOT_DIR "test\TestAssets\TestAssets.sln"
 $TPB_TargetFramework = "net451"
+$TPB_TargetFramework472 = "net472"
 $TPB_TargetFrameworkCore20 = "netcoreapp2.1"
 $TPB_TargetFrameworkUap = "uap10.0"
 $TPB_TargetFrameworkNS2_0 = "netstandard2.0"
@@ -321,7 +322,7 @@ function Publish-Package
     Publish-PackageInternal $settingsMigratorProject $TPB_TargetFramework $fullCLRPackageDir
 
     Write-Log "Package: Publish src\datacollector\datacollector.csproj"
-    Publish-PackageInternal $dataCollectorProject $TPB_TargetFramework $fullCLRPackageDir
+    Publish-PackageInternal $dataCollectorProject $TPB_TargetFramework472 $fullCLRPackageDir
     Publish-PackageInternal $dataCollectorProject $TPB_TargetFrameworkCore20 $coreCLR20PackageDir
 
     # Publish testhost
@@ -351,7 +352,7 @@ function Publish-Package
     Set-ScriptFailedOnError
 
     # Copy over the Full CLR built datacollector package assemblies to the Core CLR package folder along with testhost
-    Publish-PackageInternal $dataCollectorProject $TPB_TargetFramework $fullDestDir
+    Publish-PackageInternal $dataCollectorProject $TPB_TargetFramework472 $fullDestDir
 
     New-Item -ItemType directory -Path $fullCLRPackageDir -Force | Out-Null
     Copy-Item $testhostFullPackageDir\* $fullCLRPackageDir -Force -recurse
@@ -405,12 +406,22 @@ function Publish-Package
     # Copy Blame Datacollector to Extensions folder.
     $TPB_TargetFrameworkStandard = "netstandard2.0"
     $blameDataCollector = Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.Extensions.BlameDataCollector\bin\$TPB_Configuration"
-    $blameDataCollectorNetFull = Join-Path $blameDataCollector $TPB_TargetFramework
+    $blameDataCollectorNetFull = Join-Path $blameDataCollector $TPB_TargetFramework472
     $blameDataCollectorNetStandard = Join-Path $blameDataCollector $TPB_TargetFrameworkStandard
     Copy-Item $blameDataCollectorNetFull\Microsoft.TestPlatform.Extensions.BlameDataCollector.dll $fullCLRExtensionsDir -Force
     Copy-Item $blameDataCollectorNetFull\Microsoft.TestPlatform.Extensions.BlameDataCollector.pdb $fullCLRExtensionsDir -Force
     Copy-Item $blameDataCollectorNetStandard\Microsoft.TestPlatform.Extensions.BlameDataCollector.dll $coreCLRExtensionsDir -Force
     Copy-Item $blameDataCollectorNetStandard\Microsoft.TestPlatform.Extensions.BlameDataCollector.pdb $coreCLRExtensionsDir -Force
+    # we use this to dump processes on netcore
+    Copy-Item $blameDataCollectorNetStandard\Microsoft.Diagnostics.NETCore.Client.dll $coreCLRExtensionsDir -Force
+
+    # $null = New-Item -Force "$fullCLRExtensionsDir\procdump" -ItemType Directory
+    # $null = New-Item -Force "$coreCLRExtensionsDir\procdump" -ItemType Directory
+    # Copy-Item $blameDataCollectorNetFull\procdump.exe $fullCLRExtensionsDir\procdump -Force
+    # Copy-Item $blameDataCollectorNetFull\procdump64.exe $fullCLRExtensionsDir\procdump -Force
+    # Copy-Item $blameDataCollectorNetStandard\procdump.exe $coreCLRExtensionsDir\procdump -Force
+    # Copy-Item $blameDataCollectorNetStandard\procdump64.exe $coreCLRExtensionsDir\procdump -Force
+    # Copy-Item $blameDataCollectorNetStandard\procdump $coreCLRExtensionsDir\procdump -Force
 
     # Copy blame data collector resource dlls
     if($TPB_LocalizedBuild) {

scripts/build.sh

@@ -93,7 +93,7 @@ done
 #
 TP_ROOT_DIR=$(cd "$(dirname "$0")"; pwd -P)
 TP_TOOLS_DIR="$TP_ROOT_DIR/tools"
-TP_DOTNET_DIR="${DOTNET_CORE_SDK_DIR:-${TP_TOOLS_DIR}/dotnet}"
+TP_DOTNET_DIR="${DOTNET_CORE_SDK_DIR:-${TP_TOOLS_DIR}/dotnet-linux}"
 TP_PACKAGES_DIR="${NUGET_PACKAGES:-${TP_ROOT_DIR}/packages}"
 TP_OUT_DIR="$TP_ROOT_DIR/artifacts"
 TP_PACKAGE_PROJ_DIR="$TP_ROOT_DIR/src/package/package"
@@ -186,12 +186,12 @@ function install_cli()
         chmod u+x $install_script
 
         log "install_cli: Get the latest dotnet cli toolset..."
-        $install_script --install-dir "$TP_TOOLS_DIR/dotnet" --no-path --channel "master" --version $DOTNET_CLI_VERSION
+        $install_script --install-dir "$TP_DOTNET_DIR" --no-path --channel "master" --version $DOTNET_CLI_VERSION
 
         # Get netcoreapp1.1 shared components
-        $install_script --install-dir "$TP_TOOLS_DIR/dotnet" --no-path --channel "release/2.1.0" --version "2.1.0" --shared-runtime
+        $install_script --install-dir "$TP_DOTNET_DIR" --no-path --channel "release/2.1.0" --version "2.1.0" --runtime dotnet
         #log "install_cli: Get shared components which is compatible with dotnet cli version $DOTNET_CLI_VERSION..."
-        #$install_script --install-dir "$TP_TOOLS_DIR/dotnet" --no-path --channel "master" --version $DOTNET_RUNTIME_VERSION --shared-runtime
+        #$install_script --install-dir "$TP_DOTNET_DIR" --no-path --channel "master" --version $DOTNET_RUNTIME_VERSION --runtime dotnet
     fi
 
     local dotnet_path=$(_get_dotnet_path)

scripts/verify-nupkgs.ps1

@@ -16,7 +16,7 @@ function Verify-Nuget-Packages($packageDirectory)
                      "Microsoft.NET.Test.Sdk" = 13;
                      "Microsoft.TestPlatform" = 437;
                      "Microsoft.TestPlatform.Build" = 19;
-                     "Microsoft.TestPlatform.CLI" = 317;
+                     "Microsoft.TestPlatform.CLI" = 318;
                      "Microsoft.TestPlatform.Extensions.TrxLogger" = 33;
                      "Microsoft.TestPlatform.ObjectModel" = 62;
                      "Microsoft.TestPlatform.Portable" = 502;

src/Microsoft.TestPlatform.Extensions.BlameDataCollector/BlameCollector.cs

@@ -12,6 +12,7 @@ namespace Microsoft.TestPlatform.Extensions.BlameDataCollector
     using System.Xml;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection;
+    using Microsoft.VisualStudio.TestPlatform.Utilities;
     using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
     using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;
 
@@ -45,6 +46,8 @@ public class BlameCollector : DataCollector, ITestExecutionEnvironmentSpecifier
         private IInactivityTimer inactivityTimer;
         private TimeSpan inactivityTimespan = TimeSpan.FromMinutes(DefaultInactivityTimeInMinutes);
         private int testHostProcessId;
+        private bool dumpWasCollectedByHangDumper;
+        private string targetFramework;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="BlameCollector"/> class.
@@ -62,7 +65,7 @@ public BlameCollector()
         /// BlameReaderWriter instance.
         /// </param>
         /// <param name="processDumpUtility">
-        /// ProcessDumpUtility instance.
+        /// IProcessDumpUtility instance.
         /// </param>
         /// <param name="inactivityTimer">
         /// InactivityTimer instance.
@@ -138,6 +141,12 @@ public override void Initialize(
                 {
                     this.ValidateAndAddHangBasedProcessDumpParameters(collectHangBasedDumpNode);
                 }
+
+                var tfm = this.configurationElement[Constants.TargetFramework]?.InnerText;
+                if (!string.IsNullOrWhiteSpace(tfm))
+                {
+                    this.targetFramework = tfm;
+                }
             }
 
             this.attachmentGuid = Guid.NewGuid().ToString().Replace("-", string.Empty);
@@ -157,8 +166,10 @@ public override void Initialize(
         /// </summary>
         private void CollectDumpAndAbortTesthost()
         {
-            EqtTrace.Info(string.Format(CultureInfo.CurrentUICulture, Resources.Resources.InactivityTimeout, (int)this.inactivityTimespan.TotalMinutes));
             this.inactivityTimerAlreadyFired = true;
+            var message = string.Format(CultureInfo.CurrentUICulture, Resources.Resources.InactivityTimeout, (int)this.inactivityTimespan.TotalMinutes);
+            EqtTrace.Warning(message);
+            this.logger.LogWarning(this.context.SessionDataCollectionContext, message);
 
             try
             {
@@ -170,27 +181,28 @@ private void CollectDumpAndAbortTesthost()
                 EqtTrace.Verbose("Inactivity timer is already disposed.");
             }
 
-            if (this.collectProcessDumpOnTrigger)
-            {
-                // Detach procdump from the testhost process to prevent testhost process from crashing
-                // if/when we try to kill the existing proc dump process.
-                this.processDumpUtility.DetachFromTargetProcess(this.testHostProcessId);
-            }
-
             try
             {
-                this.processDumpUtility.StartHangBasedProcessDump(this.testHostProcessId, this.attachmentGuid, this.GetResultsDirectory(), this.processFullDumpEnabled);
+                this.processDumpUtility.StartHangBasedProcessDump(this.testHostProcessId, this.attachmentGuid, this.GetTempDirectory(), this.processFullDumpEnabled, this.targetFramework);
             }
             catch (Exception ex)
             {
-                EqtTrace.Error($"BlameCollector.CollectDumpAndAbortTesthost: Failed with error {ex}");
+                ConsoleOutput.Instance.Error(true, $"Blame: Creating hang dump failed with error {ex}.");
+            }
+
+            if (this.collectProcessDumpOnTrigger)
+            {
+                // Detach procdump from the testhost process to prevent testhost process from crashing
+                // if/when we try to kill the existing proc dump process.
+                this.processDumpUtility.DetachFromTargetProcess(this.testHostProcessId);
             }
 
             try
             {
                 var dumpFile = this.processDumpUtility.GetDumpFile();
                 if (!string.IsNullOrEmpty(dumpFile))
                 {
+                    this.dumpWasCollectedByHangDumper = true;
                     var fileTransferInformation = new FileTransferInformation(this.context.SessionDataCollectionContext, dumpFile, true, this.fileHelper);
                     this.dataCollectionSink.SendFileAsync(fileTransferInformation);
                 }
@@ -207,7 +219,17 @@ private void CollectDumpAndAbortTesthost()
 
             try
             {
-                Process.GetProcessById(this.testHostProcessId).Kill();
+                var p = Process.GetProcessById(this.testHostProcessId);
+                try
+                {
+                    if (!p.HasExited)
+                    {
+                        p.Kill();
+                    }
+                }
+                catch (InvalidOperationException)
+                {
+                }
             }
             catch (Exception ex)
             {
@@ -274,7 +296,7 @@ private void ValidateAndAddHangBasedProcessDumpParameters(XmlElement collectDump
 
                         break;
 
-                    case XmlAttribute attribute when string.Equals(attribute.Name, Constants.DumpTypeKey, StringComparison.OrdinalIgnoreCase):
+                    case XmlAttribute attribute when string.Equals(attribute.Name, Constants.HangDumpTypeKey, StringComparison.OrdinalIgnoreCase):
 
                         if (string.Equals(attribute.Value, Constants.FullConfigurationValue, StringComparison.OrdinalIgnoreCase) || string.Equals(attribute.Value, Constants.MiniConfigurationValue, StringComparison.OrdinalIgnoreCase))
                         {
@@ -365,7 +387,8 @@ private void SessionEndedHandler(object sender, SessionEndEventArgs args)
                 // And send the attachment
                 if (this.testStartCount > this.testEndCount)
                 {
-                    var filepath = Path.Combine(this.GetResultsDirectory(), Constants.AttachmentFileName + "_" + this.attachmentGuid);
+                    var filepath = Path.Combine(this.GetTempDirectory(), Constants.AttachmentFileName + "_" + this.attachmentGuid);
+
                     filepath = this.blameReaderWriter.WriteTestSequence(this.testSequence, this.testObjectDictionary, filepath);
                     var fileTranferInformation = new FileTransferInformation(this.context.SessionDataCollectionContext, filepath, true);
                     this.dataCollectionSink.SendFileAsync(fileTranferInformation);
@@ -374,7 +397,10 @@ private void SessionEndedHandler(object sender, SessionEndEventArgs args)
                 if (this.collectProcessDumpOnTrigger)
                 {
                     // If there was a test case crash or if we need to collect dump on process exit.
-                    if (this.testStartCount > this.testEndCount || this.collectDumpAlways)
+                    //
+                    // Do not try to collect dump when we already collected one from the hang dump
+                    // we won't dump the killed process again and that would just show a warning on the command line
+                    if ((this.testStartCount > this.testEndCount || this.collectDumpAlways) && !this.dumpWasCollectedByHangDumper)
                     {
                         try
                         {
@@ -404,7 +430,6 @@ private void SessionEndedHandler(object sender, SessionEndEventArgs args)
                 if (this.collectProcessDumpOnTrigger)
                 {
                     this.processDumpUtility.DetachFromTargetProcess(this.testHostProcessId);
-                    this.processDumpUtility.TerminateProcess();
                 }
 
                 this.DeregisterEvents();
@@ -428,7 +453,7 @@ private void TestHostLaunchedHandler(object sender, TestHostLaunchedEventArgs ar
 
             try
             {
-                this.processDumpUtility.StartTriggerBasedProcessDump(args.TestHostProcessId, this.attachmentGuid, this.GetResultsDirectory(), this.processFullDumpEnabled);
+                this.processDumpUtility.StartTriggerBasedProcessDump(args.TestHostProcessId, this.attachmentGuid, this.GetTempDirectory(), this.processFullDumpEnabled, ".NETFramework,Version=v4.0");
             }
             catch (TestPlatformException e)
             {
@@ -481,24 +506,15 @@ private void DeregisterEvents()
             this.events.TestCaseEnd -= this.EventsTestCaseEnd;
         }
 
-        private string GetResultsDirectory()
+        private string GetTempDirectory()
         {
-            try
+            var tmp = Path.GetTempPath();
+            if (!Directory.Exists(tmp))
             {
-                XmlElement resultsDirectoryElement = this.configurationElement["ResultsDirectory"];
-                string resultsDirectory = resultsDirectoryElement != null ? resultsDirectoryElement.InnerText : string.Empty;
-
-                return Environment.ExpandEnvironmentVariables(resultsDirectory);
+                Directory.CreateDirectory(tmp);
             }
-            catch (NullReferenceException exception)
-            {
-                if (EqtTrace.IsErrorEnabled)
-                {
-                    EqtTrace.Error("Blame Collector : " + exception);
-                }
 
-                return string.Empty;
-            }
+            return tmp;
         }
     }
 }

src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Constants.cs

@@ -53,6 +53,11 @@ internal static class Constants
         /// </summary>
         public const string DumpModeKey = "CollectDump";
 
+        /// <summary>
+        /// Configuration key name for hang dump mode
+        /// </summary>
+        public const string HangDumpModeKey = "CollectHangDump";
+
         /// <summary>
         /// Proc dump 32 bit version
         /// </summary>
@@ -63,6 +68,11 @@ internal static class Constants
         /// </summary>
         public const string Procdump64Process = "procdump64.exe";
 
+        /// <summary>
+        /// Proc dump 64 bit version
+        /// </summary>
+        public const string ProcdumpUnixProcess = "procdump";
+
         /// <summary>
         /// Configuration key name for collect dump always
         /// </summary>
@@ -85,6 +95,11 @@ internal static class Constants
         /// </summary>
         public const string DumpTypeKey = "DumpType";
 
+        /// <summary>
+        /// Configuration key name for hang dump type
+        /// </summary>
+        public const string HangDumpTypeKey = "HangDumpType";
+
         /// <summary>
         /// Configuration value for true
         /// </summary>
@@ -104,5 +119,10 @@ internal static class Constants
         /// Configuration value for mini
         /// </summary>
         public const string MiniConfigurationValue = "Mini";
+
+        /// <summary>
+        /// The target framework of test host.
+        /// </summary>
+        public const string TargetFramework = "Framework";
     }
 }

src/Microsoft.TestPlatform.Extensions.BlameDataCollector/CrashDumperFactory.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.Extensions.BlameDataCollector
+{
+    using System;
+    using System.Runtime.InteropServices;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+
+    internal class CrashDumperFactory : ICrashDumperFactory
+    {
+        public ICrashDumper Create(string targetFramework)
+        {
+            EqtTrace.Info($"CrashDumperFactory: Creating dumper for {RuntimeInformation.OSDescription} with target framework {targetFramework}.");
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                EqtTrace.Info($"CrashDumperFactory: This is Windows, returning ProcDumpCrashDumper that uses ProcDump utility.");
+                return new ProcDumpCrashDumper();
+            }
+
+            throw new PlatformNotSupportedException($"Unsupported operating system: {RuntimeInformation.OSDescription}");
+        }
+    }
+}

src/Microsoft.TestPlatform.Extensions.BlameDataCollector/DumpTypeOption.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.Extensions.BlameDataCollector
+{
+    public enum DumpTypeOption
+    {
+        Full,
+        WithHeap,
+        Mini,
+    }
+}