Initial provisions for semantic Crossgen2 PDB validation

eng/Subsets.props

@@ -338,6 +338,7 @@
                              $(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo.csproj;
                              $(CoreClrProjectRoot)tools\aot\ILCompiler\repro\repro.csproj;
                              $(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj;
+                             $(CoreClrProjectRoot)tools\PdbChecker\PdbChecker.csproj;
                              $(CoreClrProjectRoot)tools\AssemblyChecker\AssemblyChecker.csproj" Category="clr" Condition="'$(DotNetBuildFromSource)' != 'true'"/>
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" />
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler.Build.Tasks\ILCompiler.Build.Tasks.csproj" Category="clr" Condition="'$(NativeAotSupported)' == 'true'" />

src/coreclr/tools/PdbChecker/MSDiaSymbolReader.cs

@@ -0,0 +1,81 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using Dia2Lib;
+
+class MSDiaSymbolReader
+{
+    [return: MarshalAs(UnmanagedType.Interface)]
+    [DllImport("msdia140.dll", CharSet = CharSet.Unicode, ExactSpelling = true, PreserveSig = false)]
+    private static extern object DllGetClassObject(
+        [In] in Guid rclsid,
+        [In] in Guid riid);
+
+    [ComImport, ComVisible(false), Guid("00000001-0000-0000-C000-000000000046"),
+        InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
+    private interface IClassFactory
+    {
+        void CreateInstance([MarshalAs(UnmanagedType.Interface)] object? aggregator,
+                            [In] in Guid refiid,
+                            [MarshalAs(UnmanagedType.Interface)] out object createdObject);
+        void LockServer(bool incrementRefCount);
+    }
+
+    private readonly IDiaDataSource _diaDataSource;
+    private readonly IDiaSession _diaSession;
+
+    private readonly List<string> _pdbSymbols;
+
+    public MSDiaSymbolReader(string pdbFile)
+    {
+        try
+        {
+            var dia140SourceClassGuid = new Guid("{e6756135-1e65-4d17-8576-610761398c3c}");
+            IClassFactory diaClassFactory = (IClassFactory)DllGetClassObject(dia140SourceClassGuid, typeof(IClassFactory).GetTypeInfo().GUID);
+            diaClassFactory.CreateInstance(null, typeof(IDiaDataSource).GetTypeInfo().GUID, out object comObject);
+
+            _diaDataSource = (IDiaDataSource)comObject;
+            _diaDataSource.loadDataFromPdb(pdbFile);
+            _diaDataSource.openSession(out _diaSession);
+
+            _pdbSymbols = new List<string>();
+
+            _diaSession.getSymbolsByAddr(out IDiaEnumSymbolsByAddr symbolEnum);
+            int symbolsTotal = 0;
+            for (IDiaSymbol symbol = symbolEnum.symbolByRVA(0); symbol != null; symbolEnum.Next(1, out symbol, out uint fetched))
+            {
+                symbolsTotal++;
+                if (symbol.symTag == (uint)SymTagEnum.SymTagFunction || symbol.symTag == (uint)SymTagEnum.SymTagPublicSymbol)
+                {
+                    _pdbSymbols.Add(symbol.name);
+                }
+            }
+
+            Console.WriteLine("PDB file:       {0}", pdbFile);
+            Console.WriteLine("Total symbols:  {0}", symbolsTotal);
+            Console.WriteLine("Public symbols: {0}", _pdbSymbols.Count);
+        }
+        catch (Exception ex)
+        {
+            throw new Exception($"Error opening PDB file {pdbFile}", ex);
+        }
+    }
+
+    public void DumpSymbols()
+    {
+        Console.WriteLine("PDB public symbol list:");
+        foreach (string symbol in _pdbSymbols.OrderBy(s => s))
+        {
+            Console.WriteLine(symbol);
+        }
+        Console.WriteLine("End of PDB public symbol list");
+    }
+
+    public bool ContainsSymbol(string symbolName) => _pdbSymbols.Any(s => s.Contains(symbolName));
+}

src/coreclr/tools/PdbChecker/PdbChecker.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <AssemblyName>PdbChecker</AssemblyName>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>$(NetCoreAppToolCurrent)</TargetFramework>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+    <AppendTargetFrameworkToOutputPath Condition="'$(BuildingInsideVisualStudio)' == 'true'">true</AppendTargetFrameworkToOutputPath>
+    <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
+    <Nullable>enable</Nullable>
+    <OutputPath>$(RuntimeBinDir)\PdbChecker</OutputPath>
+    <RunAnalyzers>false</RunAnalyzers>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="$(TraceEventVersion)" />
+  </ItemGroup>
+
+  <Target Name="CopyMsdiaForTargetArchitecture" AfterTargets="AfterBuild" Condition="'$(TargetOS)' == 'windows'">
+    <PropertyGroup>
+      <ArchitectureFolder>$(TargetArchitecture)</ArchitectureFolder>
+      <ArchitectureFolder Condition="'$(TargetArchitecture)' == 'x64'">amd64</ArchitectureFolder>
+    </PropertyGroup>
+
+    <Copy SourceFiles="$(OutputPath)\$(ArchitectureFolder)\msdia140.dll" DestinationFolder="$(OutputPath)" />
+  </Target>
+</Project>

src/coreclr/tools/PdbChecker/PdbChecker.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32708.82
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PdbChecker", "PdbChecker.csproj", "{6247A503-5387-4BE1-ACA3-027CADA30CA9}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{6247A503-5387-4BE1-ACA3-027CADA30CA9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6247A503-5387-4BE1-ACA3-027CADA30CA9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6247A503-5387-4BE1-ACA3-027CADA30CA9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6247A503-5387-4BE1-ACA3-027CADA30CA9}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {4033231C-763B-4C57-BA35-7C1AC007AD0E}
+	EndGlobalSection
+EndGlobal

src/coreclr/tools/PdbChecker/Program.cs

@@ -0,0 +1,60 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using Dia2Lib;
+class Program
+{
+    public static int Main(string[] args)
+    {
+        try
+        {
+            TryMain(args);
+            return 0;
+        }
+        catch (Exception ex)
+        {
+            Console.Error.WriteLine("Fatal error: {0}", ex);
+            return 1;
+        }
+    }
+
+    private static void TryMain(string[] args)
+    {
+        if (args.Length == 0)
+        {
+            DisplayUsage();
+            return;
+        }
+        MSDiaSymbolReader reader = new MSDiaSymbolReader(args[0]);
+        int matchedSymbols = 0;
+        int missingSymbols = 0;
+        for (int symbolArgIndex = 1; symbolArgIndex < args.Length; symbolArgIndex++)
+        {
+            string symbolName = args[symbolArgIndex];
+            if (reader.ContainsSymbol(symbolName))
+            {
+                matchedSymbols++;
+            }
+            else
+            {
+                missingSymbols++;
+                Console.Error.WriteLine("Missing symbol: {0}", symbolName);
+            }
+        }
+        if (missingSymbols > 0)
+        {
+            reader.DumpSymbols();
+            throw new Exception($"{missingSymbols} missing symbols ({matchedSymbols} symbols matched)");
+        }
+        if (matchedSymbols > 0)
+        {
+            Console.WriteLine("Matched all {0} symbols", matchedSymbols);
+        }
+    }
+
+    private static void DisplayUsage()
+    {
+        Console.WriteLine("Usage: PdbChecker <pdb file to check> { <symbol to check for existence in the PDB file> }");
+    }
+}

src/tests/Common/CLRTest.CrossGen.targets

@@ -189,8 +189,12 @@ if /i "$(AlwaysUseCrossGen2)" == "true" (
 REM CrossGen2 Script
 if defined RunCrossGen2 (
     set ExtraCrossGen2Args=!ExtraCrossGen2Args! $(CrossGen2TestExtraArguments)
+    set CrossGen2TestCheckPdb=$(CrossGen2TestCheckPdb)
+    set __CreatePdb=$(__CreatePdb)
 
     if defined LargeVersionBubble ( set ExtraCrossGen2Args=!ExtraCrossGen2Args! --inputbubble)
+    if defined CrossGen2TestCheckPdb ( set __CreatePdb=1)
+
     set CrossGen2Status=0
     set R2RDumpStatus=0
     set compilationDoneFlagFile=!ScriptPath!IL-CG2\done
@@ -214,6 +218,7 @@ if defined RunCrossGen2 (
     if defined CompositeBuildMode (
         set ExtraCrossGen2Args=!ExtraCrossGen2Args! --composite
         set __OutputFile=!scriptPath!\composite-r2r.dll
+        set __PdbFile=!scriptPath!\composite-r2r.ni.pdb
         rem In composite mode, treat all dll's in the test folder as rooting inputs
         set __InputFile=!scriptPath!IL-CG2\*.dll
         call :CompileOneFileCrossgen2
@@ -222,6 +227,7 @@ if defined RunCrossGen2 (
         set ExtraCrossGen2Args=!ExtraCrossGen2Args! -r:!scriptPath!IL-CG2\*.dll
         for %%I in (!scriptPath!IL-CG2\*.dll) do (
             set __OutputFile=!scriptPath!%%~nI.dll
+            set __PdbFile=!scriptPath!%%~nI.ni.pdb
             set __InputFile=%%I
             call :CompileOneFileCrossgen2
             IF NOT !CrossGen2Status!==0 (
@@ -238,13 +244,14 @@ if defined RunCrossGen2 (
     set __ResponseFile=!__OutputFile!.rsp
     del /Q !__ResponseFile! 2>nul
 
-    set __Command=!_DebuggerFullPath!
     REM Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
     if defined __TestDotNetCmd (
-        set __Command=!__Command! "!__TestDotNetCmd!"
+        set __DotNet="!__TestDotNetCmd!"
     ) else (
-        set __Command=!__Command! "dotnet"
+        set __DotNet="dotnet"
     )
+    set __Command=!_DebuggerFullPath!
+    set __Command=!__Command! !__DotNet!
     set __R2RDumpCommand=!__Command! "!CORE_ROOT!\r2rdump\r2rdump.dll"
     set __R2RDumpCommand=!__R2RDumpCommand! --header --sc --in !__OutputFile! --out !__OutputFile!.r2rdump --val
     set __Command=!__Command! "!CORE_ROOT!\crossgen2\crossgen2.dll"
@@ -269,7 +276,7 @@ if defined RunCrossGen2 (
     echo -r:!CORE_ROOT!\mscorlib.dll>>!__ResponseFile!
     echo -r:!CORE_ROOT!\netstandard.dll>>!__ResponseFile!
 
-    if not "$(__CreatePdb)" == "" (
+    if defined __CreatePdb (
         echo --pdb>>!__ResponseFile!
     )
 
@@ -303,6 +310,21 @@ if defined RunCrossGen2 (
     endlocal & set "R2RDumpStatus=%R2RDumpStatus%" &set "CrossGen2Status=%CrossGen2Status%"
 
     echo %time%
+
+    if !CrossGen2Status!==0 (
+        if defined CrossGen2TestCheckPdb (
+            set __CheckPdbCommand=!__DotNet!
+            set __CheckPdbCommand=!__CheckPdbCommand! "!CORE_ROOT!\PdbChecker\PdbChecker.dll"
+            set __CheckPdbCommand=!__CheckPdbCommand! !__PdbFile! @(CheckPdbSymbol->'%22%(Identity)%22', ' ')
+            echo "!__CheckPdbCommand!"
+            call !__CheckPdbCommand!
+            if not !ERRORLEVEL!==0 (
+                echo PDB check failed for file !__PdbFile! >2
+                set CrossGen2Status=42
+            )
+        )
+    )
+
     Exit /b 0
 
 :DoneCrossgen2Operations

src/tests/Common/Directory.Build.targets

@@ -108,6 +108,9 @@
         <IncludeSubFolders>True</IncludeSubFolders>
       </RunTimeArtifactsIncludeFolders>
 
+      <!-- Used by tests checking PDB validity -->
+      <RunTimeArtifactsIncludeFolders Include="PdbChecker/" TargetDir="PdbChecker/" />
+
       <!-- Used by disasm output verification tests -->
       <RunTimeArtifactsIncludeFolders Include="SuperFileCheck/" TargetDir="SuperFileCheck/">
         <IncludeSubFolders>True</IncludeSubFolders>

src/tests/readytorun/crossgen2/crossgen2smoke.csproj

@@ -5,6 +5,13 @@
 
     <!-- This is an explicit crossgen test -->
     <AlwaysUseCrossGen2>true</AlwaysUseCrossGen2>
+    <CrossGen2TestCheckPdb>true</CrossGen2TestCheckPdb>
   </PropertyGroup>
+
+  <ItemGroup>
+    <CheckPdbSymbol Include="System.Int32 Program+SomeClass::GetHashCode()" />
+    <CheckPdbSymbol Include="System.String Program::ObjectToStringOnGenericParamTestWorker(System.__Canon&amp;)" />
+  </ItemGroup>
+
   <Import Project="smoketest.targets" />
 </Project>