Rebuild Microsoft.CodeAnalysis (#50968)

Co-authored-by: Jared Parsons <jaredpparsons@gmail.com>

.vscode/launch.json

@@ -4,6 +4,22 @@
   // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Launch BuildValidator.dll",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "build",
+      // If you have changed target frameworks, make sure to update the program path.
+      "program": "${workspaceFolder}/artifacts/bin/BuildValidator/Debug/netcoreapp3.1/BuildValidator.dll",
+      "args": [
+        "--assembliesPath", "./artifacts/obj/RunTests",
+        "--debugPath", "./artifacts/BuildValidator",
+        "--sourcePath", "."
+      ],
+      "cwd": "${workspaceFolder}",
+      "stopAtEntry": false,
+      "console": "internalConsole"
+    },
     {
       "name": "Launch RunTests.dll",
       "type": "coreclr",

azure-pipelines.yml

@@ -213,3 +213,38 @@ jobs:
       continueOnError: true
       condition: not(succeeded())
 
+- job: Correctness_Rebuild
+  pool:
+    vmImage: windows-2019
+  timeoutInMinutes: 90
+  steps:
+    - template: eng/pipelines/checkout-windows-task.yml
+
+    - task: PowerShell@2
+      displayName: Restore
+      inputs:
+        filePath: eng/build.ps1
+        arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog
+
+    - task: PowerShell@2 
+      displayName: Build
+      inputs:
+        filePath: eng/build.ps1
+        arguments: -configuration Debug -prepareMachine -ci -build -bootstrap -publish -binaryLog -skipDocumentation
+
+    - script: .\artifacts\bin\BuildValidator\Debug\net472\BuildValidator.exe --assembliesPath .\artifacts\obj\Microsoft.CodeAnalysis --debugPath .\artifacts\BuildValidator --sourcePath .
+      displayName: Run BuildValidator
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish BuildValidator debug outputs
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)/artifacts/BuildValidator'
+        ArtifactName: 'BuildValidator_DebugOut'
+        publishLocation: Container
+      continueOnError: true
+      condition: failed()
+
+    - template: eng/pipelines/publish-logs.yml
+      parameters:
+        jobName: Correctness_Rebuild
+        configuration: Debug

eng/Versions.props

@@ -96,7 +96,7 @@
     <MicrosoftIdentityModelClientsActiveDirectoryVersion>3.13.8</MicrosoftIdentityModelClientsActiveDirectoryVersion>
     <MicrosoftInternalPerformanceCodeMarkersDesignTimeVersion>15.8.27812-alpha</MicrosoftInternalPerformanceCodeMarkersDesignTimeVersion>
     <MicrosoftInternalVisualStudioShellInterop140DesignTimeVersion>14.3.25407-alpha</MicrosoftInternalVisualStudioShellInterop140DesignTimeVersion>
-    <MicrosoftMetadataVisualizerVersion>1.0.0-beta3.20174.1</MicrosoftMetadataVisualizerVersion>
+    <MicrosoftMetadataVisualizerVersion>1.0.0-beta3.21075.2</MicrosoftMetadataVisualizerVersion>
     <MicrosoftMSXMLVersion>8.0.0</MicrosoftMSXMLVersion>
     <MicrosoftNETBuildExtensionsVersion>2.2.101</MicrosoftNETBuildExtensionsVersion>
     <MicrosoftNETCorePlatformsVersion>2.1.2</MicrosoftNETCorePlatformsVersion>
@@ -205,6 +205,7 @@
     <SystemBuffersVersion>4.5.1</SystemBuffersVersion>
     <SystemCompositionVersion>1.0.31</SystemCompositionVersion>
     <SystemCodeDomVersion>4.7.0</SystemCodeDomVersion>
+    <SystemCommandLineVersion>2.0.0-beta1.20574.7</SystemCommandLineVersion>
     <SystemCommandLineExperimentalVersion>0.3.0-alpha.19577.1</SystemCommandLineExperimentalVersion>
     <SystemComponentModelCompositionVersion>4.5.0</SystemComponentModelCompositionVersion>
     <SystemDrawingCommonVersion>4.5.0</SystemDrawingCommonVersion>

eng/build-utils.ps1

@@ -333,7 +333,7 @@ function Make-BootstrapBuild([switch]$force32 = $false) {
 
   Run-MSBuild $projectPath "/restore /t:Pack /p:RoslynEnforceCodeStyle=false /p:RunAnalyzersDuringBuild=false /p:DotNetUseShippingVersions=true /p:InitialDefineConstants=BOOTSTRAP /p:PackageOutputPath=`"$dir`" /p:EnableNgenOptimization=false /p:PublishWindowsPdb=false $force32Flag" -logFileName "Bootstrap" -configuration $bootstrapConfiguration -runAnalyzers
   $packageFile = Get-ChildItem -Path $dir -Filter "$packageName.*.nupkg"
-  Unzip "$dir\$packageFile" $dir
+  Unzip (Join-Path $dir $packageFile) $dir
 
   Write-Host "Cleaning Bootstrap compiler artifacts"
   Run-MSBuild $projectPath "/t:Clean" -logFileName "BootstrapClean"

eng/test-rebuild.ps1

@@ -0,0 +1,43 @@
+<#
+  This script tests that Roslyn artifacts are rebuildable--i.e. that the source code and resources can be identified 
+#>
+
+[CmdletBinding(PositionalBinding=$false)]
+param(
+  [string]$configuration = "Debug",
+  [switch]$ci = $false,
+  [switch]$help)
+
+Set-StrictMode -version 2.0
+$ErrorActionPreference="Stop"
+
+function Print-Usage() {
+  Write-Host "Usage: test-rebuild.ps1"
+  Write-Host "  -configuration            Build configuration ('Debug' or 'Release')"
+  Write-Host "  -ci                       Set when running on CI server"
+  Write-Host "  -help                     Print help and exit"
+}
+
+try {
+  if ($help) {
+    Print-Usage
+    exit 0
+  }
+
+  . (Join-Path $PSScriptRoot "build-utils.ps1")
+  Push-Location $RepoRoot
+
+  Write-Host "Building Roslyn"
+  Exec-Console (Join-Path $PSScriptRoot "build.ps1") "-restore -build -ci:$ci -configuration:$configuration -pack -binaryLog"
+  Exec-Console "artifacts\bin\BuildValidator\$configuration\net472\BuildValidator.exe" "--assembliesPath '$ArtifactsDir/obj/Microsoft.CodeAnalysis'"
+
+  exit 0
+}
+catch [exception] {
+  Write-Host $_
+  Write-Host $_.Exception
+  exit 1
+}
+finally {
+  Pop-Location
+}

src/Compilers/CSharp/Test/Emit/PDB/PDBAsyncTests.cs

@@ -2016,11 +2016,13 @@ async partial void M() {}
 
             AssertEx.AssertEqualToleratingWhitespaceDifferences(@"
 MethodDebugInformation (index: 0x31, size: 20): 
-==================================================
-1: nil
-2: nil
-3: nil
-4:
+================================================
+   IL   
+================================================
+1: nil  
+2: nil  
+3: nil  
+4:      
 {
   Kickoff Method: 0x06000001 (MethodDef)
   Locals: 0x11000002 (StandAloneSig)

src/Compilers/CSharp/Test/Emit/PDB/PDBIteratorTests.cs

@@ -1599,12 +1599,14 @@ public class C
 
             AssertEx.AssertEqualToleratingWhitespaceDifferences(@"
 MethodDebugInformation (index: 0x31, size: 40): 
-==================================================
-1: nil
-2: nil
-3: nil
-4: nil
-5:
+================================================
+   IL   
+================================================
+1: nil  
+2: nil  
+3: nil  
+4: nil  
+5:      
 {
   Kickoff Method: 0x06000001 (MethodDef)
   Locals: 0x11000001 (StandAloneSig)
@@ -1615,12 +1617,11 @@ public class C
   IL_0030: <hidden>
   IL_0037: (5, 44) - (5, 45)
 }
-6: nil
-7: nil
-8: nil
-9: nil
-a: nil
-",
+6: nil  
+7: nil  
+8: nil  
+9: nil  
+a: nil",
                 writer.ToString());
         }
     }

src/Compilers/CSharp/Test/Emit/PDB/PDBSourceLinkTests.cs

@@ -154,6 +154,7 @@ public static void Main()
                 debugEntryPoint: null,
                 sourceLinkStream: new MemoryStream(new byte[] { 1, 2, 3 }),
                 embeddedTexts: null,
+                pdbOptionsBlobReader: null,
                 testData: new CompilationTestData()
                 {
                     SymWriterFactory = metadataProvider => new SymUnmanagedWriterWithoutSourceLinkSupport(metadataProvider)

src/Compilers/CSharp/Test/Emit/PDB/PDBTests.cs

@@ -188,6 +188,7 @@ public void SymWriterErrors()
                 debugEntryPoint: null,
                 sourceLinkStream: null,
                 embeddedTexts: null,
+                pdbOptionsBlobReader: null,
                 testData: new CompilationTestData() { SymWriterFactory = _ => new MockSymUnmanagedWriter() });
 
             result.Diagnostics.Verify(
@@ -219,6 +220,7 @@ public void SymWriterErrors2()
                 debugEntryPoint: null,
                 sourceLinkStream: null,
                 embeddedTexts: null,
+                pdbOptionsBlobReader: null,
                 testData: new CompilationTestData() { SymWriterFactory = SymWriterTestUtilities.ThrowingFactory });
 
             result.Diagnostics.Verify(
@@ -250,6 +252,7 @@ public void SymWriterErrors3()
                 debugEntryPoint: null,
                 sourceLinkStream: null,
                 embeddedTexts: null,
+                pdbOptionsBlobReader: null,
                 testData: new CompilationTestData() { SymWriterFactory = SymWriterTestUtilities.ThrowingFactory });
 
             result.Diagnostics.Verify(
@@ -281,6 +284,7 @@ public void SymWriterErrors4()
                 debugEntryPoint: null,
                 sourceLinkStream: null,
                 embeddedTexts: null,
+                pdbOptionsBlobReader: null,
                 testData: new CompilationTestData() { SymWriterFactory = _ => throw new DllNotFoundException("xxx") });
 
             result.Diagnostics.Verify(

src/Compilers/Core/Portable/CommandLine/CommonCompiler.cs

@@ -1266,6 +1266,7 @@ private void CompileAndEmit(
                             peStreamProvider,
                             refPeStreamProviderOpt,
                             pdbStreamProviderOpt,
+                            pdbOptionsBlobReader: null,
                             testSymWriterFactory: null,
                             diagnostics: diagnostics,
                             emitOptions: emitOptions,

src/Compilers/Core/Portable/Compilation/Compilation.cs

@@ -2454,6 +2454,35 @@ public EmitResult Emit(
             IEnumerable<EmbeddedText>? embeddedTexts = null,
             Stream? metadataPEStream = null,
             CancellationToken cancellationToken = default(CancellationToken))
+        {
+            return Emit(
+                peStream,
+                pdbStream,
+                xmlDocumentationStream,
+                win32Resources,
+                manifestResources,
+                options,
+                debugEntryPoint,
+                sourceLinkStream,
+                embeddedTexts,
+                metadataPEStream,
+                pdbOptionsBlobReader: null,
+                cancellationToken);
+        }
+
+        internal EmitResult Emit(
+            Stream peStream,
+            Stream? pdbStream,
+            Stream? xmlDocumentationStream,
+            Stream? win32Resources,
+            IEnumerable<ResourceDescription>? manifestResources,
+            EmitOptions? options,
+            IMethodSymbol? debugEntryPoint,
+            Stream? sourceLinkStream,
+            IEnumerable<EmbeddedText>? embeddedTexts,
+            Stream? metadataPEStream,
+            BlobReader? pdbOptionsBlobReader,
+            CancellationToken cancellationToken)
         {
             if (peStream == null)
             {
@@ -2549,6 +2578,7 @@ public EmitResult Emit(
                 debugEntryPoint,
                 sourceLinkStream,
                 embeddedTexts,
+                pdbOptionsBlobReader,
                 testData: null,
                 cancellationToken: cancellationToken);
         }
@@ -2568,6 +2598,7 @@ internal EmitResult Emit(
             IMethodSymbol? debugEntryPoint,
             Stream? sourceLinkStream,
             IEnumerable<EmbeddedText>? embeddedTexts,
+            BlobReader? pdbOptionsBlobReader,
             CompilationTestData? testData,
             CancellationToken cancellationToken)
         {
@@ -2648,6 +2679,7 @@ internal EmitResult Emit(
                         new SimpleEmitStreamProvider(peStream),
                         (metadataPEStream != null) ? new SimpleEmitStreamProvider(metadataPEStream) : null,
                         (pdbStream != null) ? new SimpleEmitStreamProvider(pdbStream) : null,
+                        pdbOptionsBlobReader,
                         testData?.SymWriterFactory,
                         diagnostics,
                         emitOptions: options,
@@ -2809,6 +2841,7 @@ internal bool SerializeToPeStream(
             EmitStreamProvider peStreamProvider,
             EmitStreamProvider? metadataPEStreamProvider,
             EmitStreamProvider? pdbStreamProvider,
+            BlobReader? pdbOptionsBlobReader,
             Func<ISymWriterMetadataProvider, SymUnmanagedWriter>? testSymWriterFactory,
             DiagnosticBag diagnostics,
             EmitOptions emitOptions,
@@ -2882,6 +2915,7 @@ internal bool SerializeToPeStream(
                         getPortablePdbStream,
                         nativePdbWriter,
                         pePdbFilePath,
+                        pdbOptionsBlobReader,
                         emitOptions.EmitMetadataOnly,
                         emitOptions.IncludePrivateMembers,
                         deterministic,
@@ -2963,6 +2997,7 @@ internal static bool SerializePeToStream(
             Func<Stream?>? getPortablePdbStreamOpt,
             Cci.PdbWriter? nativePdbWriterOpt,
             string? pdbPathOpt,
+            BlobReader? pdbOptionsBlobReader,
             bool metadataOnly,
             bool includePrivateMembers,
             bool isDeterministic,
@@ -2980,6 +3015,7 @@ internal static bool SerializePeToStream(
                 getPeStream,
                 getPortablePdbStreamOpt,
                 nativePdbWriterOpt,
+                pdbOptionsBlobReader,
                 pdbPathOpt,
                 metadataOnly,
                 deterministicPrimaryOutput,
@@ -3002,6 +3038,7 @@ internal static bool SerializePeToStream(
                     getMetadataPeStreamOpt,
                     getPortablePdbStreamOpt: null,
                     nativePdbWriterOpt: null,
+                    pdbOptionsBlobReader: null,
                     pdbPathOpt: null,
                     metadataOnly: true,
                     isDeterministic: true,
@@ -3057,6 +3094,7 @@ internal static bool SerializePeToStream(
                         metadataStream,
                         ilStream,
                         (nativePdbWriterOpt == null) ? pdbStream : null,
+                        pdbOptionsBlobReader: null,
                         out MetadataSizes metadataSizes);
 
                     writer.GetMethodTokens(updatedMethods);

src/Compilers/Core/Portable/Compilation/CompilationOptions.cs

@@ -166,7 +166,7 @@ public abstract class CompilationOptions
         /// <summary>
         /// Emit mode that favors debuggability. 
         /// </summary>
-        internal bool DebugPlusMode { get; private protected set; }
+        internal bool DebugPlusMode { get; set; }
 
         /// <summary>
         /// Specifies whether to import members with accessibility other than public or protected by default. 

src/Compilers/Core/Portable/Microsoft.CodeAnalysis.csproj

@@ -11,7 +11,8 @@
     <NoStdLib>true</NoStdLib>
     <ApplyNgenOptimization Condition="'$(TargetFramework)' == 'netstandard2.0'">partial</ApplyNgenOptimization>
     <GeneratePerformanceSensitiveAttribute>true</GeneratePerformanceSensitiveAttribute>
-
+    <EmbedUntrackedSources>true</EmbedUntrackedSources>
+
     <!-- NuGet -->
     <IsPackable>true</IsPackable>
     <PackageId>Microsoft.CodeAnalysis.Common</PackageId>
@@ -51,6 +52,7 @@
     <InternalsVisibleTo Include="Microsoft.CodeAnalysis.VisualBasic.ExpressionEvaluator.ExpressionCompiler" />
     <InternalsVisibleTo Include="csc" />
     <InternalsVisibleTo Include="csi" />
+    <InternalsVisibleTo Include="BuildValidator" />
     <InternalsVisibleTo Include="Microsoft.CodeAnalysis.Scripting" />
     <InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Scripting" />
     <InternalsVisibleTo Include="Microsoft.CodeAnalysis.VisualBasic.Scripting" />