From 12a4808ca5719ec087c2ab3fa42dac4048b3c556 Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Fri, 19 Feb 2021 12:41:00 -0800 Subject: [PATCH 1/8] Rebuild csc --- .vscode/launch.json | 3 +- azure-pipelines.yml | 2 +- eng/Versions.props | 5 +- eng/test-rebuild.ps1 | 16 ++++- src/Compilers/CSharp/csc/csc.csproj | 1 + src/Tools/BuildValidator/BuildConstructor.cs | 33 +++++----- .../BuildValidator/BuildValidator.csproj | 1 + src/Tools/BuildValidator/CompilationDiff.cs | 60 +++++++++++++++++-- .../CompilationOptionsReader.cs | 18 ++++-- .../BuildValidator/LocalReferenceResolver.cs | 2 +- src/Tools/BuildValidator/Program.cs | 19 +++--- src/Tools/ManifestGenerator/README.md | 3 +- 12 files changed, 115 insertions(+), 48 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ae371f1da31ef..dedb274341bc5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,7 +12,8 @@ // If you have changed target frameworks, make sure to update the program path. "program": "${workspaceFolder}/artifacts/bin/BuildValidator/Debug/netcoreapp3.1/BuildValidator.dll", "args": [ - "--assembliesPath", "./artifacts/obj/RunTests", + "--assembliesPath", "./artifacts/obj/csc/Debug/net472", + "--referencesPaths", "./artifacts/bin", "--debugPath", "./artifacts/BuildValidator", "--sourcePath", "." ], diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bb28e7fecfffc..b005b4ee6bcb0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -232,7 +232,7 @@ jobs: filePath: eng/build.ps1 arguments: -configuration Debug -prepareMachine -ci -build -bootstrap -publish -binaryLog -skipDocumentation - - script: .\artifacts\bin\BuildValidator\Debug\net472\BuildValidator.exe --assembliesPath .\artifacts\obj\Microsoft.CodeAnalysis --debugPath .\artifacts\BuildValidator --sourcePath . + - powershell: .\eng\test-rebuild.ps1 displayName: Run BuildValidator - task: PublishBuildArtifacts@1 diff --git a/eng/Versions.props b/eng/Versions.props index 232567d6adb1d..4f0a00a774765 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -90,9 +90,8 @@ 16.9.0-beta1.21055.5 1.5.0 3.0.0-preview1-03617-02 - 2.1.1 - 2.1.1 - 2.1.1 + 5.0.0 + 5.0.0 3.13.8 15.8.27812-alpha 14.3.25407-alpha diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index 10b83f7ce4a73..76c56eeb2555d 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -6,6 +6,7 @@ param( [string]$configuration = "Debug", [switch]$ci = $false, + [switch]$noBuild = $false, [switch]$help) Set-StrictMode -version 2.0 @@ -15,6 +16,7 @@ function Print-Usage() { Write-Host "Usage: test-rebuild.ps1" Write-Host " -configuration Build configuration ('Debug' or 'Release')" Write-Host " -ci Set when running on CI server" + Write-Host " -noBuild If set, skips building before running the rebuild" Write-Host " -help Print help and exit" } @@ -27,9 +29,17 @@ try { . (Join-Path $PSScriptRoot "build-utils.ps1") Push-Location $RepoRoot - Write-Host "Building Roslyn" - Exec-Console (Join-Path $PSScriptRoot "build.ps1") "-restore -build -ci:$ci -configuration:$configuration -pack -binaryLog" - Exec-Console "artifacts\bin\BuildValidator\$configuration\net472\BuildValidator.exe" "--assembliesPath '$ArtifactsDir/obj/Microsoft.CodeAnalysis'" + if (-not $noBuild) { + Write-Host "Building Roslyn" + Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -restore -build -ci:$ci -configuration:$configuration -pack -binaryLog } + } + + $rebuildArgs = ("--verbose" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/netcoreapp3.1`"" + + " --debugPath `"$ArtifactsDir\BuildValidator`"" + + " --sourcePath `"$RepoRoot`"" + + " --referencesPaths `"$ArtifactsDir\bin`"") + Exec-Console "$ArtifactsDir/bin/BuildValidator/$configuration/net472/BuildValidator.exe" $rebuildArgs exit 0 } diff --git a/src/Compilers/CSharp/csc/csc.csproj b/src/Compilers/CSharp/csc/csc.csproj index 01e6741bb0961..cc6e6cabf8098 100644 --- a/src/Compilers/CSharp/csc/csc.csproj +++ b/src/Compilers/CSharp/csc/csc.csproj @@ -12,6 +12,7 @@ true false true + true diff --git a/src/Tools/BuildValidator/BuildConstructor.cs b/src/Tools/BuildValidator/BuildConstructor.cs index 7919972107383..bfac2b9d93830 100644 --- a/src/Tools/BuildValidator/BuildConstructor.cs +++ b/src/Tools/BuildValidator/BuildConstructor.cs @@ -38,7 +38,7 @@ public BuildConstructor(LocalReferenceResolver referenceResolver, LocalSourceRes _logger = logger; } - public Compilation CreateCompilation(CompilationOptionsReader compilationOptionsReader, string name) + public Compilation CreateCompilation(CompilationOptionsReader compilationOptionsReader, string fileName) { var pdbCompilationOptions = compilationOptionsReader.GetMetadataCompilationOptions(); if (pdbCompilationOptions.Length == 0) @@ -62,8 +62,8 @@ public Compilation CreateCompilation(CompilationOptionsReader compilationOptions { var compilation = language switch { - LanguageNames.CSharp => CreateCSharpCompilation(name, compilationOptionsReader, sources, metadataReferences), - LanguageNames.VisualBasic => CreateVisualBasicCompilation(name, compilationOptionsReader, sources, metadataReferences), + LanguageNames.CSharp => CreateCSharpCompilation(fileName, compilationOptionsReader, sources, metadataReferences), + LanguageNames.VisualBasic => CreateVisualBasicCompilation(fileName, compilationOptionsReader, sources, metadataReferences), _ => throw new InvalidDataException($"{language} is not a known language") }; @@ -88,7 +88,10 @@ void logResolvedSources() { var sourceFileInfo = resolvedSource.SourceFileInfo; var hash = BitConverter.ToString(sourceFileInfo.Hash).Replace("-", ""); - _logger.LogInformation($@"""{resolvedSource.DisplayPath}"" - {sourceFileInfo.HashAlgorithm} - {hash}"); + var embeddedCompressedHash = sourceFileInfo.EmbeddedCompressedHash is { } compressedHash + ? ("[uncompressed]" + BitConverter.ToString(compressedHash).Replace("-", "")) + : null; + _logger.LogInformation($@"""{resolvedSource.DisplayPath}"" - {sourceFileInfo.HashAlgorithm} - {hash} - {embeddedCompressedHash}"); } } } @@ -129,20 +132,20 @@ private ImmutableArray ResolveSources( #region CSharp private Compilation CreateCSharpCompilation( - string assemblyName, + string fileName, CompilationOptionsReader optionsReader, ImmutableArray sources, ImmutableArray metadataReferences) { - var (compilationOptions, parseOptions) = CreateCSharpCompilationOptions(optionsReader, assemblyName); + var (compilationOptions, parseOptions) = CreateCSharpCompilationOptions(optionsReader, fileName); return CSharpCompilation.Create( - assemblyName, + Path.GetFileNameWithoutExtension(fileName), syntaxTrees: sources.Select(s => CSharpSyntaxTree.ParseText(s.SourceText, options: parseOptions, path: s.SourceFileInfo.SourceFilePath)).ToImmutableArray(), references: metadataReferences, options: compilationOptions); } - private (CSharpCompilationOptions, CSharpParseOptions) CreateCSharpCompilationOptions(CompilationOptionsReader optionsReader, string assemblyName) + private (CSharpCompilationOptions, CSharpParseOptions) CreateCSharpCompilationOptions(CompilationOptionsReader optionsReader, string fileName) { using var scope = _logger.BeginScope("Options"); var pdbCompilationOptions = optionsReader.GetMetadataCompilationOptions(); @@ -175,9 +178,7 @@ private Compilation CreateCSharpCompilation( optionsReader.GetOutputKind(), reportSuppressedDiagnostics: false, - // TODO: can't rely on the implicity moduleName here. In the case of .NET Core EXE the output name will - // end with .dll but the inferred name will be .exe - moduleName: assemblyName + ".dll", + moduleName: fileName, mainTypeName: optionsReader.GetMainTypeName(), scriptClassName: null, usings: null, @@ -213,10 +214,10 @@ private Compilation CreateCSharpCompilation( return (compilationOptions, parseOptions); } - private static (OptimizationLevel, bool) GetOptimizationLevel(string optimizationLevel) + private static (OptimizationLevel, bool) GetOptimizationLevel(string? optimizationLevel) => optimizationLevel switch { - "debug" => (OptimizationLevel.Debug, false), + null or "debug" => (OptimizationLevel.Debug, false), "debug-plus" => (OptimizationLevel.Debug, true), "release" => (OptimizationLevel.Release, false), _ => throw new InvalidDataException($"Optimization \"{optimizationLevel}\" level not recognized") @@ -226,14 +227,14 @@ private static (OptimizationLevel, bool) GetOptimizationLevel(string optimizatio #region Visual Basic private Compilation CreateVisualBasicCompilation( - string assemblyName, + string fileName, CompilationOptionsReader optionsReader, ImmutableArray sources, ImmutableArray metadataReferences) { var compilationOptions = CreateVisualBasicCompilationOptions(optionsReader); return VisualBasicCompilation.Create( - assemblyName, + Path.GetFileNameWithoutExtension(fileName), syntaxTrees: sources.Select(s => VisualBasicSyntaxTree.ParseText(s.SourceText, options: compilationOptions.ParseOptions, path: s.DisplayPath)).ToImmutableArray(), references: metadataReferences, options: compilationOptions); @@ -244,7 +245,7 @@ private static VisualBasicCompilationOptions CreateVisualBasicCompilationOptions var pdbCompilationOptions = optionsReader.GetMetadataCompilationOptions(); var langVersionString = pdbCompilationOptions.GetUniqueOption("language-version"); - var optimization = pdbCompilationOptions.GetUniqueOption("optimization"); + pdbCompilationOptions.TryGetUniqueOption("optimization", out var optimization); pdbCompilationOptions.TryGetUniqueOption("define", out var define); pdbCompilationOptions.TryGetUniqueOption("strict", out var strict); pdbCompilationOptions.TryGetUniqueOption("checked", out var checkedString); diff --git a/src/Tools/BuildValidator/BuildValidator.csproj b/src/Tools/BuildValidator/BuildValidator.csproj index 1634c165010d1..1f87b9e58f7e8 100644 --- a/src/Tools/BuildValidator/BuildValidator.csproj +++ b/src/Tools/BuildValidator/BuildValidator.csproj @@ -22,6 +22,7 @@ + diff --git a/src/Tools/BuildValidator/CompilationDiff.cs b/src/Tools/BuildValidator/CompilationDiff.cs index ee3828d61bfd7..f783ca715ae16 100644 --- a/src/Tools/BuildValidator/CompilationDiff.cs +++ b/src/Tools/BuildValidator/CompilationDiff.cs @@ -18,6 +18,7 @@ using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.Emit; using Microsoft.CodeAnalysis.Text; +using Microsoft.DiaSymReader.Tools; using Microsoft.Extensions.Logging; using Microsoft.Metadata.Tools; @@ -61,6 +62,13 @@ public static unsafe CompilationDiff Create( iconInIcoFormat: null); var sourceLink = optionsReader.GetSourceLinkUTF8(); + + var embeddedTexts = producedCompilation.SyntaxTrees + .Select(st => (path: st.FilePath, text: st.GetText())) + .Where(pair => pair.text.CanBeEmbedded) + .Select(pair => EmbeddedText.FromSource(pair.path, pair.text)) + .ToImmutableArray(); + var emitResult = producedCompilation.Emit( peStream: rebuildPeStream, pdbStream: null, @@ -73,10 +81,7 @@ public static unsafe CompilationDiff Create( metadataPEStream: null, pdbOptionsBlobReader: optionsReader.GetMetadataCompilationOptionsBlobReader(), sourceLinkStream: sourceLink != null ? new MemoryStream(sourceLink) : null, - embeddedTexts: producedCompilation.SyntaxTrees - .Select(st => (path: st.FilePath, text: st.GetText())) - .Where(pair => pair.text.CanBeEmbedded) - .Select(pair => EmbeddedText.FromSource(pair.path, pair.text)), + embeddedTexts: embeddedTexts, cancellationToken: CancellationToken.None); if (!emitResult.Success) @@ -141,6 +146,26 @@ public static unsafe CompilationDiff Create( writeVisualization(originalPeMdvPath, optionsReader.PeReader.GetMetadataReader()); writeVisualization(originalPdbMdvPath, optionsReader.PdbReader); + var originalPdbXmlPath = Path.Combine(originalPath, assemblyName + ".pdb.xml"); + using var originalPdbXml = File.Create(originalPdbXmlPath); + + var rebuildPdbXmlPath = Path.Combine(rebuildPath, assemblyName + ".pdb.xml"); + + var pdbToXmlOptions = PdbToXmlOptions.ResolveTokens + | PdbToXmlOptions.ThrowOnError + | PdbToXmlOptions.ExcludeScopes + | PdbToXmlOptions.IncludeSourceServerInformation + | PdbToXmlOptions.IncludeEmbeddedSources + | PdbToXmlOptions.IncludeTokens + | PdbToXmlOptions.IncludeMethodSpans; + + PdbToXmlConverter.ToXml( + new StreamWriter(originalPdbXml), + pdbStream: new UnmanagedMemoryStream(optionsReader.PdbReader.MetadataPointer, optionsReader.PdbReader.MetadataLength), + peStream: new MemoryStream(originalBytes), + options: pdbToXmlOptions, + methodName: null); + var rebuildPeMdvPath = Path.Combine(rebuildPath, assemblyName + ".pe.mdv"); var rebuildPdbMdvPath = Path.Combine(rebuildPath, assemblyName + ".pdb.mdv"); fixed (byte* ptr = rebuildBytes) @@ -156,6 +181,28 @@ public static unsafe CompilationDiff Create( { var rebuildPdbReader = provider.GetMetadataReader(MetadataReaderOptions.Default); writeVisualization(rebuildPdbMdvPath, rebuildPdbReader); + + using var rebuildPdbXml = File.Create(rebuildPdbXmlPath); + PdbToXmlConverter.ToXml( + new StreamWriter(rebuildPdbXml), + pdbStream: new UnmanagedMemoryStream(rebuildPdbReader.MetadataPointer, rebuildPdbReader.MetadataLength), + peStream: new MemoryStream(rebuildBytes), + options: pdbToXmlOptions, + methodName: null); + + using (logger.BeginScope("Rebuild Embedded Texts raw SHAs")) + { + var rebuildReader = new CompilationOptionsReader(logger, rebuildPdbReader, rebuildPeReader); + var rebuildSourceFileInfos = rebuildReader.GetSourceFileInfos(rebuildReader.GetEncoding()); + foreach (var info in rebuildSourceFileInfos) + { + if (info.EmbeddedCompressedHash is {} hash) + { + var hashString = BitConverter.ToString(hash).Replace("-", ""); + logger.LogInformation($@"""{info.SourceFilePath}"" - {hashString}"); + } + } + } } } @@ -163,11 +210,12 @@ public static unsafe CompilationDiff Create( var ildasmRebuildOutputPath = Path.Combine(rebuildPath, assemblyName + ".il"); // TODO: can we bundle ildasm in with the utility? - Process.Start(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\ildasm.exe", $@"{originalBinaryPath.FullName} /out={ildasmOriginalOutputPath}").WaitForExit(); - Process.Start(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\ildasm.exe", $@"{rebuildAssemblyPath} /out={ildasmRebuildOutputPath}").WaitForExit(); + Process.Start(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\ildasm.exe", $@"{originalBinaryPath.FullName} /all /out={ildasmOriginalOutputPath}").WaitForExit(); + Process.Start(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\ildasm.exe", $@"{rebuildAssemblyPath} /all /out={ildasmRebuildOutputPath}").WaitForExit(); File.WriteAllText(Path.Combine(assemblyDebugPath, "compare-pe.mdv.ps1"), $@"code --diff (Join-Path $PSScriptRoot ""{originalPeMdvPath.Substring(assemblyDebugPath.Length)}"") (Join-Path $PSScriptRoot ""{rebuildPeMdvPath.Substring(assemblyDebugPath.Length)}"")"); File.WriteAllText(Path.Combine(assemblyDebugPath, "compare-pdb.mdv.ps1"), $@"code --diff (Join-Path $PSScriptRoot ""{originalPdbMdvPath.Substring(assemblyDebugPath.Length)}"") (Join-Path $PSScriptRoot ""{rebuildPdbMdvPath.Substring(assemblyDebugPath.Length)}"")"); + File.WriteAllText(Path.Combine(assemblyDebugPath, "compare-pdb.xml.ps1"), $@"code --diff (Join-Path $PSScriptRoot ""{originalPdbXmlPath.Substring(assemblyDebugPath.Length)}"") (Join-Path $PSScriptRoot ""{rebuildPdbXmlPath.Substring(assemblyDebugPath.Length)}"")"); File.WriteAllText(Path.Combine(assemblyDebugPath, "compare-il.ps1"), $@"code --diff (Join-Path $PSScriptRoot ""{ildasmOriginalOutputPath.Substring(assemblyDebugPath.Length)}"") (Join-Path $PSScriptRoot ""{ildasmRebuildOutputPath.Substring(assemblyDebugPath.Length)}"")"); } } diff --git a/src/Tools/BuildValidator/CompilationOptionsReader.cs b/src/Tools/BuildValidator/CompilationOptionsReader.cs index 9b9cd0f14776e..0cdf44a87f7f1 100644 --- a/src/Tools/BuildValidator/CompilationOptionsReader.cs +++ b/src/Tools/BuildValidator/CompilationOptionsReader.cs @@ -27,17 +27,20 @@ internal readonly struct SourceFileInfo internal SourceHashAlgorithm HashAlgorithm { get; } internal byte[] Hash { get; } internal SourceText? EmbeddedText { get; } + internal byte[]? EmbeddedCompressedHash { get; } internal SourceFileInfo( string sourceFilePath, SourceHashAlgorithm hashAlgorithm, byte[] hash, - SourceText? embeddedText) + SourceText? embeddedText, + byte[]? embeddedCompressedHash) { SourceFilePath = sourceFilePath; HashAlgorithm = hashAlgorithm; Hash = hash; EmbeddedText = embeddedText; + EmbeddedCompressedHash = embeddedCompressedHash; } } @@ -192,7 +195,7 @@ public OutputKind GetOutputKind() => return (typeName, methodName); } - private SourceText? ResolveEmbeddedSource(DocumentHandle document, SourceHashAlgorithm hashAlgorithm, Encoding encoding) + private (SourceText? embeddedText, byte[]? compressedHash) ResolveEmbeddedSource(DocumentHandle document, SourceHashAlgorithm hashAlgorithm, Encoding encoding) { byte[] bytes = (from handle in PdbReader.GetCustomDebugInformation(document) let cdi = PdbReader.GetCustomDebugInformation(handle) @@ -201,14 +204,18 @@ where PdbReader.GetGuid(cdi.Kind) == EmbeddedSourceGuid if (bytes == null) { - return null; + return default; } int uncompressedSize = BitConverter.ToInt32(bytes, 0); var stream = new MemoryStream(bytes, sizeof(int), bytes.Length - sizeof(int)); + byte[]? compressedHash = null; if (uncompressedSize != 0) { + using var algorithm = CryptographicHashProvider.TryGetAlgorithm(hashAlgorithm) ?? throw new InvalidOperationException(); + compressedHash = algorithm.ComputeHash(bytes); + var decompressed = new MemoryStream(uncompressedSize); using (var deflater = new DeflateStream(stream, CompressionMode.Decompress)) @@ -227,7 +234,8 @@ where PdbReader.GetGuid(cdi.Kind) == EmbeddedSourceGuid using (stream) { // todo: IVT and EncodedStringText.Create? - return SourceText.From(stream, encoding: encoding, checksumAlgorithm: hashAlgorithm, canBeEmbedded: true); + var embeddedText = SourceText.From(stream, encoding: encoding, checksumAlgorithm: hashAlgorithm, canBeEmbedded: true); + return (embeddedText, compressedHash); } } @@ -293,7 +301,7 @@ public ImmutableArray GetSourceFileInfos(Encoding encoding) var hash = PdbReader.GetBlobBytes(document.Hash); var embeddedContent = ResolveEmbeddedSource(documentHandle, hashAlgorithm, encoding); - builder.Add(new SourceFileInfo(name, hashAlgorithm, hash, embeddedContent)); + builder.Add(new SourceFileInfo(name, hashAlgorithm, hash, embeddedContent.embeddedText, embeddedContent.compressedHash)); } return builder.MoveToImmutable(); diff --git a/src/Tools/BuildValidator/LocalReferenceResolver.cs b/src/Tools/BuildValidator/LocalReferenceResolver.cs index 72de32d852b86..bf528fcc67742 100644 --- a/src/Tools/BuildValidator/LocalReferenceResolver.cs +++ b/src/Tools/BuildValidator/LocalReferenceResolver.cs @@ -44,7 +44,7 @@ public LocalReferenceResolver(Options options, ILoggerFactory loggerFactory) using var _ = _logger.BeginScope("Assembly Reference Search Paths"); foreach (var directory in _indexDirectories) { - _logger.LogInformation($@"""{directory}"""); + _logger.LogInformation($@"""{directory.FullName}"""); } } diff --git a/src/Tools/BuildValidator/Program.cs b/src/Tools/BuildValidator/Program.cs index 941eea0f318b1..7eed14a36288e 100644 --- a/src/Tools/BuildValidator/Program.cs +++ b/src/Tools/BuildValidator/Program.cs @@ -79,17 +79,16 @@ static int HandleCommand(string assembliesPath, string sourcePath, string[]? ref // TODO: remove the DemoLoggerProvider, update this dependency, // and move to the built in logger. - var loggerFactory = new LoggerFactory( - new[] { new ConsoleLoggerProvider(new ConsoleLoggerSettings()) }, - new LoggerFilterOptions() + var loggerFactory = LoggerFactory.Create(builder => + { + builder.SetMinimumLevel((options.Verbose, options.Quiet) switch { - MinLevel = options.Verbose ? LogLevel.Trace : LogLevel.Information + (_, true) => LogLevel.Error, + (true, _) => LogLevel.Trace, + _ => LogLevel.Information }); - - if (!options.Quiet) - { - loggerFactory.AddProvider(new DemoLoggerProvider()); - } + builder.AddProvider(new DemoLoggerProvider()); + }); var logger = loggerFactory.CreateLogger(); try @@ -213,7 +212,7 @@ private static bool ValidateFiles(IEnumerable originalBinaries, BuildC var compilation = buildConstructor.CreateCompilation( optionsReader, - Path.GetFileNameWithoutExtension(originalBinary.Name)); + originalBinary.Name); var compilationDiff = CompilationDiff.Create(originalBinary, optionsReader, compilation, getDebugEntryPoint(), logger, options); return compilationDiff; diff --git a/src/Tools/ManifestGenerator/README.md b/src/Tools/ManifestGenerator/README.md index ef0eaaf06037e..d796d86bfb342 100644 --- a/src/Tools/ManifestGenerator/README.md +++ b/src/Tools/ManifestGenerator/README.md @@ -1,4 +1,3 @@ # dotnet-roslyn-manifest-generator -https://github.com/dotnet/roslyn/blob/efd69176d6ad7f9dbe1d87525201760a6c44e7a0/docs/compilers/terrapin.md#artifacts-manifest-file - +[Terrapin doc](../../../docs/compilers/terrapin.md#artifacts-manifest-file) From 7497d362dbc9019c122b1a7496a1e34e0bd2287a Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Fri, 19 Feb 2021 13:20:37 -0800 Subject: [PATCH 2/8] fix --- eng/test-rebuild.ps1 | 3 ++- .../BuildValidator/LocalReferenceResolver.cs | 5 ++++- src/Tools/BuildValidator/Options.cs | 2 +- src/Tools/BuildValidator/Program.cs | 15 ++++++++------- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index 76c56eeb2555d..c92d77c91fce5 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -35,7 +35,8 @@ try { } $rebuildArgs = ("--verbose" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/netcoreapp3.1`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis`"" + + " --assembliesPath $ArtifactsDir/obj/csc/$configuration/netcoreapp3.1" + " --debugPath `"$ArtifactsDir\BuildValidator`"" + " --sourcePath `"$RepoRoot`"" + " --referencesPaths `"$ArtifactsDir\bin`"") diff --git a/src/Tools/BuildValidator/LocalReferenceResolver.cs b/src/Tools/BuildValidator/LocalReferenceResolver.cs index bf528fcc67742..49d93eafff9a8 100644 --- a/src/Tools/BuildValidator/LocalReferenceResolver.cs +++ b/src/Tools/BuildValidator/LocalReferenceResolver.cs @@ -34,7 +34,10 @@ public LocalReferenceResolver(Options options, ILoggerFactory loggerFactory) { _indexDirectories.Add(directoryInfo); } - _indexDirectories.Add(new DirectoryInfo(options.AssembliesPath)); + foreach (var path in options.AssembliesPaths) + { + _indexDirectories.Add(new DirectoryInfo(path)); + } _indexDirectories.Add(GetNugetCacheDirectory()); foreach (var path in options.ReferencesPaths) { diff --git a/src/Tools/BuildValidator/Options.cs b/src/Tools/BuildValidator/Options.cs index b7a23f5e1c982..31f269ff34fc7 100644 --- a/src/Tools/BuildValidator/Options.cs +++ b/src/Tools/BuildValidator/Options.cs @@ -10,7 +10,7 @@ namespace BuildValidator { internal record Options( - string AssembliesPath, + string[] AssembliesPaths, string[] ReferencesPaths, string SourcePath, bool Verbose, diff --git a/src/Tools/BuildValidator/Program.cs b/src/Tools/BuildValidator/Program.cs index 7eed14a36288e..f8b7deb241fee 100644 --- a/src/Tools/BuildValidator/Program.cs +++ b/src/Tools/BuildValidator/Program.cs @@ -43,8 +43,8 @@ static int Main(string[] args) var rootCommand = new RootCommand { new Option( - "--assembliesPath", "Path to assemblies to rebuild" - ) { IsRequired = true }, + "--assembliesPath", "Path to assemblies to rebuild (can be specified one or more times)" + ) { IsRequired = true, Argument = { Arity = ArgumentArity.OneOrMore } }, new Option( "--sourcePath", "Path to sources to use in rebuild" ) { IsRequired = true }, @@ -64,11 +64,11 @@ static int Main(string[] args) "--debugPath", "Path to output debug info. Defaults to the user temp directory. Note that a unique debug path should be specified for every instance of the tool running with `--debug` enabled." ) }; - rootCommand.Handler = CommandHandler.Create(HandleCommand); + rootCommand.Handler = CommandHandler.Create(HandleCommand); return rootCommand.Invoke(args); } - static int HandleCommand(string assembliesPath, string sourcePath, string[]? referencesPaths, bool verbose, bool quiet, bool debug, string? debugPath) + static int HandleCommand(string[] assembliesPath, string sourcePath, string[]? referencesPaths, bool verbose, bool quiet, bool debug, string? debugPath) { // If user provided a debug path then assume we should write debug outputs. debug |= debugPath is object; @@ -110,10 +110,11 @@ static int HandleCommand(string assembliesPath, string sourcePath, string[]? ref var buildConstructor = new BuildConstructor(referenceResolver, sourceResolver, logger); - var artifactsDir = new DirectoryInfo(options.AssembliesPath); + var artifactsDirs = options.AssembliesPaths.Select(path => new DirectoryInfo(path)); - var filesToValidate = artifactsDir.EnumerateFiles("*.exe", SearchOption.AllDirectories) - .Concat(artifactsDir.EnumerateFiles("*.dll", SearchOption.AllDirectories)) + var filesToValidate = artifactsDirs.SelectMany(dir => + dir.EnumerateFiles("*.exe", SearchOption.AllDirectories) + .Concat(dir.EnumerateFiles("*.dll", SearchOption.AllDirectories))) .Distinct(FileNameEqualityComparer.Instance); var success = ValidateFiles(filesToValidate, buildConstructor, logger, options); From 00ed64ebaee469b675f9313d9674338bc5466049 Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Fri, 19 Feb 2021 13:20:52 -0800 Subject: [PATCH 3/8] update yml --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b005b4ee6bcb0..e82f825c2411d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -215,7 +215,8 @@ jobs: - job: Correctness_Rebuild pool: - vmImage: windows-2019 + name: NetCorePublic-Pool + queue: BuildPool.Windows.10.Amd64.Open timeoutInMinutes: 90 steps: - template: eng/pipelines/checkout-windows-task.yml From 652a528900a18306af9c83acd58a160c58435ee9 Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Fri, 19 Feb 2021 13:29:10 -0800 Subject: [PATCH 4/8] fix yml --- azure-pipelines.yml | 6 ------ eng/test-rebuild.ps1 | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e82f825c2411d..a1606682cd577 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -227,12 +227,6 @@ jobs: filePath: eng/build.ps1 arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog - - task: PowerShell@2 - displayName: Build - inputs: - filePath: eng/build.ps1 - arguments: -configuration Debug -prepareMachine -ci -build -bootstrap -publish -binaryLog -skipDocumentation - - powershell: .\eng\test-rebuild.ps1 displayName: Run BuildValidator diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index c92d77c91fce5..7c8203c680f05 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -31,11 +31,11 @@ try { if (-not $noBuild) { Write-Host "Building Roslyn" - Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -restore -build -ci:$ci -configuration:$configuration -pack -binaryLog } + Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -build -bootstrap -ci:$ci -configuration:$configuration -pack -binaryLog } } $rebuildArgs = ("--verbose" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/$configuration/netcoreapp3.1`"" + " --assembliesPath $ArtifactsDir/obj/csc/$configuration/netcoreapp3.1" + " --debugPath `"$ArtifactsDir\BuildValidator`"" + " --sourcePath `"$RepoRoot`"" + From 253522544edfcf1b1f5aabb495fe5bf356c01041 Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Fri, 19 Feb 2021 16:09:24 -0800 Subject: [PATCH 5/8] extract out ref assemblies paths --- .vscode/launch.json | 6 ++++-- eng/build-utils.ps1 | 2 +- eng/test-rebuild.ps1 | 9 ++++++--- src/Tools/BuildValidator/LocalReferenceResolver.cs | 14 -------------- src/Tools/BuildValidator/Program.cs | 14 ++++++++------ 5 files changed, 19 insertions(+), 26 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index dedb274341bc5..5c446623b4d14 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,8 +12,10 @@ // If you have changed target frameworks, make sure to update the program path. "program": "${workspaceFolder}/artifacts/bin/BuildValidator/Debug/netcoreapp3.1/BuildValidator.dll", "args": [ - "--assembliesPath", "./artifacts/obj/csc/Debug/net472", - "--referencesPaths", "./artifacts/bin", + "--assembliesPath", "./artifacts/obj/csc/Debug/netcoreapp3.1", + "--referencesPath", "./artifacts/bin", + "--referencesPath", "C:/Program Files/dotnet/packs/Microsoft.AspNetCore.App.Ref", + "--referencesPath", "C:/Program Files/dotnet/packs/Microsoft.NETCore.App.Ref", "--debugPath", "./artifacts/BuildValidator", "--sourcePath", "." ], diff --git a/eng/build-utils.ps1 b/eng/build-utils.ps1 index 6e32af8866751..3ba5c48b83003 100644 --- a/eng/build-utils.ps1 +++ b/eng/build-utils.ps1 @@ -333,7 +333,7 @@ function Make-BootstrapBuild([switch]$force32 = $false) { Run-MSBuild $projectPath "/restore /t:Pack /p:RoslynEnforceCodeStyle=false /p:RunAnalyzersDuringBuild=false /p:DotNetUseShippingVersions=true /p:InitialDefineConstants=BOOTSTRAP /p:PackageOutputPath=`"$dir`" /p:EnableNgenOptimization=false /p:PublishWindowsPdb=false $force32Flag" -logFileName "Bootstrap" -configuration $bootstrapConfiguration -runAnalyzers $packageFile = Get-ChildItem -Path $dir -Filter "$packageName.*.nupkg" - Unzip (Join-Path $dir $packageFile) $dir + Unzip (Join-Path $dir $packageFile.Name) $dir Write-Host "Cleaning Bootstrap compiler artifacts" Run-MSBuild $projectPath "/t:Clean" -logFileName "BootstrapClean" diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index 7c8203c680f05..ab0b24b4beac0 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -34,12 +34,15 @@ try { Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -build -bootstrap -ci:$ci -configuration:$configuration -pack -binaryLog } } + $dotnetInstallDir = (InitializeDotNetCli -install:$true) $rebuildArgs = ("--verbose" + - " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/$configuration/netcoreapp3.1`"" + + " --assembliesPath `"$ArtifactsDir/obj/Microsoft.CodeAnalysis/$configuration`"" + " --assembliesPath $ArtifactsDir/obj/csc/$configuration/netcoreapp3.1" + - " --debugPath `"$ArtifactsDir\BuildValidator`"" + + " --debugPath `"$ArtifactsDir/BuildValidator`"" + " --sourcePath `"$RepoRoot`"" + - " --referencesPaths `"$ArtifactsDir\bin`"") + " --referencesPath `"$ArtifactsDir/bin`"" + + " --referencesPath `"$dotnetInstallDir/packs/Microsoft.AspNetCore.App.Ref`"" + + " --referencesPath `"$dotnetInstallDir/packs/Microsoft.NETCore.App.Ref`"") Exec-Console "$ArtifactsDir/bin/BuildValidator/$configuration/net472/BuildValidator.exe" $rebuildArgs exit 0 diff --git a/src/Tools/BuildValidator/LocalReferenceResolver.cs b/src/Tools/BuildValidator/LocalReferenceResolver.cs index 49d93eafff9a8..242260162265a 100644 --- a/src/Tools/BuildValidator/LocalReferenceResolver.cs +++ b/src/Tools/BuildValidator/LocalReferenceResolver.cs @@ -30,10 +30,6 @@ internal class LocalReferenceResolver public LocalReferenceResolver(Options options, ILoggerFactory loggerFactory) { _logger = loggerFactory.CreateLogger(); - foreach (var directoryInfo in GetRefAssembliesDirectories()) - { - _indexDirectories.Add(directoryInfo); - } foreach (var path in options.AssembliesPaths) { _indexDirectories.Add(new DirectoryInfo(path)); @@ -62,16 +58,6 @@ public static DirectoryInfo GetNugetCacheDirectory() return new DirectoryInfo(nugetPackageDirectory); } - public static DirectoryInfo[] GetRefAssembliesDirectories() - { - // TODO: Don't hardcode the paths here. - return new[] - { - new DirectoryInfo(@"C:\Program Files\dotnet\packs\Microsoft.AspNetCore.App.Ref"), - new DirectoryInfo(@"C:\Program Files\dotnet\packs\Microsoft.NETCore.App.Ref") - }; - } - public string GetReferencePath(MetadataReferenceInfo referenceInfo) { if (_cache.TryGetValue(referenceInfo.Mvid, out var value)) diff --git a/src/Tools/BuildValidator/Program.cs b/src/Tools/BuildValidator/Program.cs index f8b7deb241fee..a78d68a4c0263 100644 --- a/src/Tools/BuildValidator/Program.cs +++ b/src/Tools/BuildValidator/Program.cs @@ -48,9 +48,9 @@ static int Main(string[] args) new Option( "--sourcePath", "Path to sources to use in rebuild" ) { IsRequired = true }, - new Option( - "--referencesPaths", "Additional paths to referenced assemblies" - ), + new Option( + "--referencesPath", "Path to referenced assemblies (can be specified zero or more times)" + ) { Argument = { Arity = ArgumentArity.ZeroOrMore } }, new Option( "--verbose", "Output verbose log information" ), @@ -64,18 +64,20 @@ static int Main(string[] args) "--debugPath", "Path to output debug info. Defaults to the user temp directory. Note that a unique debug path should be specified for every instance of the tool running with `--debug` enabled." ) }; + var res = rootCommand.Parse(args); + rootCommand.Handler = CommandHandler.Create(HandleCommand); return rootCommand.Invoke(args); } - static int HandleCommand(string[] assembliesPath, string sourcePath, string[]? referencesPaths, bool verbose, bool quiet, bool debug, string? debugPath) + static int HandleCommand(string[] assembliesPath, string sourcePath, string[]? referencesPath, bool verbose, bool quiet, bool debug, string? debugPath) { // If user provided a debug path then assume we should write debug outputs. debug |= debugPath is object; debugPath ??= Path.Combine(Path.GetTempPath(), $"BuildValidator"); - referencesPaths ??= Array.Empty(); + referencesPath ??= Array.Empty(); - var options = new Options(assembliesPath, referencesPaths, sourcePath, verbose, quiet, debug, debugPath); + var options = new Options(assembliesPath, referencesPath, sourcePath, verbose, quiet, debug, debugPath); // TODO: remove the DemoLoggerProvider, update this dependency, // and move to the built in logger. From 60647bf17e337f129e3b62dbb8b98a5e055a8d64 Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Fri, 19 Feb 2021 17:16:06 -0800 Subject: [PATCH 6/8] Fix formatting --- src/Tools/BuildValidator/CompilationDiff.cs | 2 +- src/Tools/BuildValidator/Program.cs | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Tools/BuildValidator/CompilationDiff.cs b/src/Tools/BuildValidator/CompilationDiff.cs index f783ca715ae16..f0729cb069dc8 100644 --- a/src/Tools/BuildValidator/CompilationDiff.cs +++ b/src/Tools/BuildValidator/CompilationDiff.cs @@ -196,7 +196,7 @@ public static unsafe CompilationDiff Create( var rebuildSourceFileInfos = rebuildReader.GetSourceFileInfos(rebuildReader.GetEncoding()); foreach (var info in rebuildSourceFileInfos) { - if (info.EmbeddedCompressedHash is {} hash) + if (info.EmbeddedCompressedHash is { } hash) { var hashString = BitConverter.ToString(hash).Replace("-", ""); logger.LogInformation($@"""{info.SourceFilePath}"" - {hashString}"); diff --git a/src/Tools/BuildValidator/Program.cs b/src/Tools/BuildValidator/Program.cs index a78d68a4c0263..734921b08fc0a 100644 --- a/src/Tools/BuildValidator/Program.cs +++ b/src/Tools/BuildValidator/Program.cs @@ -64,8 +64,6 @@ static int Main(string[] args) "--debugPath", "Path to output debug info. Defaults to the user temp directory. Note that a unique debug path should be specified for every instance of the tool running with `--debug` enabled." ) }; - var res = rootCommand.Parse(args); - rootCommand.Handler = CommandHandler.Create(HandleCommand); return rootCommand.Invoke(args); } @@ -79,9 +77,8 @@ static int HandleCommand(string[] assembliesPath, string sourcePath, string[]? r var options = new Options(assembliesPath, referencesPath, sourcePath, verbose, quiet, debug, debugPath); - // TODO: remove the DemoLoggerProvider, update this dependency, - // and move to the built in logger. - var loggerFactory = LoggerFactory.Create(builder => + // TODO: remove the DemoLoggerProvider or convert it to something more permanent + var loggerFactory = LoggerFactory.Create(builder => { builder.SetMinimumLevel((options.Verbose, options.Quiet) switch { From b236c5b396c413e45b3d3a7df0d6c661a00ab20e Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Mon, 22 Feb 2021 13:00:02 -0800 Subject: [PATCH 7/8] Update azure-pipelines.yml Co-authored-by: Jared Parsons --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a1606682cd577..13b78bf4cffba 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -227,7 +227,7 @@ jobs: filePath: eng/build.ps1 arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog - - powershell: .\eng\test-rebuild.ps1 + - powershell: .\eng\test-rebuild.ps1 -ci displayName: Run BuildValidator - task: PublishBuildArtifacts@1 From ef2e9b00f8268aee2ccb4a3cddd9a5c8541cd8bb Mon Sep 17 00:00:00 2001 From: Rikki Gibson Date: Mon, 22 Feb 2021 13:01:03 -0800 Subject: [PATCH 8/8] Update eng/test-rebuild.ps1 --- eng/test-rebuild.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/test-rebuild.ps1 b/eng/test-rebuild.ps1 index ab0b24b4beac0..3bb1aac9eb35e 100644 --- a/eng/test-rebuild.ps1 +++ b/eng/test-rebuild.ps1 @@ -16,7 +16,7 @@ function Print-Usage() { Write-Host "Usage: test-rebuild.ps1" Write-Host " -configuration Build configuration ('Debug' or 'Release')" Write-Host " -ci Set when running on CI server" - Write-Host " -noBuild If set, skips building before running the rebuild" + Write-Host " -noBuild If set, skips running a bootstrap build before running the rebuild" Write-Host " -help Print help and exit" }