ArPow stage 1: local source-build infrastructure (#51765)

* ArPow stage 1: local source-build infra

azure-pipelines.yml

@@ -148,6 +148,8 @@ jobs:
     configuration: Debug
     testArguments: --testCoreClr --helixQueueName OSX.1014.Amd64.Open
 
+- template: eng/common/templates/jobs/source-build.yml
+
 # Build Correctness Jobs
 
 - job: Correctness_Determinism
@@ -198,30 +200,6 @@ jobs:
         publishLocation: Container
       condition: succeeded()
 
-- job: Correctness_SourceBuild
-  pool:
-   name: NetCorePublic-Pool
-   queue: BuildPool.Ubuntu.1604.amd64.Open
-  timeoutInMinutes: 90
-  steps:
-    - template: eng/pipelines/checkout-unix-task.yml
-
-    - script: ./eng/cibuild.sh --configuration Debug --prepareMachine --binaryLog --sourceBuild
-      displayName: Build
-
-    - template: eng/pipelines/publish-logs.yml
-      parameters:
-        jobName: Correctness_SourceBuild
-        configuration: Release
-    - task: PublishBuildArtifacts@1
-      displayName: Publish Logs
-      inputs:
-        PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/Debug'
-        ArtifactName: 'SourceBuild_Test'
-        publishLocation: Container
-      continueOnError: true
-      condition: not(succeeded())
-
 - job: Correctness_Rebuild
   pool:
     name: NetCorePublic-Pool

eng/SourceBuild.props

@@ -0,0 +1,33 @@
+<Project>
+
+  <PropertyGroup>
+    <GitHubRepositoryName>roslyn</GitHubRepositoryName>
+    <SourceBuildManagedOnly>true</SourceBuildManagedOnly>
+  </PropertyGroup>
+
+  <Target Name="ApplySourceBuildPatchFiles"
+          Condition="
+            '$(ArcadeBuildFromSource)' == 'true' and
+            '$(ArcadeInnerBuildFromSource)' == 'true'"
+          BeforeTargets="Execute">
+    <ItemGroup>
+      <SourceBuildPatchFile Include="$(RepositoryEngineeringDir)source-build-patches\*.patch" />
+    </ItemGroup>
+
+    <Exec
+      Command="git apply --ignore-whitespace --whitespace=nowarn &quot;%(SourceBuildPatchFile.FullPath)&quot;"
+      WorkingDirectory="$(RepoRoot)"
+      Condition="'@(SourceBuildPatchFile)' != ''" />
+  </Target>
+
+  <!--
+    The build script passes in the full path of the sln to build.  This must be overridden in order to build
+    the cloned source in the inner build.
+  -->
+  <Target Name="ConfigureInnerBuildArg" BeforeTargets="GetSourceBuildCommandConfiguration">
+    <PropertyGroup>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:Projects="$(InnerSourceBuildRepoRoot)\Compilers.sln"</InnerBuildArgs>
+    </PropertyGroup>
+  </Target>
+
+</Project>

eng/SourceBuildPrebuiltBaseline.xml

@@ -0,0 +1,5 @@
+<UsageData>
+  <IgnorePatterns>
+    <UsagePattern IdentityGlob="*/*" />
+  </IgnorePatterns>
+</UsageData>

eng/Version.Details.xml

@@ -1,11 +1,22 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Dependencies>
   <ProductDependencies>
+    <Dependency Name="XliffTasks" Version="1.0.0-beta.21215.1">
+      <Uri>https://github.com/dotnet/xliff-tasks</Uri>
+      <Sha>7e80445ee82adbf9a8e6ae601ac5e239d982afaa</Sha>
+      <SourceBuild RepoName="xliff-tasks" ManagedOnly="true" />
+    </Dependency>
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build" Version="0.1.0-alpha.1.21126.2">
+      <Uri>https://github.com/dotnet/source-build</Uri>
+      <Sha>c06ae1212bd69e9fe52bed0b0a7d79cc8ea39054</Sha>
+      <SourceBuild RepoName="source-build" ManagedOnly="true"/>
+    </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
     <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21216.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>53fe29e220fc0db05eafd5c6bc6c8fb9ee7cec7c</Sha>
+      <SourceBuild RepoName="arcade" ManagedOnly="true"/>
     </Dependency>
     <Dependency Name="Microsoft.Net.Compilers.Toolset" Version="3.10.0-3.21216.10">
       <Uri>https://github.com/dotnet/roslyn</Uri>

eng/build.sh

@@ -306,7 +306,7 @@ function BuildSolution {
     /p:ContinuousIntegrationBuild=$ci \
     /p:TreatWarningsAsErrors=true \
     /p:TestRuntimeAdditionalArguments=$test_runtime_args \
-    /p:DotNetBuildFromSource=$source_build \
+    /p:ArcadeBuildFromSource=$source_build \
     $test_runtime \
     $mono_tool \
     $generate_documentation_file \

eng/source-build-patches/0001-Conditionally-remove-net472-from-TargetFrameworks.patch

@@ -0,0 +1,32 @@
+From b3ea2bb3b55f654b32475f214e3d3a4b39e3d3eb Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Mon, 15 Jul 2019 19:26:42 +0000
+Subject: [PATCH 1/5] Conditionally remove net472 from TargetFrameworks
+
+---
+ .../Microsoft.Net.Compilers.Toolset.Package.csproj             | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj b/src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj
+index 9e646ebab5b..7a27d77877f 100644
+--- a/src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj
++++ b/src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj
+@@ -2,6 +2,7 @@
+ <Project Sdk="Microsoft.NET.Sdk">
+   <PropertyGroup>
+     <TargetFrameworks>netcoreapp3.1;net472</TargetFrameworks>
++    <TargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true'">netcoreapp3.1</TargetFrameworks>
+
+     <IsPackable>true</IsPackable>
+     <NuspecPackageId>Microsoft.Net.Compilers.Toolset</NuspecPackageId>
+@@ -51,7 +52,7 @@
+       <_File Include="@(CoreClrCompilerBinArtifact)" TargetDir="tasks/netcoreapp3.1/bincore"/>
+       <_File Include="@(CoreClrCompilerBinRuntimesArtifact)" TargetDir="tasks/netcoreapp3.1/bincore/runtimes"/>
+
+-      <_File Include="$(MSBuildProjectDirectory)\build\**\*.*" Condition="'$(TargetFramework)' == 'net472'" TargetDir="build" />
++      <_File Include="$(MSBuildProjectDirectory)\build\**\*.*" Condition="'$(TargetFramework)' == 'net472' or '$(DotNetBuildFromSource)' == 'true'" TargetDir="build" />
+
+       <TfmSpecificPackageFile Include="@(_File)" PackagePath="%(_File.TargetDir)/%(_File.RecursiveDir)%(_File.FileName)%(_File.Extension)" />
+     </ItemGroup>
+-- 
+2.18.0

eng/source-build-patches/0002-Do-not-build-NET-Fx-binaries-in-source-build.patch

@@ -0,0 +1,24 @@
+From 23af45da9c3fc6718a7d6e1ddcb4891374d40309 Mon Sep 17 00:00:00 2001
+From: adaggarwal <aditya.aggarwal@microsoft.com>
+Date: Wed, 20 Nov 2019 22:31:11 +0000
+Subject: [PATCH 2/5] Do not build NET Fx binaries in source build
+
+---
+ eng/Versions.props | 2 +-
+ 1 file changed, 1 insertions(+), 1 deletions(-)
+
+diff --git a/eng/Versions.props b/eng/Versions.props
+index e7eb20eff11..dc3a737f86a 100644
+--- a/eng/Versions.props
++++ b/eng/Versions.props
+@@ -253,7 +253,7 @@
+     <UsingToolSymbolUploader>true</UsingToolSymbolUploader>
+     <UsingToolNuGetRepack>true</UsingToolNuGetRepack>
+     <UsingToolVSSDK>true</UsingToolVSSDK>
+-    <UsingToolNetFrameworkReferenceAssemblies>true</UsingToolNetFrameworkReferenceAssemblies>
++    <UsingToolNetFrameworkReferenceAssemblies Condition="'$(DotNetBuildFromSource)' != 'true'">true</UsingToolNetFrameworkReferenceAssemblies>
+     <UsingToolIbcOptimization>true</UsingToolIbcOptimization>
+     <UsingToolVisualStudioIbcTraining>true</UsingToolVisualStudioIbcTraining>
+     <UsingToolXliff>true</UsingToolXliff>
+--
+2.18.0

eng/source-build-patches/0004-Include-Compilers-package-projects-in-source-build.patch

@@ -0,0 +1,52 @@
+From 17c34e0cdbc1a2219dd1a7457c16e32706ee59e3 Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Mon, 25 Nov 2019 17:10:34 +0000
+Subject: [PATCH 4/5] Include Compilers package projects in source-build
+
+---
+ .../Microsoft.NETCore.Compilers.Package.csproj                  | 2 +-
+ .../Microsoft.Net.Compilers.Toolset.Package.csproj              | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/NuGet/Microsoft.NETCore.Compilers/Microsoft.NETCore.Compilers.Package.csproj b/src/NuGet/Microsoft.NETCore.Compilers/Microsoft.NETCore.Compilers.Package.csproj
+index a7802b01f5..dcb5df09f2 100644
+--- a/src/NuGet/Microsoft.NETCore.Compilers/Microsoft.NETCore.Compilers.Package.csproj
++++ b/src/NuGet/Microsoft.NETCore.Compilers/Microsoft.NETCore.Compilers.Package.csproj
+@@ -4,6 +4,7 @@
+     <TargetFramework>netcoreapp3.1</TargetFramework>
+
+     <IsPackable>true</IsPackable>
++    <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
+     <NuspecPackageId>Microsoft.NETCore.Compilers</NuspecPackageId>
+     <IncludeBuildOutput>false</IncludeBuildOutput>
+     <DevelopmentDependency>true</DevelopmentDependency>
+@@ -17,7 +18,6 @@
+   <ItemGroup>
+     <ProjectReference Include="..\..\Compilers\CSharp\csc\csc.csproj"/>
+     <ProjectReference Include="..\..\Compilers\VisualBasic\vbc\vbc.csproj"/>
+-    <ProjectReference Include="..\..\Interactive\csi\csi.csproj"/>
+     <ProjectReference Include="..\..\Compilers\Core\MSBuildTask\Microsoft.Build.Tasks.CodeAnalysis.csproj"/>
+     <ProjectReference Include="..\..\Compilers\Server\VBCSCompiler\VBCSCompiler.csproj"/>
+   </ItemGroup>
+diff --git a/src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj b/src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj
+index 5f8c77e6e2..2178f6f5ee 100644
+--- a/src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj
++++ b/src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj
+@@ -5,6 +5,7 @@
+     <TargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true'">netcoreapp3.1</TargetFrameworks>
+
+     <IsPackable>true</IsPackable>
++    <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
+     <NuspecPackageId>Microsoft.Net.Compilers.Toolset</NuspecPackageId>
+     <IncludeBuildOutput>false</IncludeBuildOutput>
+     <DevelopmentDependency>true</DevelopmentDependency>
+@@ -31,7 +32,6 @@
+   <ItemGroup>
+     <ProjectReference Include="..\..\Compilers\CSharp\csc\csc.csproj" PrivateAssets="All"/>
+     <ProjectReference Include="..\..\Compilers\VisualBasic\vbc\vbc.csproj" PrivateAssets="All"/>
+-    <ProjectReference Include="..\..\Interactive\csi\csi.csproj" PrivateAssets="All"/>
+     <ProjectReference Include="..\..\Compilers\Core\MSBuildTask\Microsoft.Build.Tasks.CodeAnalysis.csproj" PrivateAssets="All"/>
+     <ProjectReference Include="..\..\Compilers\Server\VBCSCompiler\VBCSCompiler.csproj" PrivateAssets="All"/>
+
+-- 
+2.17.1

eng/source-build-patches/0005-Don-t-restore-tools-in-source-build.patch

@@ -0,0 +1,28 @@
+From 93a4b6a0a67a211a2e69b198b0e6a98a941d64ce Mon Sep 17 00:00:00 2001
+From: Chris Rummel <crummel@microsoft.com>
+Date: Fri, 4 Sep 2020 09:32:49 -0500
+Subject: [PATCH] Don't restore tools in source-build
+
+---
+ eng/build.sh | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/eng/build.sh b/eng/build.sh
+index eab7422de0a..cf04036b5be 100755
+--- a/eng/build.sh
++++ b/eng/build.sh
+@@ -319,9 +319,9 @@ if [[ "$restore" == true || "$test_core_clr" == true ]]; then
+   install=true
+ fi
+ InitializeDotNetCli $install
+-if [[ "$restore" == true ]]; then
+-  dotnet tool restore
+-fi
++# if [[ "$restore" == true ]]; then
++#   dotnet tool restore
++# fi
+
+ bootstrap_dir=""
+ if [[ "$bootstrap" == true ]]; then
+-- 
+2.21.3

eng/source-build-patches/0006-Dirty-revert-of-https-github.com-dotnet-roslyn-issue.patch

@@ -0,0 +1,25 @@
+From ee8cef65cd2bc322fe005c3c8b96ecbf39f167e0 Mon Sep 17 00:00:00 2001
+From: Chris Rummel <crummel@microsoft.com>
+Date: Fri, 4 Sep 2020 11:50:51 -0500
+Subject: [PATCH 6/6] Dirty revert of
+ https://github.com/dotnet/roslyn/issues/10785
+
+---
+ src/Compilers/Shared/BuildClient.cs | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/Compilers/Shared/BuildClient.cs b/src/Compilers/Shared/BuildClient.cs
+index 4d1660794cb..cb606f3a957 100644
+--- a/src/Compilers/Shared/BuildClient.cs
++++ b/src/Compilers/Shared/BuildClient.cs
+@@ -84,7 +84,7 @@ internal static int Run(IEnumerable<string> arguments, RequestLanguage language,
+             {
+                 // Register encodings for console
+                 // https://github.com/dotnet/roslyn/issues/10785
+-                System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
++                //System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
+             }
+
+             var client = new BuildClient(language, compileFunc, logger);
+-- 
+2.21.3

eng/source-build-patches/0008-Update-a-S.IO.Pipes.AccessControl-to-use-reference-p.patch

@@ -0,0 +1,27 @@
+From 5bc570fdbc5bf41e9da0d152ef8677dd3dd361ec Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Thu, 24 Sep 2020 15:19:14 +0000
+Subject: [PATCH] Update a S.IO.Pipes.AccessControl to use reference packages
+
+---
+ src/Compilers/Directory.Build.props | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+ create mode 100644 src/Compilers/Directory.Build.props
+
+diff --git a/src/Compilers/Directory.Build.props b/src/Compilers/Directory.Build.props
+new file mode 100644
+index 0000000..f6dd261
+--- /dev/null
++++ b/src/Compilers/Directory.Build.props
+@@ -0,0 +1,9 @@
++<Project>
++  <Import Project="$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))" />
++  <PropertyGroup>
++    <!-- When building from source, lock these versions to reference package versions
++         rather than getting the versions from package flow.  Source-built versions of
++         these packages only support building net5.0 -->
++    <SystemIOPipesAccessControlVersion>4.5.1</SystemIOPipesAccessControlVersion>
++  </PropertyGroup>
++</Project>
+-- 
+1.8.3.1

eng/source-build-patches/0008-remove-RichCodeNav.EnvVarDump-on-source-build.patch

@@ -0,0 +1,24 @@
+From 25257a04d8b3299be86cc41e4db1f0de4823024b Mon Sep 17 00:00:00 2001
+From: Tom Deseyn <tom.deseyn@gmail.com>
+Date: Mon, 19 Oct 2020 09:33:59 +0200
+Subject: [PATCH] roslyn: remove RichCodeNav.EnvVarDump on source-build
+
+---
+ eng/targets/Settings.props | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/eng/targets/Settings.props b/eng/targets/Settings.props
+index c3ddc1bc17b..0cbd7e8c40e 100644
+--- a/eng/targets/Settings.props
++++ b/eng/targets/Settings.props
+@@ -131,7 +131,7 @@
+   <!-- 
+      Code indexing targets to help generating LSIF from indexing builds.
+   -->
+-  <ItemGroup>
++  <ItemGroup Condition="'$(DotNetBuildFromSource)' != 'true'">
+     <PackageReference Include="RichCodeNav.EnvVarDump" Version="$(RichCodeNavEnvVarDumpVersion)" PrivateAssets="all" />
+   </ItemGroup>
+
+-- 
+2.26.2

eng/source-build-patches/0009-Don-t-build-VBCSCompiler-for-net472.patch

@@ -0,0 +1,31 @@
+From b39388d2e77c2821f7bbb4976fb8429dfffb8c6b Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Mon, 12 Oct 2020 19:37:27 +0000
+Subject: [PATCH] Don't build VBCSCompiler for net472
+
+Don't build VBCSCompiler for net472 because it can't load
+System.Collections.Immutable.  In the non-source-build build, this
+package is loaded from the restored package, but with a tarball,
+the package comes from previously source-built, which is only built
+for net50.  It then tries to fall back to referenceassemblies which
+doesn't contain the package.
+See https://github.com/dotnet/source-build/issues/1798
+---
+ src/Compilers/Server/VBCSCompiler/VBCSCompiler.csproj | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/Compilers/Server/VBCSCompiler/VBCSCompiler.csproj b/src/Compilers/Server/VBCSCompiler/VBCSCompiler.csproj
+index 3fde7d7..ec4dd26 100644
+--- a/src/Compilers/Server/VBCSCompiler/VBCSCompiler.csproj
++++ b/src/Compilers/Server/VBCSCompiler/VBCSCompiler.csproj
+@@ -5,7 +5,7 @@
+   <PropertyGroup>
+     <OutputType>Exe</OutputType>
+     <LargeAddressAware>true</LargeAddressAware>
+-    <TargetFrameworks>netcoreapp3.1;net472</TargetFrameworks>
++    <TargetFrameworks>netcoreapp3.1</TargetFrameworks>
+     <UseVSHostingProcess>false</UseVSHostingProcess>
+     <ServerGarbageCollection>true</ServerGarbageCollection>
+     <UseAppHost>false</UseAppHost>
+-- 
+1.8.3.1

eng/source-build-patches/0011-Build-Microsoft.CodeAnalysis-for-source-build.patch

@@ -0,0 +1,36 @@
+From eee8808e3b9ebb9ec66835858b1ad46ec8d5688f Mon Sep 17 00:00:00 2001
+From: Omair Majid <omajid@redhat.com>
+Date: Mon, 9 Nov 2020 18:43:03 -0500
+Subject: [PATCH] Build Microsoft.CodeAnalysis for source-build
+
+---
+ src/NuGet/Microsoft.CodeAnalysis.Package.csproj | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/src/NuGet/Microsoft.CodeAnalysis.Package.csproj b/src/NuGet/Microsoft.CodeAnalysis.Package.csproj
+index ac7d9993db0..a25843f0e1c 100644
+--- a/src/NuGet/Microsoft.CodeAnalysis.Package.csproj
++++ b/src/NuGet/Microsoft.CodeAnalysis.Package.csproj
+@@ -2,11 +2,13 @@
+ <Project Sdk="Microsoft.NET.Sdk">
+   <PropertyGroup>
+     <TargetFrameworks>net472;netstandard2.0</TargetFrameworks>
++    <TargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true'">netstandard2.0</TargetFrameworks>
+
+     <!-- NuGet -->
+     <IsPackable>true</IsPackable>
+     <PackageId>Microsoft.CodeAnalysis</PackageId>
+     <IncludeBuildOutput>false</IncludeBuildOutput>
++    <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
+     <PackageDescription>
+       .NET Compiler Platform ("Roslyn").
+
+@@ -25,4 +27,4 @@
+     <ProjectReference Include="..\Workspaces\CSharp\Portable\Microsoft.CodeAnalysis.CSharp.Workspaces.csproj" />
+     <ProjectReference Include="..\Workspaces\VisualBasic\Portable\Microsoft.CodeAnalysis.VisualBasic.Workspaces.vbproj" />
+   </ItemGroup>
+-</Project>
+\ No newline at end of file
++</Project>
+-- 
+2.26.2