Changes to enable building msbuild with Unified Build

eng/AfterSigning.targets

@@ -1,12 +1,15 @@
 <Project>
-    <Target Name="CopyAMD64Symbols" Condition="'$(OfficialBuild)' == 'true' AND '$(ArcadeBuildFromSource)' != 'true'" BeforeTargets="Build">
+    <!-- Don't attempt to copy the PDBs at all when building on core on non-windows platforms. -->
+    <Target Name="CopyAMD64Symbols" Condition="'$(OfficialBuild)' == 'true' AND $([MSBuild]::IsOSPlatform('windows'))" BeforeTargets="Build">
         <Copy
             SourceFiles="$(ArtifactsBinDir)MSBuild\x64\Release\net472\MSBuild.pdb"
             DestinationFolder="$(ArtifactsSymStoreDirectory)\MSBuild\net472\amd64"
             />
+        <!-- Excluded when building using msbuild core because msbuild core doesn't support targeting 3.5
+             See src\MSBuildTaskHost\MSBuildTaskHost.csproj-->
         <Copy
             SourceFiles="$(ArtifactsBinDir)MSBuildTaskHost\x64\Release\net35\MSBuildTaskHost.pdb"
             DestinationFolder="$(ArtifactsSymStoreDirectory)\MSBuildTaskHost\net35\amd64"
-            />
+            Condition="'$(MSBuildRuntimeType)' != 'Core' and '$(MonoBuild)' != 'true'"/>
     </Target>
 </Project>

eng/Packages.props

@@ -23,13 +23,26 @@
     <PackageVersion Include="System.Net.Http" Version="$(SystemNetHttpVersion)" />
     <PackageVersion Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
     <PackageVersion Include="System.Reflection.MetadataLoadContext" Version="$(SystemReflectionMetadataLoadContextVersion)" />
-    <PackageVersion Include="System.Resources.Extensions" Version="$(SystemResourcesExtensionsPackageVersion)" />
+    <PackageVersion Include="System.Resources.Extensions" Version="$(SystemResourcesExtensionsVersion)" />
     <PackageVersion Include="System.Runtime.CompilerServices.Unsafe" Version="$(SystemRuntimeCompilerServicesUnsafeVersion)" />
     <PackageVersion Include="System.Security.Principal.Windows" Version="$(SystemSecurityPrincipalWindowsVersion)" />
     <PackageVersion Include="System.Text.Encoding.CodePages" Version="$(SystemTextEncodingCodePagesVersion)" />
     <PackageVersion Include="System.Text.Json" Version="$(SystemTextJsonVersion)" />
     <PackageVersion Include="System.Threading.Channels" Version="$(SystemThreadingChannelsVersion)" />
     <PackageVersion Include="System.Threading.Tasks.Dataflow" Version="$(SystemThreadingTasksDataflowVersion)" />
     <PackageVersion Include="xunit.console" Version="$(XUnitVersion)" />
+
+  </ItemGroup>
+
+  <!-- This set of packages will use different versions based on building binaries for VS. In non-UB modes this has no effect. -->
+  <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">
+    <PackageVersion Update="System.Text.Json" Version="$(SystemTextJsonNetFrameworkVersion)" />
+    <PackageVersion Update="System.Collections.Immutable" Version="$(SystemCollectionsImmutableNetFrameworkVersion)" />
+    <PackageVersion Update="System.Configuration.ConfigurationManager" Version="$(SystemConfigurationConfigurationManagerNetFrameworkVersion)" />
+    <PackageVersion Update="System.Reflection.Metadata" Version="$(SystemReflectionMetadataNetFrameworkVersion)" />
+    <PackageVersion Update="System.Reflection.MetadataLoadContext" Version="$(SystemReflectionMetadataLoadContextNetFrameworkVersion)" />
+    <PackageVersion Update="System.Resources.Extensions" Version="$(SystemResourcesExtensionsNetFrameworkVersion)" />
+    <PackageVersion Update="System.Threading.Tasks.Dataflow" Version="$(SystemThreadingTasksDataflowNetFrameworkVersion)" />
+    <PackageVersion Update="System.Threading.Channels" Version="$(SystemThreadingChannelsNetFrameworkVersion)" />
   </ItemGroup>
 </Project>

eng/SourceBuild.props

@@ -7,12 +7,13 @@
     <SourceBuildManagedOnly>true</SourceBuildManagedOnly>
   </PropertyGroup>
 
-  <Target Name="ConfigureInnerBuildArgs"
-          BeforeTargets="GetSourceBuildCommandConfiguration"
-          Condition="'$(ArcadeBuildFromSource)' == 'true'">
+  <Target Name="ConfigureInnerBuildArgs" BeforeTargets="GetSourceBuildCommandConfiguration">
     <PropertyGroup>
-      <InnerBuildArgs>$(InnerBuildArgs) /p:Projects="$(InnerSourceBuildRepoRoot)MSBuild.SourceBuild.slnf"</InnerBuildArgs>
-      <!-- Disable package validation as source build filters out target frameworks. -->
+      <!-- Filter down projects aggressively in source-only modes. -->
+      <InnerBuildArgs Condition="'$(ArcadeBuildFromSource)' == 'true' or '$(DotNetBuildSourceOnly)' == 'true'">$(InnerBuildArgs) /p:Projects="$(InnerSourceBuildRepoRoot)MSBuild.SourceBuild.slnf"</InnerBuildArgs>
+
+      <!-- Disable package validation as unified build may upgrade the net core target framework or
+           skip building net framework targets in source-only modes. -->
       <InnerBuildArgs>$(InnerBuildArgs) /p:EnablePackageValidation=false</InnerBuildArgs>
     </PropertyGroup>
   </Target>

eng/Versions.props

@@ -25,9 +25,29 @@
     <MicrosoftVSSDKBuildToolsVersion>16.7.13</MicrosoftVSSDKBuildToolsVersion>
   </PropertyGroup>
   <!-- Production Dependencies -->
+  <!-- There is some delicacy and balance between several scenarios here:
+       - MSBuild may build against older packages and run against newer ones within the .NET SDK (build against .NET 8 binaries, run in NET 9 SDK).
+       - MSBuild may build against newer packages and run against newer ones within the .NET SDK when built as part of the product in Unified Build.
+       - MSBuild must build against specific older packages and run against older assemblies in the GAC in VS scenarios.
+       - MSBuild must build against newer packages when it will redistribute those packages in source-only builds for Linux distributions.
+
+       Unified Build (including source only builds) sets package versions by importing a package props file after
+       Versions.props which overrides versions that are mentioned in the Version.Details.xml file. This will upgrade some dependencies
+       that are redistributed by msbuild from the mentioned version to the live version produced by runtime, for instance.
+
+       This is not desired behavior in .NET Framework scenarios, where the version must match what VS will be shipping.
+       To achieve these many behaviors, we specify two version properties for these select packages:
+       - *Version - Set to default value that the repo will use when building in isolation.
+       - *NetFrameworkVersion, set to the corresponding *Version.
+
+       In UB, the *Version property will be overridden with the live built version, but *NetFrameworkVersion will retain the original value.
+       In Packages.props or in specific references, we then conditionalize based on the target framework or potentially build scenario
+       to achieve the desired result. -->
   <PropertyGroup>
     <SystemCollectionsImmutableVersion>8.0.0</SystemCollectionsImmutableVersion>
+    <SystemCollectionsImmutableNetFrameworkVersion>$(SystemCollectionsImmutableVersion)</SystemCollectionsImmutableNetFrameworkVersion>
     <SystemConfigurationConfigurationManagerVersion>8.0.0</SystemConfigurationConfigurationManagerVersion>
+    <SystemConfigurationConfigurationManagerNetFrameworkVersion>$(SystemConfigurationConfigurationManagerVersion)</SystemConfigurationConfigurationManagerNetFrameworkVersion>
     <!--
         Modifying the version of System.Memory is very high impact and causes downstream breaks in third-party tooling that uses the MSBuild API.
         When updating the version of System.Memory file a breaking change here: https://github.com/dotnet/docs/issues/new?assignees=gewarren&labels=breaking-change%2CPri1%2Cdoc-idea&template=breaking-change.yml&title=%5BBreaking+change%5D%3A+
@@ -36,12 +56,19 @@
     <SystemMemoryVersion>4.5.5</SystemMemoryVersion>
     <SystemNetHttpVersion>4.3.4</SystemNetHttpVersion>
     <SystemReflectionMetadataLoadContextVersion>8.0.0</SystemReflectionMetadataLoadContextVersion>
+    <SystemReflectionMetadataLoadContextNetFrameworkVersion>$(SystemReflectionMetadataLoadContextVersion)</SystemReflectionMetadataLoadContextNetFrameworkVersion>
     <SystemReflectionMetadataVersion>8.0.0</SystemReflectionMetadataVersion>
-    <SystemResourcesExtensionsPackageVersion>8.0.0</SystemResourcesExtensionsPackageVersion>
+    <SystemReflectionMetadataNetFrameworkVersion>$(SystemReflectionMetadataVersion)</SystemReflectionMetadataNetFrameworkVersion>
+    <SystemResourcesExtensionsVersion>8.0.0</SystemResourcesExtensionsVersion>
+    <SystemResourcesExtensionsNetFrameworkVersion>$(SystemResourcesExtensionsVersion)</SystemResourcesExtensionsNetFrameworkVersion>
     <SystemSecurityPrincipalWindowsVersion>5.0.0</SystemSecurityPrincipalWindowsVersion>
     <SystemTextEncodingCodePagesVersion>7.0.0</SystemTextEncodingCodePagesVersion>
+    <SystemTextJsonVersion>8.0.0</SystemTextJsonVersion>
+    <SystemTextJsonNetFrameworkVersion>$(SystemTextJsonVersion)</SystemTextJsonNetFrameworkVersion>
     <SystemThreadingChannelsVersion>8.0.0</SystemThreadingChannelsVersion>
+    <SystemThreadingChannelsNetFrameworkVersion>$(SystemThreadingChannelsVersion)</SystemThreadingChannelsNetFrameworkVersion>
     <SystemThreadingTasksDataflowVersion>8.0.0</SystemThreadingTasksDataflowVersion>
+    <SystemThreadingTasksDataflowNetFrameworkVersion>$(SystemThreadingTasksDataflowVersion)</SystemThreadingTasksDataflowNetFrameworkVersion>
   </PropertyGroup>
   <!-- Toolset Dependencies -->
   <PropertyGroup>
@@ -55,7 +82,6 @@
     <MicrosoftNetCompilersToolsetVersion>4.9.0-3.23629.3</MicrosoftNetCompilersToolsetVersion>
     <NuGetBuildTasksVersion>6.9.0-preview.1.70</NuGetBuildTasksVersion>
     <SystemRuntimeCompilerServicesUnsafeVersion>6.0.0</SystemRuntimeCompilerServicesUnsafeVersion>
-    <SystemTextJsonVersion>8.0.0</SystemTextJsonVersion>
   </PropertyGroup>
   <Target Name="OverrideArcadeFileVersion" AfterTargets="_InitializeAssemblyVersion">
     <!-- See https://github.com/dotnet/arcade/issues/3386

src/Build/Microsoft.Build.csproj

@@ -15,7 +15,9 @@
     <EnableDefaultItems>false</EnableDefaultItems>
 
     <GenerateReferenceAssemblySource>true</GenerateReferenceAssemblySource>
-    <CreateTlb>true</CreateTlb>
+    <!-- Do not create Tlbs when building in .NET product mode. The packages are not shipped to VS,
+         only their contents redisted within the SDK. -->
+    <CreateTlb Condition="'$(DotNetBuild)' != 'true'">true</CreateTlb>
     <IsPackable>true</IsPackable>
     <PackageDescription>This package contains the $(MSBuildProjectName) assembly which is used to create, edit, and evaluate MSBuild projects.</PackageDescription>
     <IncludeSatelliteOutputInPack>false</IncludeSatelliteOutputInPack>

src/Deprecated/Engine/Microsoft.Build.Engine.csproj

@@ -4,7 +4,9 @@
     <NoWarn>$(NoWarn);618</NoWarn>
     <GenerateReferenceAssemblySource>true</GenerateReferenceAssemblySource>
     <EnableDefaultItems>false</EnableDefaultItems>
-    <CreateTlb>true</CreateTlb>
+    <!-- Do not create Tlbs when building in .NET product mode. The packages are not shipped to VS,
+         only their contents redisted within the SDK. -->
+    <CreateTlb Condition="'$(DotNetBuild)' != 'true'">true</CreateTlb>
     <AdditionalTlbExpAsmPaths>$(XMakeRefPath)</AdditionalTlbExpAsmPaths>
     <PublishTlbPath>$(XMakeRefPath)</PublishTlbPath>
     <GenerateAssemblyRefs>true</GenerateAssemblyRefs>

src/Directory.Build.targets

@@ -118,7 +118,10 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="CreateTypeLib" BeforeTargets="AfterBuild" Inputs="$(TargetPath)" Outputs="$(TargetDir)$(TargetName).tlb;$(TargetDir)x64\$(TargetName).tlb" Condition="'$(BuildingInsideVisualStudio)' != 'true' and '$(CreateTlb)' == 'true' and $([MSBuild]::IsOSPlatform('windows')) and '$(TargetFrameworkIdentifier)' == '.NETFramework' and '$(MSBuildRuntimeType)' != 'Core'">
+  <!-- This target creates a type library for the assemblies in a package. It requires desktop msbuild to locate the tooling
+       Type libraries are still needed by Visual Studio in some cases, like the legacy C# project system.
+       It calls into msbuild via COM, and needs the type library to do so. -->
+  <Target Name="CreateTypeLib" BeforeTargets="AfterBuild" Inputs="$(TargetPath)" Outputs="$(TargetDir)$(TargetName).tlb;$(TargetDir)x64\$(TargetName).tlb" Condition="'$(BuildingInsideVisualStudio)' != 'true' and '$(CreateTlb)' == 'true' and $([MSBuild]::IsOSPlatform('windows')) and '$(TargetFrameworkIdentifier)' == '.NETFramework'">
     <PropertyGroup>
       <TlbExpPath>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPathToDotNetFrameworkSdkFile('tlbexp.exe'))</TlbExpPath>
       <!-- Provide a mechanism for turning on verbose TlbExp output for diagnosing issues -->

src/Framework/Microsoft.Build.Framework.csproj

@@ -3,7 +3,9 @@
     <TargetFrameworks>$(LibraryTargetFrameworks)</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <GenerateReferenceAssemblySource>true</GenerateReferenceAssemblySource>
-    <CreateTlb>true</CreateTlb>
+    <!-- Do not create Tlbs when building in .NET product mode. The packages are not shipped to VS,
+         only their contents redisted within the SDK. -->
+    <CreateTlb Condition="'$(DotNetBuild)' != 'true'">true</CreateTlb>
     <IsPackable>true</IsPackable>
     <PackageDescription>This package contains the $(MSBuildProjectName) assembly which is a common assembly used by other MSBuild assemblies.</PackageDescription>
     <IncludeSatelliteOutputInPack>false</IncludeSatelliteOutputInPack>

src/MSBuild.Bootstrap/MSBuild.Bootstrap.csproj

@@ -20,7 +20,7 @@
   <ItemGroup>
     <!-- This file is needed so the dotnet CLI knows how to map preview SDK versions to tfms (because tfms do not have preview information on them) -->
     <!-- This is because according to semver, 2.1.0-preview is not >= 2.1.0 -->
-    <Content Include="$(RepoRoot).dotnet\sdk\$(DotNetCliVersion)\Microsoft.NETCoreSdk.BundledVersions.props" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="$(DotNetRoot)sdk\$(DotNetCliVersion)\Microsoft.NETCoreSdk.BundledVersions.props" CopyToOutputDirectory="PreserveNewest" />
 
     <!-- Include NuGet build tasks -->
     <PackageReference Include="NuGet.Build.Tasks" />
@@ -44,7 +44,7 @@
 
     <Content Include="$(MSBuildExtensionsPath)\**\*" LinkBase="Extensions" CopyToOutputDirectory="PreserveNewest" />
 
-    <Content Include="$(RepoRoot).dotnet\sdk\$(DotNetCliVersion)\RuntimeIdentifierGraph.json" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="$(DotNetRoot)sdk\$(DotNetCliVersion)\RuntimeIdentifierGraph.json" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
 
   <!-- Use deps file from this project with additional dependencies listed instead of the one generated in the MSBuild project -->

src/MSBuild/MSBuild.csproj

@@ -239,7 +239,7 @@
 
     <ItemGroup>
       <_OurFiles Include="$(OutputPath)%(_TargetFrameworks.Identity)\MSBuild.exe.config" TargetFramework="%(_TargetFrameworks.Identity)" Condition=" '%(_TargetFrameworks.Identity)' == '$(FullFrameworkTFM)' " />
-      <_OurFiles Include="$(OutputPath)%(_TargetFrameworks.Identity)\Microsoft.Build.Framework.tlb" TargetFramework="%(_TargetFrameworks.Identity)" Condition=" '%(_TargetFrameworks.Identity)' == '$(FullFrameworkTFM)' " />
+      <_OurFiles Include="$(OutputPath)%(_TargetFrameworks.Identity)\Microsoft.Build.Framework.tlb" TargetFramework="%(_TargetFrameworks.Identity)" Condition=" '%(_TargetFrameworks.Identity)' == '$(FullFrameworkTFM)' and '$(CreateTlb)' == 'true'" />
       <_OurFiles Include="$(OutputPath)%(_TargetFrameworks.Identity)\Microsoft.Data.Entity.targets" TargetFramework="%(_TargetFrameworks.Identity)" Condition=" '%(_TargetFrameworks.Identity)' == '$(FullFrameworkTFM)' " />
       <_OurFiles Include="$(OutputPath)%(_TargetFrameworks.Identity)\Microsoft.ServiceModel.targets" TargetFramework="%(_TargetFrameworks.Identity)" Condition=" '%(_TargetFrameworks.Identity)' == '$(FullFrameworkTFM)' " />
       <_OurFiles Include="$(OutputPath)%(_TargetFrameworks.Identity)\Microsoft.WinFx.targets" TargetFramework="%(_TargetFrameworks.Identity)" Condition=" '%(_TargetFrameworks.Identity)' == '$(FullFrameworkTFM)' " />

src/Package/DevDivPackage/DevDivPackage.csproj

@@ -7,6 +7,8 @@
     <PlatformTarget>x86</PlatformTarget>
     <DisableSemVer2>true</DisableSemVer2>
     <IsVisualStudioInsertionPackage>true</IsVisualStudioInsertionPackage>
+    <!-- VS insertion packages are not needed when doing a .NET product build. -->
+    <ExcludeFromDotNetBuild>true</ExcludeFromDotNetBuild>
   </PropertyGroup>
 
   <ItemGroup>

src/Tasks/Microsoft.Build.Tasks.csproj

@@ -11,7 +11,9 @@
     <AssemblyName>Microsoft.Build.Tasks.Core</AssemblyName>
     <RootNamespace>Microsoft.Build.Tasks</RootNamespace>
     <DefineConstants>$(DefineConstants);MICROSOFT_BUILD_TASKS</DefineConstants>
-    <CreateTlb>true</CreateTlb>
+    <!-- Do not create Tlbs when building in .NET product mode. The packages are not shipped to VS,
+         only their contents redisted within the SDK. -->
+    <CreateTlb Condition="'$(DotNetBuild)' != 'true'">true</CreateTlb>
     <IsPackable>true</IsPackable>
     <PackageDescription>This package contains the $(MSBuildProjectName) assembly which implements the commonly used tasks of MSBuild.</PackageDescription>
     <IncludeSatelliteOutputInPack>false</IncludeSatelliteOutputInPack>