Put artifacts in well-known locations

.devcontainer/vmr-source-build/README.md

@@ -13,7 +13,7 @@ This Codespace can help you debug the source build of .NET. In case you have run
 `/workspaces/dotnet` with the PR changes pulled into it. You can then attempt to source-build
 the VMR which is what the VMR leg in the installer PR build doing. This build takes about 45
 minutes and, after completion, produces an archived .NET SDK located in
-`/workspaces/dotnet/artifacts/x64/Release`.
+`/workspaces/dotnet/artifacts/assets/Release`.
 
 ## Build the SDK
 

eng/pipelines/templates/jobs/vmr-build.yml

@@ -139,7 +139,6 @@ jobs:
           vmrPath: $(vmrPath)
           vmrBranch: ${{ parameters.vmrBranch }}
           targetRef: $(Build.SourceVersion) # Synchronize the current installer commit
-          architecture: ${{ parameters.architecture }}
 
   - ${{ if parameters.buildFromArchive }}:
     - script: |
@@ -348,7 +347,9 @@ jobs:
 
         CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.binlog"
         CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.log"
-        CopyWithRelativeFolders "artifacts/prebuilt-report/" $targetFolder
+        if ("${{ parameters.buildSourceOnly }}" -eq $True) {
+          CopyWithRelativeFolders "artifacts/prebuilt-report/" $targetFolder
+        }
         CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.log"
         CopyWithRelativeFolders "test/"                      $targetFolder "*.binlog"
@@ -370,7 +371,9 @@ jobs:
         cd "$(sourcesPath)"
         find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
-        find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
+        if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
+          find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
+        fi
         find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
         find test/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
@@ -400,8 +403,15 @@ jobs:
         publishRunAttachments: true
         testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName)
 
-  - publish: '$(sourcesPath)/artifacts/${{ parameters.architecture }}/Release/'
+  - publish: '$(sourcesPath)/artifacts/assets/Release/'
     artifact: $(Agent.JobName)_Artifacts
     displayName: Publish Artifacts
     condition: succeededOrFailed()
     continueOnError: true
+
+  # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
+  - ${{ if ne(parameters.buildSourceOnly, 'true') }}:
+    - publish: '$(sourcesPath)/artifacts/packages/Release/'
+      artifact: $(Agent.JobName)_Packages
+      displayName: Publish Packages
+      condition: ${{ parameters.buildSourceOnly }}

eng/pipelines/templates/steps/vmr-pull-updates.yml

@@ -17,11 +17,6 @@ parameters:
   type: string
   default: $(Agent.BuildDirectory)/vmr
 
-- name: architecture
-  displayName: Architecture used for build artifacts
-  type: string
-  default: 'x64'
-
 steps:
 - checkout: self
   displayName: Clone dotnet/installer
@@ -41,16 +36,6 @@ steps:
   displayName: Prepare branch ${{ parameters.vmrBranch }}
   workingDirectory: ${{ parameters.vmrPath }}
 
-- script: |
-    mkdir -p ${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release
-  displayName: Create artifacts folder (Unix)
-  condition: ne(variables['Agent.OS'], 'Windows_NT')
-
-- powershell: |
-    New-Item -ItemType Directory -Path ${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release
-  displayName: Create artifacts folder (Windows)
-  condition: eq(variables['Agent.OS'], 'Windows_NT')
-
 - script: |
     git config --global user.name "dotnet-maestro[bot]"
     git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com"

src/SourceBuild/content/.devcontainer/README.md

@@ -10,7 +10,7 @@
 
 This Codespace can help you debug the source build of .NET. This build takes about
 45 minutes and, after completion, produces an archived .NET SDK located in
-`/workspaces/dotnet/artifacts/x64/Release`. In case you selected the `prebuilt-sdk`
+`/workspaces/dotnet/artifacts/assets/Release`. In case you selected the `prebuilt-sdk`
 Codespace, the built-from-source SDK will already be there.
 
 ## Build the SDK

src/SourceBuild/content/Directory.Build.props

@@ -92,6 +92,9 @@
     <ArtifactsObjDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'obj'))</ArtifactsObjDir>
     <ArtifactsBinDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin'))</ArtifactsBinDir>
     <ArtifactsLogDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'log', '$(Configuration)'))</ArtifactsLogDir>
+    <ArtifactsPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(Configuration)'))</ArtifactsPackagesDir>
+    <ArtifactsNonShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'NonShipping'))</ArtifactsNonShippingPackagesDir>
+    <ArtifactsShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'Shipping'))</ArtifactsShippingPackagesDir>
     <RepositoryEngineeringDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'eng'))</RepositoryEngineeringDir>
 
     <!-- ProjectLayout.props -->
@@ -168,14 +171,8 @@
     <ArcadeBootstrapPackageDir Condition="'$(DotNetBuildSourceOnly)' != 'true'">$(NuGetPackageRoot)</ArcadeBootstrapPackageDir>
 
     <VSMSBuildSdkResolversDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers'))</VSMSBuildSdkResolversDir>
-
-    <!-- Shared output and intermediate output path folders that are architecture and configuration specific. -->
-    <SharedOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedOutputPath>
-    <SharedIntermediateOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedIntermediateOutputPath>
-
-    <SourceBuiltBlobFeedDir>$([MSBuild]::NormalizeDirectory('$(SharedIntermediateOutputPath)', 'blob-feed'))</SourceBuiltBlobFeedDir>
-    <SourceBuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'packages'))</SourceBuiltPackagesPath>
-    <SourceBuiltAssetsDir>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'assets'))</SourceBuiltAssetsDir>
+    <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
+    <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
     <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>

src/SourceBuild/content/Directory.Build.targets

@@ -7,7 +7,7 @@
       <SdkFilenamePrefix>dotnet-sdk-</SdkFilenamePrefix>
     </PropertyGroup>
     <ItemGroup>
-      <SdkTarballItem Include="$(SourceBuiltAssetsDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsShippingPackagesDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
     </ItemGroup>
 
     <!--

src/SourceBuild/content/README.md

@@ -109,21 +109,21 @@ In case you don't want to / cannot prepare your environment per the requirements
         ./build.sh -sb --clean-while-building
         ```
 
-    The resulting SDK is placed at `artifacts/[your-arch]/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz` (for Unix) or `artifacts/[your-arch]/Release/dotnet-sdk-9.0.100-[your-RID].zip` (for Windows).
+    The resulting SDK is placed at `artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz` (for Unix) or `artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].zip` (for Windows).
 
 4. *(Optional)* **Unpack and install the .NET SDK**
 
     For Unix:
     ```bash
     mkdir -p $HOME/dotnet
-    tar zxf artifacts/[your-arch]/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz -C $HOME/dotnet
+    tar zxf artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz -C $HOME/dotnet
     ln -s $HOME/dotnet/dotnet /usr/bin/dotnet
     ```
 
     For Windows:
     ```cmd
     mkdir %userprofile%\dotnet
-    tar -xf artifacts/<arch>/Release/dotnet-sdk-9.0.100-[your RID].zip -C %userprofile%\dotnet
+    tar -xf artifacts/assets/Release/dotnet-sdk-9.0.100-[your RID].zip -C %userprofile%\dotnet
     set "PATH=%userprofile%\dotnet;%PATH%"
     ```
 
@@ -152,7 +152,7 @@ git clone https://github.com/dotnet/dotnet .
 ./prep.sh && ./build.sh -sb --clean-while-building
 
 mkdir -p $HOME/.dotnet
-tar -zxf artifacts/x64/Release/dotnet-sdk-9.0.100-centos.8-x64.tar.gz -C $HOME/.dotnet
+tar -zxf artifacts/assets/Release/dotnet-sdk-9.0.100-centos.8-x64.tar.gz -C $HOME/.dotnet
 ln -s $HOME/.dotnet/dotnet /usr/bin/dotnet
 ```
 

src/SourceBuild/content/build.proj

@@ -19,21 +19,6 @@
              StopOnFirstFailure="true" />
   </Target>
 
-  <!-- Copies the output assets of the builds to the output path. -->
-  <Target Name="CopyBinariesToBinFolder"
-          AfterTargets="Build">
-    <ItemGroup>
-      <BinaryToCopy Include="$(SourceBuiltAssetsDir)*.*"
-                    Exclude="$(SourceBuiltAssetsDir)*.nupkg;
-                             $(SourceBuiltAssetsDir)*.requires_nupkg_signing" />
-    </ItemGroup>
-
-    <Copy SourceFiles="@(BinaryToCopy)"
-          DestinationFolder="$(SharedOutputPath)"
-          SkipUnchangedFiles="true"
-          Condition="'@(BinaryToCopy)'!=''" />
-  </Target>
-
   <Import Project="$(RepositoryEngineeringDir)build.sourcebuild.targets" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
 
 </Project>

src/SourceBuild/content/eng/build.sourcebuild.targets

@@ -15,45 +15,17 @@
     <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
   </Target>
 
-  <Target Name="DiscoverSymbolsTarballs"
-          AfterTargets="Build">
-    <ItemGroup>
-      <SymbolsTarball Include="$(SharedOutputPath)Symbols.*$(ArchiveExtension)" />
-    </ItemGroup>
-  </Target>
-
-  <Target Name="ExtractSymbolsTarballs"
-          AfterTargets="Build"
-          DependsOnTargets="DiscoverSymbolsTarballs"
-          Outputs="%(SymbolsTarball.Identity)">
-
-    <PropertyGroup>
-      <Filename>$([System.IO.Path]::GetFileName('%(SymbolsTarball.Identity)'))</Filename>
-      <RepositoryName>$(Filename.Split('.')[1])</RepositoryName>
-      <UnifiedSymbolsLayout>$(ArtifactsTmpDir)Symbols</UnifiedSymbolsLayout>
-      <DestinationFolder>$(UnifiedSymbolsLayout)/$(RepositoryName)</DestinationFolder>
-    </PropertyGroup>
-
-    <MakeDir Directories="$(DestinationFolder)" />
-    <Exec Command="tar -xzf %(SymbolsTarball.Identity) -C $(DestinationFolder)"
-          WorkingDirectory="$(SymbolsRoot)" />
-
-    <Delete Files="%(SymbolsTarball.Identity)" />
-  </Target>
-
   <!-- After building, repackage symbols into a single tarball. -->
   <Target Name="RepackageSymbols"
           AfterTargets="Build"
-          DependsOnTargets="
-            DetermineSourceBuiltSdkVersion;
-            DiscoverSymbolsTarballs;
-            ExtractSymbolsTarballs">
+          DependsOnTargets="DetermineSourceBuiltSdkVersion">
     <PropertyGroup>
-      <UnifiedSymbolsTarball>$(SharedOutputPath)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
+      <UnifiedSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
     </PropertyGroup>
 
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(UnifiedSymbolsTarball)'))" />
     <Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
-          WorkingDirectory="$(UnifiedSymbolsLayout)" />
+          WorkingDirectory="$(IntermediateSymbolsRootDir)" />
 
     <Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />
   </Target>
@@ -64,33 +36,33 @@
           AfterTargets="Build"
           DependsOnTargets="RepackageSymbols">
     <ItemGroup>
-      <SdkTarballItem Include="$(SharedOutputPath)dotnet-sdk-*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsAssetsDir)dotnet-sdk-*$(ArchiveExtension)" />
     </ItemGroup>
 
     <PropertyGroup>
-      <SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
-      <SdkSymbolsTarball>$(SharedOutputPath)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
-      <SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
+      <SdkSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
+
+      <IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
+      <IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
       <SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
     </PropertyGroup>
 
-    <MakeDir Directories="$(SdkLayout)" />
-    <Exec Command="tar -xzf $(SdkTarball) -C $(SdkLayout)"
-          WorkingDirectory="$(SharedOutputPath)" />
+    <MakeDir Directories="$(IntermediateSdkLayout)" />
+    <Exec Command="tar -xzf $(SdkTarball) -C $(IntermediateSdkLayout)" />
 
-    <CreateSdkSymbolsLayout SdkLayoutPath="$(SdkLayout)"
-                            AllSymbolsPath="$(UnifiedSymbolsLayout)"
-                            SdkSymbolsLayoutPath="$(SdkSymbolsLayout)"
+    <CreateSdkSymbolsLayout SdkLayoutPath="$(IntermediateSdkLayout)"
+                            AllSymbolsPath="$(IntermediateSymbolsRootDir)"
+                            SdkSymbolsLayoutPath="$(IntermediateSdkSymbolsLayout)"
                             FailOnMissingPDBs="false" />
 
     <Exec Command="tar --numeric-owner -czf $(SdkSymbolsTarball) *"
-          WorkingDirectory="$(SdkSymbolsLayout)" />
+          WorkingDirectory="$(IntermediateSdkSymbolsLayout)" />
 
     <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
 
-    <RemoveDir Directories="$(UnifiedSymbolsLayout)" />
-    <RemoveDir Directories="$(SdkSymbolsLayout)" />
-    <RemoveDir Directories="$(SdkLayout)" />
+    <RemoveDir Directories="$(IntermediateSymbolsRootDir)" />
+    <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
+    <RemoveDir Directories="$(IntermediateSdkLayout)" />
   </Target>
 
   <!--
@@ -111,21 +83,16 @@
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)ReportPoisonUsage.complete" >
     <ItemGroup>
-      <FinalCliTarball Include="$(SharedOutputPath)**/*$(ArchiveExtension)" />
+      <FinalCliTarball Include="$(ArtifactsAssetsDir)*$(ArchiveExtension)" />
     </ItemGroup>
 
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(FinalCliTarball) for poisoned files." />
 
-    <ItemGroup>
-      <NonShippingPackagesList Include="$(PackageListsDir)**/$(NonShippingPackagesListPrefix)*" />
-    </ItemGroup>
-
     <CheckForPoison FilesToCheck="@(FinalCliTarball)"
                     ProjectDirPath="$(RepoRoot)"
                     HashCatalogFilePath="$(PoisonReportDataFile)"
                     MarkerFileName="$(PoisonMarkerFile)"
-                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)"
-                    NonShippingPackagesListFiles="@(NonShippingPackagesList)" />
+                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
 
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />
 
@@ -155,8 +122,8 @@
 
   <Target Name="RunSmokeTest">
     <ItemGroup>
-      <SdkTarballItem Include="$(SharedOutputPath)**/dotnet-sdk*$(ArchiveExtension)" />
-      <SourceBuiltArtifactsItem Include="$(SharedOutputPath)**/$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsAssetsDir)dotnet-sdk*$(ArchiveExtension)" />
+      <SourceBuiltArtifactsItem Include="$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
     </ItemGroup>
 
     <PropertyGroup>
@@ -203,17 +170,18 @@
           DependsOnTargets="DetermineSourceBuiltSdkVersion"
           Condition="'@(SmokeTestsPrereqs->Count())' != '0'">
     <PropertyGroup>
-      <SmokeTestPrereqsTarballName>$(SharedOutputPath)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarballName>
+      <SmokeTestPrereqsTarball>$(ArtifactsAssetsDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarball>
       <SmokeTestsPrereqPackagesDir>$(SmokeTestsArtifactsDir)prereq-packages/</SmokeTestsPrereqPackagesDir>
     </PropertyGroup>
 
     <Copy SourceFiles="@(SmokeTestsPrereqs)"
           DestinationFolder="$(SmokeTestsPrereqPackagesDir)" />
 
-    <Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarballName) ."
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SmokeTestPrereqsTarball)'))" />
+    <Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarball) ."
           WorkingDirectory="$(SmokeTestsPrereqPackagesDir)"/>
 
-    <Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarballName)'" />
+    <Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarball)'" />
   </Target>
 
   <Target Name="CreatePrebuiltsTarball"
@@ -236,13 +204,14 @@
           DependsOnTargets="DetermineSourceBuiltSdkVersion"
           Condition="'@(PrebuiltFile->Count())' != '0'">
     <PropertyGroup>
-      <TarballFilePath>$(SharedOutputPath)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</TarballFilePath>
-      <TarballWorkingDir>$(ResultingPrebuiltPackagesDir)</TarballWorkingDir>
+      <PrebuiltsTarball>$(ArtifactsAssetsDir)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</PrebuiltsTarball>
+      <PrebuiltsTarballWorkingDir>$(ResultingPrebuiltPackagesDir)</PrebuiltsTarballWorkingDir>
     </PropertyGroup>
 
-    <Exec Command="tar --numeric-owner -zcf $(TarballFilePath) -C $(TarballWorkingDir) ." />
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(PrebuiltsTarball)'))" />
+    <Exec Command="tar --numeric-owner -zcf $(PrebuiltsTarball) -C $(PrebuiltsTarballWorkingDir) ." />
 
-    <Message Text="Tarball '$(TarballFilePath)' was successfully created from '$(TarballWorkingDir)'" Importance="High" />
+    <Message Text="Tarball '$(PrebuiltsTarball)' was successfully created from '$(PrebuiltsTarballWorkingDir)'" Importance="High" />
   </Target>
 
   <Target Name="ErrorOnPrebuilts"