Use stages pipeline

- #11924
- change ci.yml to use stages and post-build.yml
  - add use of publish-build-assets.yml and post-build.yml
  - publish in last build step of each job
    - pass more MSBuild properties into builds
  - use distinct `$(AssetManifestFileName)` values per job
    - set global property to override what's hard-coded in Publish.proj
- change codesign-xplat.yml to use empty.proj and normal Arcade signing and publication process
  - remove XPlatPackageSigner.proj
- change default-build.yml to use job.yml
  - remove unused parameters e.g. `matrix`, `poolName`, `variables`
  - use `enableMicrobuild` and `enablePublishTestResults` to eliminate duplicate build steps
- add .dll's and .exe's as files to sign w/ Microsoft400
  - add signcheck exclusions
- remove custom manifest generation i.e. the `GenerateBuildAssetManifest` target and related artifacts
  - use regular Arcade publish infrastructure
- update docker infrastructure to use same paths in and out of the container
- correct typo in build.sh

nits:
- upload logs in first artifact
- remove attempts to package non-existent VSIX
  - follow-up to 29cf7ec
- respect verbosity setting in build.sh
- add more information to Artifacts.md
- enable test signing in internal PRs

Author: dougbu

Diff for: .azure/pipelines/ci.yml

@@ -14,18 +14,41 @@ jobs:
     agentOs: Windows
     installNodeJs: false
     installJdk: false
+    variables:
+    - DOTNET_HOME: $(Build.SourcesDirectory)/.dotnet
     steps:
     - task: DownloadBuildArtifacts@0
       displayName: Download ${{ parameters.inputName }} artifacts
       inputs:
         artifactName: ${{ parameters.inputName }}_Packages
-        downloadPath: $(Build.StagingDirectory)/deps/
+        downloadPath: $(Build.StagingDirectory)/downloaded_packages/
         itemPattern: '**/*.nupkg'
-    - task: MSBuild@1
-      displayName: Code-sign .nupkg files
+    - task: CopyFiles@2
+      displayName: Copy packages to ArtifactsShippingPackagesDir
       inputs:
-        solution: eng\tools\XplatPackageSigner\XplatPackageSigner.proj
-        msbuildArguments: /p:SignType=$(_SignType) /p:DirectoryToSign=$(Build.StagingDirectory)\deps\${{ parameters.inputName }}_Packages\
+        sourceFolder: $(Build.StagingDirectory)/downloaded_packages/
+        contents: '**/*.nupkg'
+        targetFolder: $(Build.SourcesDirectory)/artifacts/packages/$(BuildConfiguration)/shipping/
+        flattenFolders: true
+    - powershell: . ./activate.ps1
+      displayName: Activate
+    - script: restore.cmd -ci
+      displayName: Restore tools
+    - powershell: .\eng\common\build.ps1
+        -ci
+        -restore
+        -sign
+        -publish
+        -configuration $(BuildConfiguration)
+        -projects $(Build.SourcesDirectory)/eng/empty.proj
+        /p:AssetManifestFileName=aspnetcore-${{ parameters.inputName }}-signed.xml
+        /p:DotNetSignType=$(_SignType)
+        $(_BuildArgs)
+        $(_PublishArgs)
+      displayName: Sign and publish packages
     artifacts:
     - name: ${{ parameters.inputName }}_Packages_Signed
-      path: $(Build.StagingDirectory)\deps\${{ parameters.inputName }}_Packages\
+      path: artifacts/packages/
+    - name: CodeSign_Xplat_${{ parameters.inputName }}_Logs
+      path: artifacts/log/
+      publishOnError: true

Diff for: .azure/pipelines/jobs/codesign-xplat.yml

@@ -221,7 +221,7 @@ if [ "$build_deps" = false ]; then
     msbuild_args[${#msbuild_args[*]}]="-p:BuildProjectReferences=false"
 fi
 
-if [ "$build_managed" = true ] || (["$build_all" = true ] && [ "$build_managed" != false ]); then
+if [ "$build_managed" = true ] || ([ "$build_all" = true ] && [ "$build_managed" != false ]); then
     if [ -z "$build_nodejs" ]; then
         if [ -x "$(command -v node)" ]; then
             __warn "Building of C# project is enabled and has dependencies on NodeJS projects. Building of NodeJS projects is enabled since node is detected on PATH."
@@ -266,6 +266,10 @@ if [ -z "$configuration" ]; then
 fi
 msbuild_args[${#msbuild_args[*]}]="-p:Configuration=$configuration"
 
+# Set verbosity
+echo "Setting msbuild verbosity to $verbosity"
+msbuild_args[${#msbuild_args[*]}]="-verbosity:$verbosity"
+
 # Initialize global variables need to be set before the import of Arcade is imported
 restore=$run_restore
 

Diff for: .azure/pipelines/jobs/default-build.yml

@@ -124,6 +124,7 @@ docker build "$(dirname "$dockerfile")" \
     --build-arg "USER=$(whoami)" \
     --build-arg "USER_ID=$(id -u)" \
     --build-arg "GROUP_ID=$(id -g)" \
+    --build-arg "WORKDIR=$DIR" \
     --tag $tagname \
     -f "$dockerfile"
 
@@ -138,7 +139,7 @@ docker run \
     -e BUILD_SOURCEBRANCH \
     -e DOTNET_CLI_TELEMETRY_OPTOUT \
     -e Configuration \
-    -v "$DIR:/code/build" \
+    -v "$DIR:$DIR" \
     ${docker_args[@]+"${docker_args[@]}"} \
     $tagname \
     ./build.sh \

Diff for: build.sh

@@ -3,22 +3,36 @@ Artifacts
 
 Building this repo produces build artifacts in the directory structure described below. Build outputs are organized into logical groups based on artifact type and the intended usage of the artifacts.
 
+See also https://github.com/dotnet/arcade/blob/master/Documentation/ArcadeSdk.md This repo follows _most_ of the conventions described there.
+
 ```
 artifacts/
   installers/
     $(Configuration)/
-        *.msi            = Windows installers
-        *.deb, *.rpm     = Linux installers
-        *.zip, *.tar.gz  = archives versions of installers
+      *.msi            = Windows installers
+      *.deb, *.rpm     = Linux installers
+      *.zip, *.tar.gz  = archives versions of installers
+  log/
+    runningProcesses*.txt = Process list from just before build completed
+    runningProcesses*.bak = Process list from two minutes before runningProcesses*.txt files were written
+    *.binlog           = Binary logs for a few build phases e.g. site extension build
+    **/
+      *.log            = Log files for test runs and individual tests
+    $(Configuration)/
+      *.binlog         = Binary logs for most build phases
   packages/
     $(Configuration)/
-        Shipping/        = Packages which are intended for use by customers. These, along with installers, represent the 'product'.
-            *.nupkg      = NuGet packages which ship to nuget.org
-            *.jar        = Java packages which ship to Maven Central and others
-            *.tgz        = NPM packages which ship to npmjs.org
-        NonShipping/
-            *.nupkg      = NuGet packages for internal use only. Used to hand off bits to Microsoft partner teams. Not intended for use by customers.
+      Shipping/        = Packages which are intended for use by customers. These, along with installers, represent the 'product'.
+        *.nupkg        = NuGet packages which ship to nuget.org
+        *.jar          = Java packages which ship to Maven Central and others
+        *.tgz          = NPM packages which ship to npmjs.org
+      NonShipping/
+        *.nupkg        = NuGet packages for internal use only. Used to hand off bits to Microsoft partner teams. Not intended for use by customers.
+  symbols/
+    $(Configuration)/
+      $(TargetFramework)/
+        *.pdb          = Loose symbol files for symbol server publication. Special cases where *.symbols.nupkg packaging is cumbersome.
   VSSetup/
     $(Configuration)/
-        *.vsix           = Visual Studio extensions
+      *.vsix           = Visual Studio extensions. None currently exist.
 ```

Diff for: dockerbuild.sh

@@ -17,15 +17,4 @@
       SharedFrameworkTargetFramework="netcoreapp$(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion)" />
   </Target>
 
-  <Target Name="GenerateBuildAssetManifest" AfterTargets="Pack" Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(DotNetBuildFromSource)' != 'true'">
-    <!-- Generate build manifests. These manifests are used by Maestro and the Build Asset Registry to flow dependencies to other repos. -->
-    <MSBuild Projects="$(MSBuildThisFileDirectory)tools\Maestro\Maestro.csproj"
-          Targets="Restore"
-          Properties="__DummyTarget=Restore" />
-
-    <MSBuild Projects="$(MSBuildThisFileDirectory)tools\Maestro\Maestro.csproj"
-             Targets="GenerateBuildAssetManifest"
-             Properties="__DummyTarget=GenerateBuildAssetManifest" />
-  </Target>
-
 </Project>

Diff for: docs/Artifacts.md

@@ -0,0 +1,11 @@
+<Project>
+  <!-- The one use of ArtifactsDir in Publish.proj adds an additional slash, confusing itself. -->
+  <PropertyGroup Condition=" HasTrailingSlash('$(ArtifactsDir)') ">
+    <ArtifactsDir>$(ArtifactsDir.Substring(0, $([MSBuild]::Subtract($(ArtifactsDir.Length), 1))))</ArtifactsDir>
+  </PropertyGroup>
+
+  <!-- Include our "loose" PDBs when publishing. -->
+  <ItemGroup>
+    <FilesToPublishToSymbolServer Include="$(SymbolsOutputPath)**\*.pdb" />
+  </ItemGroup>
+</Project>

Diff for: eng/AfterSolutionBuild.targets

@@ -21,6 +21,7 @@
     -->
     <FileExtensionSignInfo Include=".jar" CertificateName="MicrosoftJARSHA2" />
     <FileExtensionSignInfo Include=".ps1;.psd1;.psm1;.psc1" CertificateName="Microsoft400" />
+    <FileExtensionSignInfo Include=".dll;.exe" CertificateName="Microsoft400" />
     <FileExtensionSignInfo Include=".nupkg" CertificateName="NuGet" />
     <FileExtensionSignInfo Include=".vsix" CertificateName="VsixSHA2" />
     <FileExtensionSignInfo Include=".zip" CertificateName="None" />
@@ -84,6 +85,13 @@
     <!-- Exclude the apphost because this is expected to be code-signed by customers after the SDK modifies it. -->
     <FileSignInfo Include="apphost.exe" CertificateName="None" />
 
+    <SignValidationExclusionList Include="**/*.js" />
+    <SignValidationExclusionList Include="**/*.otf" />
+    <SignValidationExclusionList Include="**/*.ttf" />
+    <SignValidationExclusionList Include="**/ancm.mof" />
+    <SignValidationExclusionList Include="**/apphost.exe" />
+    <SignValidationExclusionList Include="**/wix*" />
+
     <!--
       These files should already be signed by the .NET Core team. They have to be listed again here because we recreate a redistributable which includes the Microsoft.NETCore.App runtime.
       List all combinations of Windows RID's because CI may build multiple combinations of artitectures on the same machine.

Diff for: eng/Publishing.props

@@ -2,8 +2,9 @@ FROM microsoft/dotnet:2.1.0-preview1-runtime-deps-alpine
 ARG USER
 ARG USER_ID
 ARG GROUP_ID
+ARG WORKDIR
 
-WORKDIR /code/build
+WORKDIR ${WORKDIR}
 RUN mkdir -p "/home/$USER" && chown "${USER_ID}:${GROUP_ID}" "/home/$USER"
 ENV HOME "/home/$USER"
 

Diff for: eng/Signing.props

@@ -3,8 +3,9 @@ FROM microsoft/dotnet:2.1-runtime-deps-bionic
 ARG USER
 ARG USER_ID
 ARG GROUP_ID
+ARG WORKDIR
 
-WORKDIR /code/build
+WORKDIR ${WORKDIR}
 RUN mkdir -p "/home/$USER" && chown "${USER_ID}:${GROUP_ID}" "/home/$USER"
 ENV HOME "/home/$USER"
 

Diff for: eng/docker/alpine.Dockerfile

@@ -5,8 +5,9 @@ FROM mcr.microsoft.com/dotnet-buildtools/prereqs:rhel-7-rpmpkg-e1b4a89-201753110
 ARG USER
 ARG USER_ID
 ARG GROUP_ID
+ARG WORKDIR
 
-WORKDIR /code/build
+WORKDIR ${WORKDIR}
 
 RUN useradd -m ${USER} --uid ${USER_ID} -g root
 RUN echo '${USER} ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

Diff for: eng/docker/bionic.Dockerfile

@@ -2,8 +2,9 @@ FROM mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine
 ARG USER
 ARG USER_ID
 ARG GROUP_ID
+ARG WORKDIR
 
-WORKDIR /code/build
+WORKDIR ${WORKDIR}
 RUN mkdir -p "/home/$USER" && chown "${USER_ID}:${GROUP_ID}" "/home/$USER"
 ENV HOME "/home/$USER"
 

Diff for: eng/docker/rhel.Dockerfile

@@ -0,0 +1,5 @@
+<!-- eng\common\build.ps1 fails without a project to build, so we give it this empty project to satisfy its desires -->
+<Project>
+  <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+  <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+</Project>