Backport pipeline changes (#8780)

* Backport changes to get CI working in runtime repository.

* /root/coresetup->/root/runtime

* Delete broken RPM symlink that might still be around if the RPM packaging job failed.

* Add back old Documentation folder excludes with a more specific include for the manpages folder.

* Try other variable format for template path.

* Treat subsetArg as a parameter with a default value instead of as a variable.

* Pass subset arg to template.

* Try other variable format.

* Try another variant.

* Switch back variable format now that I got a different (seemingly actionable) error.

* Fix passing the subset arg.

Author: jkoritzinsky

azure-pipelines.yml

@@ -1,12 +1,20 @@
 trigger:
-  batch: true
   branches:
     include:
     - master
     - release/3.*
   paths:
+    include:
+    - '*'
+    - docs/installer/manpages/*
+    - Documentation/manpages/*
     exclude:
     - Documentation/*
+    - src/coreclr/*
+    - src/libraries/*
+    - eng/pipelines/coreclr/*
+    - eng/pipelines/libraries/*
+    - docs/*
     - README.md
     - CONTRIBUTING.md
     - LICENSE.TXT
@@ -18,8 +26,17 @@ pr:
     - master
     - release/3.*
   paths:
+    include:
+    - '*'
+    - docs/installer/manpages/*
+    - Documentation/manpages/*
     exclude:
     - Documentation/*
+    - src/coreclr/*
+    - src/libraries/*
+    - eng/pipelines/coreclr/*
+    - eng/pipelines/libraries/*
+    - docs/*
     - README.md
     - CONTRIBUTING.md
     - LICENSE.TXT
@@ -53,11 +70,22 @@ variables:
     - name: SignType
       value: $[ coalesce(variables.OfficialSignType, 'real') ]
 
+  - ${{ if contains(variables['Build.DefinitionName'], 'runtime') }}:
+    - name: pipelinesPath
+      value: /eng/pipelines/installer
+    - name: buildScriptFileName
+      value: installer
+  - ${{ if not(contains(variables['Build.DefinitionName'], 'runtime')) }}:
+    - name: pipelinesPath
+      value: /eng/pipelines
+    - name: buildScriptFileName
+      value: build
+
 stages:
 - stage: Build
   jobs:
   # -------- Build Bash legs (Linux and FreeBSD) --------
-  - template: /eng/pipelines/jobs/bash-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml
     parameters:
       crossBuild: true
       name: Linux_Arm
@@ -67,7 +95,7 @@ stages:
       skipTests: true
       targetArchitecture: arm
 
-  - template: /eng/pipelines/jobs/bash-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml
     parameters:
       crossBuild: true
       name: Linux_Arm64
@@ -79,7 +107,7 @@ stages:
 
   # # Tizen build only for PR build
   # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-  #   - template: /eng/pipelines/jobs/bash-build.yml
+  #   - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml
   #     parameters:
   #       additionalMSBuildArgs: /p:OverridePackageSource=https:%2F%2Ftizen.myget.org/F/dotnet-core/api/v3/index.json /p:OutputRid=tizen.5.0.0-armel
   #       additionalRunArgs: -e ROOTFS_DIR=/crossrootfs/armel.tizen.build
@@ -91,7 +119,7 @@ stages:
   #       skipTests: true
   #       targetArchitecture: armel
 
-  - template: /eng/pipelines/jobs/bash-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml
     parameters:
       additionalMSBuildArgs: /p:OutputRid=linux-musl-arm64
       crossBuild: true
@@ -102,15 +130,15 @@ stages:
       skipTests: true
       targetArchitecture: arm64
 
-  - template: /eng/pipelines/jobs/bash-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml
     parameters:
       additionalMSBuildArgs: /p:OutputRid=linux-musl-x64
       name: Linux_x64_Alpine39
       dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-WithNode-0fc54a3-20190918214015
       portableBuild: false
       targetArchitecture: x64
 
-  - template: /eng/pipelines/jobs/bash-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml
     parameters:
       name: Linux_x64_glibc
       dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-50f0d02-20190918214028
@@ -129,7 +157,7 @@ stages:
       portableBuild: true
       targetArchitecture: x64
 
-  - template: /eng/pipelines/jobs/bash-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml
     parameters:
       additionalMSBuildArgs: /p:OutputRid=rhel.6-x64
       name: Linux_x64_Rhel6
@@ -138,34 +166,34 @@ stages:
       targetArchitecture: x64
 
   # -------- Build OSX (macOS) leg --------
-  - template: /eng/pipelines/jobs/osx-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/osx-build.yml
     parameters:
       name: OSX
 
   # -------- Build Windows legs --------
   # Windows Arm
-  - template: /eng/pipelines/jobs/windows-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/windows-build.yml
     parameters:
       name: Windows_Arm
       skipTests: true
       targetArchitecture: arm
 
   # Windows Arm64
-  - template: /eng/pipelines/jobs/windows-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/windows-build.yml
     parameters:
       name: Windows_Arm64
       skipTests: true
       targetArchitecture: arm64
 
   # Windows x64
-  - template: /eng/pipelines/jobs/windows-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/windows-build.yml
     parameters:
       name: Windows_x64
       publishRidAgnosticPackages: true
       targetArchitecture: x64
 
   # Windows x86
-  - template: /eng/pipelines/jobs/windows-build.yml
+  - template: ${{ variables['pipelinesPath'] }}/jobs/windows-build.yml
     parameters:
       name: Windows_x86
       targetArchitecture: x86
@@ -176,7 +204,7 @@ stages:
     dependsOn: Build
     jobs:
     # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing.
-    - template: /eng/pipelines/jobs/prepare-signed-artifacts.yml
+    - template: ${{ variables['pipelinesPath'] }}/jobs/prepare-signed-artifacts.yml
       parameters:
         PublishRidAgnosticPackagesFromJobName: Windows_x64
     # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact.
@@ -188,7 +216,7 @@ stages:
           name: NetCoreInternal-Pool
           queue: buildpool.windows.10.amd64.vs2017
 
-  - template: /eng/pipelines/stages/publish.yml
+  - template: ${{ variables['pipelinesPath'] }}/stages/publish.yml
     parameters:
       # Publish channel configuration. The publish template wraps Arcade publishing and adds some
       # extras that aren't implemented in Arcade yet.

eng/pipelines/jobs/bash-build.yml

@@ -40,8 +40,8 @@ jobs:
       ${{ if ne(parameters.name, 'FreeBSD_x64')}}:
         RunArguments: >-
           docker run --privileged --rm
-          -v "$(Build.SourcesDirectory):/root/coresetup"
-          -w="/root/coresetup"
+          -v "$(Build.SourcesDirectory):/root/runtime"
+          -w="/root/runtime"
           $(PreserveNuGetAuthDockerArgs)
           ${{ parameters.additionalRunArgs }}
           ${{ parameters.dockerImage }}
@@ -50,12 +50,15 @@ jobs:
         RunArguments: export DotNetBootstrapCliTarPath=/dotnet-sdk-freebsd-x64.tar &&
 
       ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-        BuildScript: ./eng/install-nuget-credprovider-then-build.sh
-        MSBuildScript: /root/coresetup/eng/install-nuget-credprovider-then-msbuild.sh
+        ${{ if not(contains(variables['Build.DefinitionName'], 'runtime')) }}:
+          BuildScript: ./eng/install-nuget-credprovider-then-build.sh
+        ${{ if contains(variables['Build.DefinitionName'], 'runtime') }}:
+          BuildScript: ./eng/install-nuget-credprovider-then-build.sh --subsetCategory installer
+        MSBuildScript: /root/runtime/eng/install-nuget-credprovider-then-msbuild.sh
 
       ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}:
-        BuildScript: ./build.sh
-        MSBuildScript: /root/coresetup/eng/common/msbuild.sh
+        BuildScript: ./$(buildScriptFileName).sh
+        MSBuildScript: /root/runtime/eng/common/msbuild.sh
 
       CommonMSBuildArgs: >-
         /p:Configuration=$(_BuildConfig)
@@ -89,11 +92,16 @@ jobs:
 
       DockerRunMSBuild: >-
         docker run
-        -v $(Build.SourcesDirectory):/root/coresetup
+        -v $(Build.SourcesDirectory):/root/runtime
         -v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/
-        -w=/root/coresetup
+        -w=/root/runtime
         $(PreserveNuGetAuthDockerArgs)
 
+      ${{ if contains(variables['Build.DefinitionName'], 'runtime') }}:
+        installersSubsetArg: --subset Installers
+      ${{ if not(contains(variables['Build.DefinitionName'], 'runtime')) }}:
+        installersSubsetArg: /p:Subset=Installer
+
     steps:
 
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -139,6 +147,7 @@ jobs:
             distroRid: ${{ packageBuild.imageRid }}
             image: ${{ packageBuild.image }}
             packageStepDescription: Runtime Deps, Runtime, Framework Packs installers
+            subsetArg: $(installersSubsetArg)
 
         - ${{ each rid in packageBuild.rids }}:
           # Build distro-dependent packages.
@@ -149,6 +158,7 @@ jobs:
               outputRidArg: /p:OutputRid=${{ rid }}-${{ parameters.targetArchitecture }}
               packageStepDescription: Runtime Deps installers
               packagingArgs: /p:BuildDistroIndependentInstallers=false
+              subsetArg: $(installersSubsetArg)
 
     # Files may be owned by root because builds don't set user ID. The next AzDO step runs 'find' in
     # the source tree, which fails due to permissions in the 'NetCore*-Int-Pool' queues. This step

eng/pipelines/jobs/osx-build.yml

@@ -26,7 +26,7 @@ jobs:
     - task: NuGetAuthenticate@0
 
   - script: >-
-      $(Build.SourcesDirectory)/build.sh --ci --test
+      $(Build.SourcesDirectory)/$(buildScriptFileName).sh --ci --test
       /p:OfficialBuildId=$(OfficialBuildId)
       /p:StripSymbols=true
       $(CommonMSBuildArgs)

eng/pipelines/jobs/steps/build-linux-package.yml

@@ -5,6 +5,7 @@ parameters:
   outputRidArg: ''
   packageStepDescription: null
   packagingArgs: ''
+  subsetArg: ''
 
 steps:
 - ${{ if eq(parameters.buildTraversalBuildDependencies, true) }}:
@@ -13,7 +14,7 @@ steps:
       df -h
       $(DockerRunMSBuild) ${{ parameters.image }} $(MSBuildScript) \
         --ci \
-        /root/coresetup/tools-local/tasks/installer.tasks/installer.tasks.csproj \
+        /root/runtime/tools-local/tasks/installer.tasks/installer.tasks.csproj \
         /t:Restore /t:Build /t:CreateHostMachineInfoFile \
         $(CommonMSBuildArgs) \
         /bl:msbuild.${{ parameters.distroRid }}.traversaldependencies.binlog
@@ -24,12 +25,12 @@ steps:
     df -h
     $(DockerRunMSBuild) ${{ parameters.image }} $(BuildScript) \
       --ci \
-      /p:Subset=Installer \
+      ${{ parameters.subsetArg }} \
       /p:UsePrebuiltPortableBinariesForInstallers=true \
       $(PackagePortableBitsArgs) \
       /p:GenerateProjectInstallers=true \
       ${{ parameters.packagingArgs }} \
       $(CommonMSBuildArgs) \
       ${{ parameters.outputRidArg }} \
-      /bl:msbuild.${{ parameters.distroRid }}.installers.binlog
+      /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.distroRid }}.installers.binlog
   displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.distroRid }}

eng/pipelines/jobs/steps/upload-job-artifacts.yml

@@ -35,6 +35,13 @@ steps:
   continueOnError: true
   condition: always()
 
+- task: DeleteFiles@1
+  displayName: Delete broken symlinks.
+  inputs:
+    SourceFolder: '$(Build.SourcesDirectory)'
+    Contents: artifacts/obj/rhel.7-x64.Debug/packages/dotnet-host/5.0.0/rpmLayoutDirectory/package_root/usr/bin/dotnet
+  condition: succeededOrFailed()
+
 - task: CopyFiles@2
   displayName: Prepare BuildLogs staging directory
   inputs:

eng/pipelines/jobs/windows-build.yml

@@ -58,7 +58,7 @@ jobs:
       displayName: Clear NuGet http cache (if exists)
 
     - script: >-
-        build.cmd -ci -test
+        $(buildScriptFileName).cmd -ci -test
         $(CommonMSBuildArgs)
         $(MsbuildSigningArguments)
       displayName: Build

eng/pipelines/stages/publish.yml

@@ -40,7 +40,7 @@ stages:
     variables:
     - template: /eng/common/templates/post-build/common-variables.yml
     jobs:
-    - template: /eng/pipelines/jobs/run-publish-project.yml
+    - template: $(pipelinesPath)/jobs/run-publish-project.yml
       parameters:
         projectName: publish-final
         dependency: ${{ dependency }}

src/corehost/build.sh

@@ -268,7 +268,7 @@ if [ $__CrossBuild == 1 ]; then
     fi
     export TARGET_BUILD_ARCH=$__build_arch_lowcase
     export __DistroRid=$__rid_plat
-    cmake "$DIR" -G "Unix Makefiles" $__cmake_defines -DCLI_CMAKE_HOST_VER=$__host_ver -DCLI_CMAKE_COMMON_HOST_VER=$__apphost_ver -DCLI_CMAKE_HOST_FXR_VER=$__fxr_ver -DCLI_CMAKE_HOST_POLICY_VER=$__policy_ver -DCLI_CMAKE_PKG_RID=$__base_rid -DCLI_CMAKE_COMMIT_HASH=$__commit_hash -DCMAKE_INSTALL_PREFIX=$__cmake_bin_prefix -DCMAKE_TOOLCHAIN_FILE=$DIR/../../cross/toolchain.cmake
+    cmake "$DIR" -G "Unix Makefiles" $__cmake_defines -DCLI_CMAKE_HOST_VER=$__host_ver -DCLI_CMAKE_COMMON_HOST_VER=$__apphost_ver -DCLI_CMAKE_HOST_FXR_VER=$__fxr_ver -DCLI_CMAKE_HOST_POLICY_VER=$__policy_ver -DCLI_CMAKE_PKG_RID=$__base_rid -DCLI_CMAKE_COMMIT_HASH=$__commit_hash -DCMAKE_INSTALL_PREFIX=$__cmake_bin_prefix -DCMAKE_TOOLCHAIN_FILE=$RootRepo/cross/toolchain.cmake
 else
     cmake "$DIR" -G "Unix Makefiles" $__cmake_defines -DCLI_CMAKE_HOST_VER=$__host_ver -DCLI_CMAKE_COMMON_HOST_VER=$__apphost_ver -DCLI_CMAKE_HOST_FXR_VER=$__fxr_ver -DCLI_CMAKE_HOST_POLICY_VER=$__policy_ver -DCLI_CMAKE_PKG_RID=$__base_rid -DCLI_CMAKE_COMMIT_HASH=$__commit_hash -DCMAKE_INSTALL_PREFIX=$__cmake_bin_prefix
 fi

src/signing/Directory.Build.props

@@ -4,9 +4,9 @@
 
   <PropertyGroup>
     <TargetFramework>$(NETCoreAppFramework)</TargetFramework>
+    <IsRuntimeRepository Condition="Exists('$(MSBuildThisFileDirectory)..\..\..\.dotnet-runtime-placeholder')">true</IsRuntimeRepository>
 
-    <!-- Skip signing steps by default for non-official builds. -->
-    <SkipSigning Condition="'$(SkipSigning)' == '' and '$(OfficialBuild)' != 'true'">true</SkipSigning>
+    <!-- Skip signing steps by default for non-official builds. Skip signing in the runtime repository until we onboard unified signing scripts. -->
+    <SkipSigning Condition="('$(SkipSigning)' == '' and '$(OfficialBuild)' != 'true') or '$(IsRuntimeRepository)' == 'true'">true</SkipSigning>
   </PropertyGroup>
-
 </Project>