Add source-index build job to pipeline

.azure/pipelines/ci.yml

@@ -18,12 +18,18 @@ pr:
     include:
     - '*'
 
-# Choose whether to skip tests when running pipeline manually.
 parameters:
+# Choose whether to skip tests when running pipeline manually.
 - name: skipTests
   default: false
   displayName: Skip tests?
   type: boolean
+# Choose whether to test source indexing. Ignored in public builds.
+# Will cause inaccessible links on https://source.dot.net/ unless commits are also available in GitHub.
+- name: testSourceIndexing
+  default: false
+  displayName: Test source indexing? !Danger! see comments in YAML.
+  type: boolean
 
 variables:
 - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
@@ -72,6 +78,12 @@ variables:
     value: -ExcludeCIBinaryLog
   - name: WindowsArm64InstallersLogArgs
     value: -ExcludeCIBinaryLog
+  # Variables for source indexing afterBuild step and job.
+  - name: sourceIndexPackageVersion
+    value: 1.0.1-20210614.1
+  - name: sourceIndexPackageSource
+    value: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+  - group: source-dot-net stage1 variables
 - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
   - name: _BuildArgs
     value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)'
@@ -126,7 +138,7 @@ stages:
   displayName: Build
   jobs:
   # Code check
-  - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), in(variables['Build.Reason'], 'Manual')) }}:
+  - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual')) }}:
     - template: jobs/default-build.yml
       parameters:
         jobName: Code_check
@@ -150,10 +162,10 @@ stages:
       agentOs: Windows
       steps:
       - script: "echo ##vso[build.addbuildtag]daily-build"
-        condition: and(notin(variables['Build.Reason'], 'PullRequest'), notin(variables['DotNetFinalVersionKind'], 'release', 'prerelease'))
+        condition: and(notin(variables['Build.Reason'], 'PullRequest'), notin(variables.DotNetFinalVersionKind, 'release', 'prerelease'))
         displayName: 'Set CI tags'
       - script: "echo ##vso[build.addbuildtag]release-candidate"
-        condition: and(notin(variables['Build.Reason'], 'PullRequest'), in(variables['DotNetFinalVersionKind'], 'release', 'prerelease'))
+        condition: and(notin(variables['Build.Reason'], 'PullRequest'), in(variables.DotNetFinalVersionKind, 'release', 'prerelease'))
         displayName: 'Set CI tags'
 
       # !!! NOTE !!! Some of these steps have disabled code signing.
@@ -241,7 +253,7 @@ stages:
         displayName: Publish
 
       # A few files must also go to the VS package feed.
-      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables['PostBuildSign'], 'true')) }}:
+      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.PostBuildSign, 'true')) }}:
         - task: NuGetCommand@2
           displayName: Push Visual Studio packages
           inputs:
@@ -271,12 +283,26 @@ stages:
         -pack
         -noBuildNodeJS
         -noBuildJava
+        -binaryLog
         /p:DotNetSignType=$(_SignType)
         /p:OnlyPackPlatformSpecificPackages=true
         /p:AssetManifestFileName=aspnetcore-win-arm.xml
         $(_BuildArgs)
         $(_PublishArgs)
         $(_InternalRuntimeDownloadArgs)
+      ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+        afterBuild:
+        - powershell: . $(Build.SourcesDirectory)/activate.ps1;
+            dotnet tool install BinLogToSln
+            --version $(SourceIndexPackageVersion)
+            --add-source $(SourceIndexPackageSource)
+            --tool-path $(Build.SourcesDirectory)/.tools;
+            $(Build.SourcesDirectory)/.tools/BinLogToSln
+            -i $(Build.SourcesDirectory)/artifacts/log/Release/Build.binlog
+            -r $(Build.SourcesDirectory)
+            -n $(Build.Repository.Name)
+            -o $(Build.ArtifactStagingDirectory)/sourceIndex/
+          displayName: Process binary log into indexable sln
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -286,6 +312,9 @@ stages:
         includeForks: true
       - name: Windows_arm_Packages
         path: artifacts/packages/
+      - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+          - name: Windows_arm_SourceIndex
+            path: $(Build.ArtifactStagingDirectory)/sourceIndex/
 
   # Build Windows ARM64
   - template: jobs/default-build.yml
@@ -337,7 +366,7 @@ stages:
         displayName: Build Arm64 Installers
 
       # A few files must also go to the VS package feed.
-      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables['PostBuildSign'], 'true')) }}:
+      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.PostBuildSign, 'true')) }}:
         - task: NuGetCommand@2
           displayName: Push Visual Studio packages
           inputs:
@@ -374,7 +403,7 @@ stages:
       - name: MacOS_arm64_Packages
         path: artifacts/packages/
 
-  - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+  - ${{ if ne(variables.PostBuildSign, 'true') }}:
     - template: jobs/codesign-xplat.yml
       parameters:
         inputName: MacOS_arm64
@@ -405,7 +434,7 @@ stages:
       - name: MacOS_x64_Packages
         path: artifacts/packages/
 
-  - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+  - ${{ if ne(variables.PostBuildSign, 'true') }}:
     - template: jobs/codesign-xplat.yml
       parameters:
         inputName: MacOS_x64
@@ -430,37 +459,19 @@ stages:
             $(_BuildArgs)
             $(_InternalRuntimeDownloadArgs)
         displayName: Run build.sh
-      - script: |
-          git clean -xfd src/**/obj/
-          ./dockerbuild.sh bionic \
-            --ci \
-            --nobl \
-            --arch x64 \
-            --build-installers \
-            --no-build-deps \
-            --no-build-nodejs \
-            -p:OnlyPackPlatformSpecificPackages=true \
-            -p:BuildRuntimeArchive=false \
-            -p:LinuxInstallerType=deb \
-            $(_BuildArgs) \
-            $(_InternalRuntimeDownloadArgs)
+      - script: git clean -xfd src/**/obj/;
+          ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs
+          -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb
+          $(_BuildArgs)
+          $(_InternalRuntimeDownloadArgs)
         displayName: Build Debian installers
-      - script: |
-          git clean -xfd src/**/obj/
-          ./dockerbuild.sh rhel \
-            --ci \
-            --nobl \
-            --arch x64 \
-            --build-installers \
-            --no-build-deps \
-            --no-build-nodejs \
-            -p:OnlyPackPlatformSpecificPackages=true \
-            -p:BuildRuntimeArchive=false \
-            -p:LinuxInstallerType=rpm \
-            -p:AssetManifestFileName=aspnetcore-Linux_x64.xml \
-            $(_BuildArgs) \
-            $(_PublishArgs) \
-            $(_InternalRuntimeDownloadArgs)
+      - script: git clean -xfd src/**/obj/;
+          ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs
+          -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm
+          -p:AssetManifestFileName=aspnetcore-Linux_x64.xml
+          $(_BuildArgs)
+          $(_PublishArgs)
+          $(_InternalRuntimeDownloadArgs)
         displayName: Build RPM installers
       installNodeJs: false
       installJdk: false
@@ -472,7 +483,7 @@ stages:
       - name: Linux_x64_Packages
         path: artifacts/packages/
 
-  - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+  - ${{ if ne(variables.PostBuildSign, 'true') }}:
     - template: jobs/codesign-xplat.yml
       parameters:
         inputName: Linux_x64
@@ -504,7 +515,7 @@ stages:
       - name: Linux_arm_Packages
         path: artifacts/packages/
 
-  - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+  - ${{ if ne(variables.PostBuildSign, 'true') }}:
     - template: jobs/codesign-xplat.yml
       parameters:
         inputName: Linux_arm
@@ -536,7 +547,7 @@ stages:
       - name: Linux_arm64_Packages
         path: artifacts/packages/
 
-  - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+  - ${{ if ne(variables.PostBuildSign, 'true') }}:
     - template: jobs/codesign-xplat.yml
       parameters:
         inputName: Linux_arm64
@@ -571,7 +582,7 @@ stages:
       - name: Linux_musl_x64_Packages
         path: artifacts/packages/
 
-  - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+  - ${{ if ne(variables.PostBuildSign, 'true') }}:
     - template: jobs/codesign-xplat.yml
       parameters:
         inputName: Linux_musl_x64
@@ -606,7 +617,7 @@ stages:
       - name: Linux_musl_arm_Packages
         path: artifacts/packages/
 
-  - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+  - ${{ if ne(variables.PostBuildSign, 'true') }}:
     - template: jobs/codesign-xplat.yml
       parameters:
         inputName: Linux_musl_arm
@@ -641,12 +652,12 @@ stages:
       - name: Linux_musl_arm64_Packages
         path: artifacts/packages/
 
-  - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+  - ${{ if ne(variables.PostBuildSign, 'true') }}:
     - template: jobs/codesign-xplat.yml
       parameters:
         inputName: Linux_musl_arm64
 
-  - ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), in(variables['Build.Reason'], 'Manual'))) }}:
+  - ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual'))) }}:
     # Test jobs
     - template: jobs/default-build.yml
       parameters:
@@ -755,15 +766,15 @@ stages:
         buildScript: './eng/build.sh $(_PublishArgs) --no-build-repo-tasks'
         skipPublishValidation: true
 
-  # Publish to the BAR
+  # Publish to the BAR and perform source indexing. Wait until everything else is done.
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - template: /eng/common/templates/job/publish-build-assets.yml
       parameters:
         dependsOn:
           - Windows_build
           - Windows_arm_build
           - Windows_arm64_build
-          - ${{ if ne(variables['PostBuildSign'], 'true') }}:
+          - ${{ if ne(variables.PostBuildSign, 'true') }}:
             - CodeSign_Xplat_MacOS_arm64
             - CodeSign_Xplat_MacOS_x64
             - CodeSign_Xplat_Linux_x64
@@ -772,7 +783,7 @@ stages:
             - CodeSign_Xplat_Linux_musl_x64
             - CodeSign_Xplat_Linux_musl_arm
             - CodeSign_Xplat_Linux_musl_arm64
-          - ${{ if eq(variables['PostBuildSign'], 'true') }}:
+          - ${{ if eq(variables.PostBuildSign, 'true') }}:
             - MacOs_arm64_build
             - MacOs_x64_build
             - Linux_x64_build
@@ -781,12 +792,81 @@ stages:
             - Linux_musl_x64_build
             - Linux_musl_arm_build
             - Linux_musl_arm64_build
-          # In addition to the dependencies above, ensure the build was successful overall.
+          # In addition to the dependencies above that provide assets, ensure the build was successful overall.
+          - ${{ if in(variables['Build.Reason'], 'Manual') }}:
+            - Code_check
+            - ${{ if ne(parameters.skipTests, 'true') }}:
+              - Windows_Test
+              - MacOS_Test
+              - Linux_Test
+              - Helix_x64
           - Source_Build_Managed
         pool:
           vmImage: vs2017-win2016
         publishUsingPipelines: ${{ variables._PublishUsingPipelines }}
         enablePublishBuildArtifacts: true # publish artifacts/log files
+    - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+      - job: SourceIndexUpload
+        displayName: Upload indexable solution
+        dependsOn:
+          - Windows_arm_build
+          # In addition to the dependency above that provides assets, ensure the build was successful overall.
+          - Windows_build
+          - Windows_arm64_build
+          - ${{ if ne(variables.PostBuildSign, 'true') }}:
+            - CodeSign_Xplat_MacOS_arm64
+            - CodeSign_Xplat_MacOS_x64
+            - CodeSign_Xplat_Linux_x64
+            - CodeSign_Xplat_Linux_arm
+            - CodeSign_Xplat_Linux_arm64
+            - CodeSign_Xplat_Linux_musl_x64
+            - CodeSign_Xplat_Linux_musl_arm
+            - CodeSign_Xplat_Linux_musl_arm64
+          - ${{ if eq(variables.PostBuildSign, 'true') }}:
+            - MacOs_arm64_build
+            - MacOs_x64_build
+            - Linux_x64_build
+            - Linux_arm_build
+            - Linux_arm64_build
+            - Linux_musl_x64_build
+            - Linux_musl_arm_build
+            - Linux_musl_arm64_build
+          - ${{ if in(variables['Build.Reason'], 'Manual') }}:
+            - Code_check
+            - ${{ if ne(parameters.skipTests, 'true') }}:
+              - Windows_Test
+              - MacOS_Test
+              - Linux_Test
+              - Helix_x64
+          - Source_Build_Managed
+        pool:
+          name: NetCore1ESPool-Internal
+          # Visual Studio Enterprise - no BuildTools agents exist internally and job must run on Windows
+          demands: ImageOverride -equals Build.Server.Amd64.VS2019
+        steps:
+          - task: DownloadBuildArtifacts@0
+            inputs:
+              artifactName: Windows_arm_SourceIndex
+              cleanDestinationFolder: true
+          # Ignore repository's global.json and any existing .NET SDK.
+          - task: UseDotNet@2
+            displayName: Use .NET Core sdk 3.1
+            inputs:
+              packageType: sdk
+              version: 3.1.x
+              installationPath: $(Agent.TempDirectory)/.dotnet
+              workingDirectory: $(Agent.TempDirectory)
+          - script: $(Agent.TempDirectory)/.dotnet/dotnet tool install UploadIndexStage1
+              --version $(SourceIndexPackageVersion)
+              --add-source $(SourceIndexPackageSource)
+              --tool-path $(Agent.TempDirectory)/.tools &&
+              $(Agent.TempDirectory)/.tools/UploadIndexStage1
+              -i $(System.ArtifactsDirectory)/Windows_arm_SourceIndex/
+              -n $(Build.Repository.Name)
+            displayName: Upload indexable solution
+            workingDirectory: $(Agent.TempDirectory)
+            env:
+              BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
 
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - template: /eng/common/templates/post-build/post-build.yml

.azure/pipelines/jobs/default-build.yml

@@ -280,7 +280,11 @@ jobs:
         condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true')))
         continueOnError: true
         inputs:
-          pathtoPublish: $(Build.SourcesDirectory)/${{ artifact.path }}
+          # Assume runtime variable values are absolute paths already.
+          ${{ if startsWith(artifact.path, '$(') }}:
+            pathToPublish: ${{ artifact.path }}
+          ${{ if not(startsWith(artifact.path, '$(')) }}:
+            pathToPublish: $(Build.SourcesDirectory)/${{ artifact.path }}
           ${{ if eq(artifact.name, '') }}:
             artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration)
           ${{ if ne(artifact.name, '') }}: