diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index 0c33783282f5..397d97c0fecb 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -18,12 +18,18 @@ pr: include: - '*' -# Choose whether to skip tests when running pipeline manually. parameters: +# Choose whether to skip tests when running pipeline manually. - name: skipTests default: false displayName: Skip tests? type: boolean +# Choose whether to test source indexing. Ignored in public builds. +# Will cause inaccessible links on https://source.dot.net/ unless commits are also available in GitHub. +- name: testSourceIndexing + default: false + displayName: Test source indexing? !Danger! see comments in YAML. + type: boolean variables: - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE @@ -72,6 +78,12 @@ variables: value: -ExcludeCIBinaryLog - name: WindowsArm64InstallersLogArgs value: -ExcludeCIBinaryLog + # Variables for source indexing afterBuild step and job. + - name: sourceIndexPackageVersion + value: 1.0.1-20210614.1 + - name: sourceIndexPackageSource + value: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + - group: source-dot-net stage1 variables - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - name: _BuildArgs value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)' @@ -126,7 +138,7 @@ stages: displayName: Build jobs: # Code check - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), in(variables['Build.Reason'], 'Manual')) }}: + - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual')) }}: - template: jobs/default-build.yml parameters: jobName: Code_check @@ -150,10 +162,10 @@ stages: agentOs: Windows steps: - script: "echo ##vso[build.addbuildtag]daily-build" - condition: and(notin(variables['Build.Reason'], 'PullRequest'), notin(variables['DotNetFinalVersionKind'], 'release', 'prerelease')) + condition: and(notin(variables['Build.Reason'], 'PullRequest'), notin(variables.DotNetFinalVersionKind, 'release', 'prerelease')) displayName: 'Set CI tags' - script: "echo ##vso[build.addbuildtag]release-candidate" - condition: and(notin(variables['Build.Reason'], 'PullRequest'), in(variables['DotNetFinalVersionKind'], 'release', 'prerelease')) + condition: and(notin(variables['Build.Reason'], 'PullRequest'), in(variables.DotNetFinalVersionKind, 'release', 'prerelease')) displayName: 'Set CI tags' # !!! NOTE !!! Some of these steps have disabled code signing. @@ -241,7 +253,7 @@ stages: displayName: Publish # A few files must also go to the VS package feed. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables['PostBuildSign'], 'true')) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.PostBuildSign, 'true')) }}: - task: NuGetCommand@2 displayName: Push Visual Studio packages inputs: @@ -271,12 +283,26 @@ stages: -pack -noBuildNodeJS -noBuildJava + -binaryLog /p:DotNetSignType=$(_SignType) /p:OnlyPackPlatformSpecificPackages=true /p:AssetManifestFileName=aspnetcore-win-arm.xml $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) + ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + afterBuild: + - powershell: . $(Build.SourcesDirectory)/activate.ps1; + dotnet tool install BinLogToSln + --version $(SourceIndexPackageVersion) + --add-source $(SourceIndexPackageSource) + --tool-path $(Build.SourcesDirectory)/.tools; + $(Build.SourcesDirectory)/.tools/BinLogToSln + -i $(Build.SourcesDirectory)/artifacts/log/Release/Build.binlog + -r $(Build.SourcesDirectory) + -n $(Build.Repository.Name) + -o $(Build.ArtifactStagingDirectory)/sourceIndex/ + displayName: Process binary log into indexable sln installNodeJs: false installJdk: false artifacts: @@ -286,6 +312,9 @@ stages: includeForks: true - name: Windows_arm_Packages path: artifacts/packages/ + - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + - name: Windows_arm_SourceIndex + path: $(Build.ArtifactStagingDirectory)/sourceIndex/ # Build Windows ARM64 - template: jobs/default-build.yml @@ -337,7 +366,7 @@ stages: displayName: Build Arm64 Installers # A few files must also go to the VS package feed. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables['PostBuildSign'], 'true')) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.PostBuildSign, 'true')) }}: - task: NuGetCommand@2 displayName: Push Visual Studio packages inputs: @@ -374,7 +403,7 @@ stages: - name: MacOS_arm64_Packages path: artifacts/packages/ - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - template: jobs/codesign-xplat.yml parameters: inputName: MacOS_arm64 @@ -405,7 +434,7 @@ stages: - name: MacOS_x64_Packages path: artifacts/packages/ - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - template: jobs/codesign-xplat.yml parameters: inputName: MacOS_x64 @@ -430,37 +459,19 @@ stages: $(_BuildArgs) $(_InternalRuntimeDownloadArgs) displayName: Run build.sh - - script: | - git clean -xfd src/**/obj/ - ./dockerbuild.sh bionic \ - --ci \ - --nobl \ - --arch x64 \ - --build-installers \ - --no-build-deps \ - --no-build-nodejs \ - -p:OnlyPackPlatformSpecificPackages=true \ - -p:BuildRuntimeArchive=false \ - -p:LinuxInstallerType=deb \ - $(_BuildArgs) \ - $(_InternalRuntimeDownloadArgs) + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) displayName: Build Debian installers - - script: | - git clean -xfd src/**/obj/ - ./dockerbuild.sh rhel \ - --ci \ - --nobl \ - --arch x64 \ - --build-installers \ - --no-build-deps \ - --no-build-nodejs \ - -p:OnlyPackPlatformSpecificPackages=true \ - -p:BuildRuntimeArchive=false \ - -p:LinuxInstallerType=rpm \ - -p:AssetManifestFileName=aspnetcore-Linux_x64.xml \ - $(_BuildArgs) \ - $(_PublishArgs) \ - $(_InternalRuntimeDownloadArgs) + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) displayName: Build RPM installers installNodeJs: false installJdk: false @@ -472,7 +483,7 @@ stages: - name: Linux_x64_Packages path: artifacts/packages/ - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - template: jobs/codesign-xplat.yml parameters: inputName: Linux_x64 @@ -504,7 +515,7 @@ stages: - name: Linux_arm_Packages path: artifacts/packages/ - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - template: jobs/codesign-xplat.yml parameters: inputName: Linux_arm @@ -536,7 +547,7 @@ stages: - name: Linux_arm64_Packages path: artifacts/packages/ - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - template: jobs/codesign-xplat.yml parameters: inputName: Linux_arm64 @@ -571,7 +582,7 @@ stages: - name: Linux_musl_x64_Packages path: artifacts/packages/ - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - template: jobs/codesign-xplat.yml parameters: inputName: Linux_musl_x64 @@ -606,7 +617,7 @@ stages: - name: Linux_musl_arm_Packages path: artifacts/packages/ - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - template: jobs/codesign-xplat.yml parameters: inputName: Linux_musl_arm @@ -641,12 +652,12 @@ stages: - name: Linux_musl_arm64_Packages path: artifacts/packages/ - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - template: jobs/codesign-xplat.yml parameters: inputName: Linux_musl_arm64 - - ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), in(variables['Build.Reason'], 'Manual'))) }}: + - ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual'))) }}: # Test jobs - template: jobs/default-build.yml parameters: @@ -755,7 +766,7 @@ stages: buildScript: './eng/build.sh $(_PublishArgs) --no-build-repo-tasks' skipPublishValidation: true - # Publish to the BAR + # Publish to the BAR and perform source indexing. Wait until everything else is done. - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/job/publish-build-assets.yml parameters: @@ -763,7 +774,7 @@ stages: - Windows_build - Windows_arm_build - Windows_arm64_build - - ${{ if ne(variables['PostBuildSign'], 'true') }}: + - ${{ if ne(variables.PostBuildSign, 'true') }}: - CodeSign_Xplat_MacOS_arm64 - CodeSign_Xplat_MacOS_x64 - CodeSign_Xplat_Linux_x64 @@ -772,7 +783,7 @@ stages: - CodeSign_Xplat_Linux_musl_x64 - CodeSign_Xplat_Linux_musl_arm - CodeSign_Xplat_Linux_musl_arm64 - - ${{ if eq(variables['PostBuildSign'], 'true') }}: + - ${{ if eq(variables.PostBuildSign, 'true') }}: - MacOs_arm64_build - MacOs_x64_build - Linux_x64_build @@ -781,12 +792,81 @@ stages: - Linux_musl_x64_build - Linux_musl_arm_build - Linux_musl_arm64_build - # In addition to the dependencies above, ensure the build was successful overall. + # In addition to the dependencies above that provide assets, ensure the build was successful overall. + - ${{ if in(variables['Build.Reason'], 'Manual') }}: + - Code_check + - ${{ if ne(parameters.skipTests, 'true') }}: + - Windows_Test + - MacOS_Test + - Linux_Test + - Helix_x64 - Source_Build_Managed pool: vmImage: vs2017-win2016 publishUsingPipelines: ${{ variables._PublishUsingPipelines }} enablePublishBuildArtifacts: true # publish artifacts/log files + - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + - job: SourceIndexUpload + displayName: Upload indexable solution + dependsOn: + - Windows_arm_build + # In addition to the dependency above that provides assets, ensure the build was successful overall. + - Windows_build + - Windows_arm64_build + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - CodeSign_Xplat_MacOS_arm64 + - CodeSign_Xplat_MacOS_x64 + - CodeSign_Xplat_Linux_x64 + - CodeSign_Xplat_Linux_arm + - CodeSign_Xplat_Linux_arm64 + - CodeSign_Xplat_Linux_musl_x64 + - CodeSign_Xplat_Linux_musl_arm + - CodeSign_Xplat_Linux_musl_arm64 + - ${{ if eq(variables.PostBuildSign, 'true') }}: + - MacOs_arm64_build + - MacOs_x64_build + - Linux_x64_build + - Linux_arm_build + - Linux_arm64_build + - Linux_musl_x64_build + - Linux_musl_arm_build + - Linux_musl_arm64_build + - ${{ if in(variables['Build.Reason'], 'Manual') }}: + - Code_check + - ${{ if ne(parameters.skipTests, 'true') }}: + - Windows_Test + - MacOS_Test + - Linux_Test + - Helix_x64 + - Source_Build_Managed + pool: + name: NetCore1ESPool-Internal + # Visual Studio Enterprise - no BuildTools agents exist internally and job must run on Windows + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: + - task: DownloadBuildArtifacts@0 + inputs: + artifactName: Windows_arm_SourceIndex + cleanDestinationFolder: true + # Ignore repository's global.json and any existing .NET SDK. + - task: UseDotNet@2 + displayName: Use .NET Core sdk 3.1 + inputs: + packageType: sdk + version: 3.1.x + installationPath: $(Agent.TempDirectory)/.dotnet + workingDirectory: $(Agent.TempDirectory) + - script: $(Agent.TempDirectory)/.dotnet/dotnet tool install UploadIndexStage1 + --version $(SourceIndexPackageVersion) + --add-source $(SourceIndexPackageSource) + --tool-path $(Agent.TempDirectory)/.tools && + $(Agent.TempDirectory)/.tools/UploadIndexStage1 + -i $(System.ArtifactsDirectory)/Windows_arm_SourceIndex/ + -n $(Build.Repository.Name) + displayName: Upload indexable solution + workingDirectory: $(Agent.TempDirectory) + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/post-build/post-build.yml diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index b8eaacd63816..025cf7a3a6fc 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -280,7 +280,11 @@ jobs: condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true'))) continueOnError: true inputs: - pathtoPublish: $(Build.SourcesDirectory)/${{ artifact.path }} + # Assume runtime variable values are absolute paths already. + ${{ if startsWith(artifact.path, '$(') }}: + pathToPublish: ${{ artifact.path }} + ${{ if not(startsWith(artifact.path, '$(')) }}: + pathToPublish: $(Build.SourcesDirectory)/${{ artifact.path }} ${{ if eq(artifact.name, '') }}: artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration) ${{ if ne(artifact.name, '') }}: