Skip to content

Commit fc286da

Browse files
dougbudougbu
authored
Add source-index build job to pipeline (#37028)
- #33534 - copy source-index-stage1 template into ci.yml and split steps in two: - current template not well suited to reuse a binary log from another job - first, process Windows_arm_build binary log in that job - align binary log with source-indexing tool support i.e. always use Windows - much faster than Windows x64/x86 build - provides another binary log, even in internal builds - second, do source index upload after other jobs - do not update https://source.dot.net/ from failed builds - add `testSourceIndexing` parameter - use 1ES agent for source-index upload - `windows-latest` might work but prefer to protect binary log nits: - remove unnecessary backslashes in the Linux_x64_build job's `script`s - instead separate commands with a semicolon - confirm test build success before BAR updates in manual builds - merge a couple of `in(...)` expressions
1 parent 5d185c4 commit fc286da

File tree

2 files changed

+134
-50
lines changed

2 files changed

+134
-50
lines changed

‎.azure/pipelines/ci.yml‎

Lines changed: 129 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,18 @@ pr:
1818
include:
1919
- '*'
2020

21-
# Choose whether to skip tests when running pipeline manually.
2221
parameters:
22+
# Choose whether to skip tests when running pipeline manually.
2323
- name: skipTests
2424
default: false
2525
displayName: Skip tests?
2626
type: boolean
27+
# Choose whether to test source indexing. Ignored in public builds.
28+
# Will cause inaccessible links on https://source.dot.net/ unless commits are also available in GitHub.
29+
- name: testSourceIndexing
30+
default: false
31+
displayName: Test source indexing? !Danger! see comments in YAML.
32+
type: boolean
2733

2834
variables:
2935
- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
@@ -72,6 +78,12 @@ variables:
7278
value: -ExcludeCIBinaryLog
7379
- name: WindowsArm64InstallersLogArgs
7480
value: -ExcludeCIBinaryLog
81+
# Variables for source indexing afterBuild step and job.
82+
- name: sourceIndexPackageVersion
83+
value: 1.0.1-20210614.1
84+
- name: sourceIndexPackageSource
85+
value: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
86+
- group: source-dot-net stage1 variables
7587
- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
7688
- name: _BuildArgs
7789
value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)'
@@ -126,7 +138,7 @@ stages:
126138
displayName: Build
127139
jobs:
128140
# Code check
129-
- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), in(variables['Build.Reason'], 'Manual')) }}:
141+
- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual')) }}:
130142
- template: jobs/default-build.yml
131143
parameters:
132144
jobName: Code_check
@@ -150,10 +162,10 @@ stages:
150162
agentOs: Windows
151163
steps:
152164
- script: "echo ##vso[build.addbuildtag]daily-build"
153-
condition: and(notin(variables['Build.Reason'], 'PullRequest'), notin(variables['DotNetFinalVersionKind'], 'release', 'prerelease'))
165+
condition: and(notin(variables['Build.Reason'], 'PullRequest'), notin(variables.DotNetFinalVersionKind, 'release', 'prerelease'))
154166
displayName: 'Set CI tags'
155167
- script: "echo ##vso[build.addbuildtag]release-candidate"
156-
condition: and(notin(variables['Build.Reason'], 'PullRequest'), in(variables['DotNetFinalVersionKind'], 'release', 'prerelease'))
168+
condition: and(notin(variables['Build.Reason'], 'PullRequest'), in(variables.DotNetFinalVersionKind, 'release', 'prerelease'))
157169
displayName: 'Set CI tags'
158170

159171
# !!! NOTE !!! Some of these steps have disabled code signing.
@@ -241,7 +253,7 @@ stages:
241253
displayName: Publish
242254

243255
# A few files must also go to the VS package feed.
244-
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables['PostBuildSign'], 'true')) }}:
256+
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.PostBuildSign, 'true')) }}:
245257
- task: NuGetCommand@2
246258
displayName: Push Visual Studio packages
247259
inputs:
@@ -271,12 +283,26 @@ stages:
271283
-pack
272284
-noBuildNodeJS
273285
-noBuildJava
286+
-binaryLog
274287
/p:DotNetSignType=$(_SignType)
275288
/p:OnlyPackPlatformSpecificPackages=true
276289
/p:AssetManifestFileName=aspnetcore-win-arm.xml
277290
$(_BuildArgs)
278291
$(_PublishArgs)
279292
$(_InternalRuntimeDownloadArgs)
293+
${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
294+
afterBuild:
295+
- powershell: . $(Build.SourcesDirectory)/activate.ps1;
296+
dotnet tool install BinLogToSln
297+
--version $(SourceIndexPackageVersion)
298+
--add-source $(SourceIndexPackageSource)
299+
--tool-path $(Build.SourcesDirectory)/.tools;
300+
$(Build.SourcesDirectory)/.tools/BinLogToSln
301+
-i $(Build.SourcesDirectory)/artifacts/log/Release/Build.binlog
302+
-r $(Build.SourcesDirectory)
303+
-n $(Build.Repository.Name)
304+
-o $(Build.ArtifactStagingDirectory)/sourceIndex/
305+
displayName: Process binary log into indexable sln
280306
installNodeJs: false
281307
installJdk: false
282308
artifacts:
@@ -286,6 +312,9 @@ stages:
286312
includeForks: true
287313
- name: Windows_arm_Packages
288314
path: artifacts/packages/
315+
- ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
316+
- name: Windows_arm_SourceIndex
317+
path: $(Build.ArtifactStagingDirectory)/sourceIndex/
289318

290319
# Build Windows ARM64
291320
- template: jobs/default-build.yml
@@ -337,7 +366,7 @@ stages:
337366
displayName: Build Arm64 Installers
338367

339368
# A few files must also go to the VS package feed.
340-
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables['PostBuildSign'], 'true')) }}:
369+
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.PostBuildSign, 'true')) }}:
341370
- task: NuGetCommand@2
342371
displayName: Push Visual Studio packages
343372
inputs:
@@ -373,7 +402,7 @@ stages:
373402
- name: MacOS_arm64_Packages
374403
path: artifacts/packages/
375404

376-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
405+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
377406
- template: jobs/codesign-xplat.yml
378407
parameters:
379408
inputName: MacOS_arm64
@@ -403,7 +432,7 @@ stages:
403432
- name: MacOS_x64_Packages
404433
path: artifacts/packages/
405434

406-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
435+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
407436
- template: jobs/codesign-xplat.yml
408437
parameters:
409438
inputName: MacOS_x64
@@ -428,37 +457,19 @@ stages:
428457
$(_BuildArgs)
429458
$(_InternalRuntimeDownloadArgs)
430459
displayName: Run build.sh
431-
- script: |
432-
git clean -xfd src/**/obj/
433-
./dockerbuild.sh bionic \
434-
--ci \
435-
--nobl \
436-
--arch x64 \
437-
--build-installers \
438-
--no-build-deps \
439-
--no-build-nodejs \
440-
-p:OnlyPackPlatformSpecificPackages=true \
441-
-p:BuildRuntimeArchive=false \
442-
-p:LinuxInstallerType=deb \
443-
$(_BuildArgs) \
444-
$(_InternalRuntimeDownloadArgs)
460+
- script: git clean -xfd src/**/obj/;
461+
./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs
462+
-p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb
463+
$(_BuildArgs)
464+
$(_InternalRuntimeDownloadArgs)
445465
displayName: Build Debian installers
446-
- script: |
447-
git clean -xfd src/**/obj/
448-
./dockerbuild.sh rhel \
449-
--ci \
450-
--nobl \
451-
--arch x64 \
452-
--build-installers \
453-
--no-build-deps \
454-
--no-build-nodejs \
455-
-p:OnlyPackPlatformSpecificPackages=true \
456-
-p:BuildRuntimeArchive=false \
457-
-p:LinuxInstallerType=rpm \
458-
-p:AssetManifestFileName=aspnetcore-Linux_x64.xml \
459-
$(_BuildArgs) \
460-
$(_PublishArgs) \
461-
$(_InternalRuntimeDownloadArgs)
466+
- script: git clean -xfd src/**/obj/;
467+
./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs
468+
-p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm
469+
-p:AssetManifestFileName=aspnetcore-Linux_x64.xml
470+
$(_BuildArgs)
471+
$(_PublishArgs)
472+
$(_InternalRuntimeDownloadArgs)
462473
displayName: Build RPM installers
463474
installNodeJs: false
464475
artifacts:
@@ -469,7 +480,7 @@ stages:
469480
- name: Linux_x64_Packages
470481
path: artifacts/packages/
471482

472-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
483+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
473484
- template: jobs/codesign-xplat.yml
474485
parameters:
475486
inputName: Linux_x64
@@ -500,7 +511,7 @@ stages:
500511
- name: Linux_arm_Packages
501512
path: artifacts/packages/
502513

503-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
514+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
504515
- template: jobs/codesign-xplat.yml
505516
parameters:
506517
inputName: Linux_arm
@@ -531,7 +542,7 @@ stages:
531542
- name: Linux_arm64_Packages
532543
path: artifacts/packages/
533544

534-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
545+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
535546
- template: jobs/codesign-xplat.yml
536547
parameters:
537548
inputName: Linux_arm64
@@ -565,7 +576,7 @@ stages:
565576
- name: Linux_musl_x64_Packages
566577
path: artifacts/packages/
567578

568-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
579+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
569580
- template: jobs/codesign-xplat.yml
570581
parameters:
571582
inputName: Linux_musl_x64
@@ -599,7 +610,7 @@ stages:
599610
- name: Linux_musl_arm_Packages
600611
path: artifacts/packages/
601612

602-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
613+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
603614
- template: jobs/codesign-xplat.yml
604615
parameters:
605616
inputName: Linux_musl_arm
@@ -633,12 +644,12 @@ stages:
633644
- name: Linux_musl_arm64_Packages
634645
path: artifacts/packages/
635646

636-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
647+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
637648
- template: jobs/codesign-xplat.yml
638649
parameters:
639650
inputName: Linux_musl_arm64
640651

641-
- ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), in(variables['Build.Reason'], 'Manual'))) }}:
652+
- ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual'))) }}:
642653
# Test jobs
643654
- template: jobs/default-build.yml
644655
parameters:
@@ -747,15 +758,15 @@ stages:
747758
buildScript: './eng/build.sh $(_PublishArgs) --no-build-repo-tasks'
748759
skipPublishValidation: true
749760

750-
# Publish to the BAR
761+
# Publish to the BAR and perform source indexing. Wait until everything else is done.
751762
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
752763
- template: /eng/common/templates/job/publish-build-assets.yml
753764
parameters:
754765
dependsOn:
755766
- Windows_build
756767
- Windows_arm_build
757768
- Windows_arm64_build
758-
- ${{ if ne(variables['PostBuildSign'], 'true') }}:
769+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
759770
- CodeSign_Xplat_MacOS_arm64
760771
- CodeSign_Xplat_MacOS_x64
761772
- CodeSign_Xplat_Linux_x64
@@ -764,7 +775,7 @@ stages:
764775
- CodeSign_Xplat_Linux_musl_x64
765776
- CodeSign_Xplat_Linux_musl_arm
766777
- CodeSign_Xplat_Linux_musl_arm64
767-
- ${{ if eq(variables['PostBuildSign'], 'true') }}:
778+
- ${{ if eq(variables.PostBuildSign, 'true') }}:
768779
- MacOs_arm64_build
769780
- MacOs_x64_build
770781
- Linux_x64_build
@@ -773,12 +784,81 @@ stages:
773784
- Linux_musl_x64_build
774785
- Linux_musl_arm_build
775786
- Linux_musl_arm64_build
776-
# In addition to the dependencies above, ensure the build was successful overall.
787+
# In addition to the dependencies above that provide assets, ensure the build was successful overall.
788+
- ${{ if in(variables['Build.Reason'], 'Manual') }}:
789+
- Code_check
790+
- ${{ if ne(parameters.skipTests, 'true') }}:
791+
- Windows_Test
792+
- MacOS_Test
793+
- Linux_Test
794+
- Helix_x64
777795
- Source_Build_Managed
778796
pool:
779797
vmImage: vs2017-win2016
780798
publishUsingPipelines: ${{ variables._PublishUsingPipelines }}
781799
enablePublishBuildArtifacts: true # publish artifacts/log files
800+
- ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
801+
- job: SourceIndexUpload
802+
displayName: Upload indexable solution
803+
dependsOn:
804+
- Windows_arm_build
805+
# In addition to the dependency above that provides assets, ensure the build was successful overall.
806+
- Windows_build
807+
- Windows_arm64_build
808+
- ${{ if ne(variables.PostBuildSign, 'true') }}:
809+
- CodeSign_Xplat_MacOS_arm64
810+
- CodeSign_Xplat_MacOS_x64
811+
- CodeSign_Xplat_Linux_x64
812+
- CodeSign_Xplat_Linux_arm
813+
- CodeSign_Xplat_Linux_arm64
814+
- CodeSign_Xplat_Linux_musl_x64
815+
- CodeSign_Xplat_Linux_musl_arm
816+
- CodeSign_Xplat_Linux_musl_arm64
817+
- ${{ if eq(variables.PostBuildSign, 'true') }}:
818+
- MacOs_arm64_build
819+
- MacOs_x64_build
820+
- Linux_x64_build
821+
- Linux_arm_build
822+
- Linux_arm64_build
823+
- Linux_musl_x64_build
824+
- Linux_musl_arm_build
825+
- Linux_musl_arm64_build
826+
- ${{ if in(variables['Build.Reason'], 'Manual') }}:
827+
- Code_check
828+
- ${{ if ne(parameters.skipTests, 'true') }}:
829+
- Windows_Test
830+
- MacOS_Test
831+
- Linux_Test
832+
- Helix_x64
833+
- Source_Build_Managed
834+
pool:
835+
name: NetCore1ESPool-Internal
836+
# Visual Studio Enterprise - no BuildTools agents exist internally and job must run on Windows
837+
demands: ImageOverride -equals Build.Server.Amd64.VS2019
838+
steps:
839+
- task: DownloadBuildArtifacts@0
840+
inputs:
841+
artifactName: Windows_arm_SourceIndex
842+
cleanDestinationFolder: true
843+
# Ignore repository's global.json and any existing .NET SDK.
844+
- task: UseDotNet@2
845+
displayName: Use .NET Core sdk 3.1
846+
inputs:
847+
packageType: sdk
848+
version: 3.1.x
849+
installationPath: $(Agent.TempDirectory)/.dotnet
850+
workingDirectory: $(Agent.TempDirectory)
851+
- script: $(Agent.TempDirectory)/.dotnet/dotnet tool install UploadIndexStage1
852+
--version $(SourceIndexPackageVersion)
853+
--add-source $(SourceIndexPackageSource)
854+
--tool-path $(Agent.TempDirectory)/.tools &&
855+
$(Agent.TempDirectory)/.tools/UploadIndexStage1
856+
-i $(System.ArtifactsDirectory)/Windows_arm_SourceIndex/
857+
-n $(Build.Repository.Name)
858+
displayName: Upload indexable solution
859+
workingDirectory: $(Agent.TempDirectory)
860+
env:
861+
BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
782862

783863
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
784864
- template: /eng/common/templates/post-build/post-build.yml

‎.azure/pipelines/jobs/default-build.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,11 @@ jobs:
277277
condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true')))
278278
continueOnError: true
279279
inputs:
280-
pathtoPublish: $(Build.SourcesDirectory)/${{ artifact.path }}
280+
# Assume runtime variable values are absolute paths already.
281+
${{ if startsWith(artifact.path, '$(') }}:
282+
pathToPublish: ${{ artifact.path }}
283+
${{ if not(startsWith(artifact.path, '$(')) }}:
284+
pathToPublish: $(Build.SourcesDirectory)/${{ artifact.path }}
281285
${{ if eq(artifact.name, '') }}:
282286
artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration)
283287
${{ if ne(artifact.name, '') }}:

0 commit comments

Comments
 (0)