Remove checkout bundle and job to improve reliability (#46107)

* Remove checkout bundle and job to improve reliability * comment * Fix build error * Rename evaluate paths job to a valid name * Fix references to evaluate path variables * Condition evaluate paths dependency on common parameter * Bring back fetch depth to 20 * PR Feedback
dotnet · Dec 17, 2020 · f8a83c8 · f8a83c8
1 parent d3a04b2
commit f8a83c8
Show file tree

Hide file tree

Showing 67 changed files with 218 additions and 313 deletions.
diff --git a/eng/pipelines/common/checkout-job.yml b/eng/pipelines/common/checkout-job.yml
diff --git a/eng/pipelines/common/clone-checkout-bundle-step.yml b/eng/pipelines/common/clone-checkout-bundle-step.yml
diff --git a/...es/common/checkout-and-evaluate-paths.yml → ...pelines/common/evaluate-default-paths.yml b/...es/common/checkout-and-evaluate-paths.yml → ...pelines/common/evaluate-default-paths.yml
@@ -1,8 +1,9 @@
+# Template to evaluate common paths in different pipelines.
 parameters:
   extraSubsets: ''
 
 jobs:
-- template: /eng/pipelines/common/checkout-job.yml
+- template: /eng/pipelines/common/evaluate-paths-job.yml
   parameters:
     paths:
     - subset: coreclr

diff --git a/eng/pipelines/common/evaluate-paths-job.yml b/eng/pipelines/common/evaluate-paths-job.yml
@@ -0,0 +1,52 @@
+### Job used to evaluate changed paths on a pull request to emit variables to condition jobs based on paths.
+
+parameters:
+  # Object containing subset include and exclude paths in an array form.
+  # Scenarios:
+  #  1. exclude paths are specified
+  #     Will include all paths except the ones in the exclude list.
+  #  2. include paths are specified
+  #     Will only include paths specified in the list.
+  #  3. exclude + include:
+  #     1st we evaluate changes for all paths except ones in excluded list. If we can't find
+  #     any applicable changes like that, then we evaluate changes for incldued paths
+  #     if any of these two finds changes, then a variable will be set to true.
+  #  In order to consume this variable you need to reference it via: $[ dependencies.evaluate_paths.outputs['SetPathVars_<subset>.containschange'] ]
+  #
+  #  Array form example
+  #  paths:
+  #  - subset: coreclr
+  #    include:
+  #    - src/libraries/System.Private.CoreLib/*
+  #    exclude:
+  #    - src/libraries/*
+  #
+  #  This example will include ALL path changes under src/libraries/ except the ones under src/libraries/*!System.Private.CoreLib/*
+  paths: []
+
+
+jobs:
+  - job: evaluate_paths
+    displayName: Evaluate Paths
+    pool:
+      vmImage: 'macOS-10.14'
+
+    steps:
+    - checkout: self
+      clean: true
+      fetchDepth: $(checkoutFetchDepth)
+
+    - ${{ if ne(parameters.paths[0], '') }}:
+      - ${{ each path in parameters.paths }}:
+        - template: evaluate-changed-paths.yml
+          parameters:
+            subsetName: ${{ path.subset }} 
+            arguments:
+            # The commit that we're building is always a merge commit that is merging into the target branch.
+            # So the first parent of the commit is on the target branch and the second parent is on the source branch.
+            - --difftarget HEAD^1
+            - --subset ${{ path.subset }}
+            - ${{ if ne(path.include[0], '') }}:
+              - --includepaths '${{ join('+', path.include) }}'
+            - ${{ if ne(path.exclude[0], '') }}:
+              - --excludepaths '${{ join('+', path.exclude) }}'
diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
@@ -15,6 +15,7 @@ parameters:
   platform: ''
   condition: true
   shouldContinueOnError: false
+  dependOnEvaluatePaths: false
   isOfficialBuild: false
   runtimeFlavor: 'coreclr'
   helixQueues: ''
@@ -26,14 +27,16 @@ jobs:
   parameters:
     name: ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }}
     displayName: ${{ format('Build {0}{1} {2} {3} {4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }}
-    dependsOn: checkout
     pool: ${{ parameters.pool }}
     container: ${{ parameters.container }}
     condition: and(succeeded(), ${{ parameters.condition }})
     timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
     workspace:
       clean: all
 
+    ${{ if eq(parameters.dependOnEvaluatePaths, true) }}:
+      dependsOn: evaluate_paths
+
     variables:
       - name: _osParameter
         value: -os ${{ parameters.osGroup }}
@@ -66,7 +69,9 @@ jobs:
         - ${{ variable }}
 
     steps:
-    - template: /eng/pipelines/common/clone-checkout-bundle-step.yml
+    - checkout: self
+      clean: true
+      fetchDepth: $(checkoutFetchDepth)
 
     - ${{ if eq(parameters.isOfficialBuild, true) }}:
       - template: /eng/pipelines/common/restore-internal-tools.yml

diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml
@@ -23,6 +23,7 @@ parameters:
   runtimeFlavorDisplayName: 'CoreCLR'
   runtimeVariant: ''
   dependsOn: []
+  dependOnEvaluatePaths: false
 
 ### Build managed test components (native components are getting built as part
 ### of the the product build job).
@@ -47,6 +48,7 @@ jobs:
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     variables: ${{ parameters.variables }}
     pool: ${{ parameters.pool }}
+    dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
 
     # Test jobs should continue on error for internal builds
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:

diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml
@@ -23,6 +23,7 @@ parameters:
   runtimeFlavorDisplayName: 'CoreCLR'
   shouldContinueOnError: false
   dependsOn: []
+  dependOnEvaluatePaths: false
 
 ### Test run job
 
@@ -46,6 +47,7 @@ jobs:
     runtimeVariant: ${{ parameters.runtimeVariant }}
     pool: ${{ parameters.pool }}
     condition: ${{ parameters.condition }}
+    dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
 
     # Test jobs should continue on error for internal builds
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:

diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml
@@ -3,6 +3,10 @@ variables:
   value: ${{ and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}
 - name: isFullMatrix
   value: ${{ and(eq(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}
+
+# We only run evaluate paths on runtime and runtime-staging pipelines on PRs
+- name: dependOnEvaluatePaths
+  value: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging')) }}
 - name: debugOnPrReleaseOnRolling
   ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     value: Release

diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml
@@ -20,6 +20,9 @@ jobs:
 
     shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }}
 
+    # keep in sync with /eng/pipelines/common/variables.yml
+    dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging')) }}
+
     variables:
       # Disable component governance in our CI builds. These builds are not shipping nor
       # are they a service. Also the component governance jobs issue lots of inconsequential
@@ -30,6 +33,9 @@ jobs:
       - name: runCodesignValidationInjection
         value: false
 
+      - name: checkoutFetchDepth
+        value: 20
+
       - name: buildConfigUpper
         ${{ if eq(parameters.jobParameters.buildConfig, 'debug') }}:
           value: 'Debug'

diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml
@@ -26,11 +26,6 @@ trigger:
 
 jobs:
 
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
-
 #
 # Debug builds
 #

diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/crossgen2-composite.yml b/eng/pipelines/coreclr/crossgen2-composite.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/crossgen2-gcstress.yml b/eng/pipelines/coreclr/crossgen2-gcstress.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/crossgen2-outerloop.yml b/eng/pipelines/coreclr/crossgen2-outerloop.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/crossgen2.yml b/eng/pipelines/coreclr/crossgen2.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/gc-longrunning.yml b/eng/pipelines/coreclr/gc-longrunning.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/gc-simulator.yml b/eng/pipelines/coreclr/gc-simulator.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/gcstress-extra.yml b/eng/pipelines/coreclr/gcstress-extra.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/ilasm.yml b/eng/pipelines/coreclr/ilasm.yml
@@ -18,10 +18,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/jit-experimental.yml b/eng/pipelines/coreclr/jit-experimental.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/jitrollingbuild.yml b/eng/pipelines/coreclr/jitrollingbuild.yml
@@ -12,10 +12,6 @@ trigger:
 pr: none
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/jitstress-isas-arm.yml b/eng/pipelines/coreclr/jitstress-isas-arm.yml
@@ -9,10 +9,6 @@ schedules:
   always: true
 
 jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters: