Bring back checkout bundle for PRs until AzDo fixes PR job checkouts (#…

…46649)
dotnet · Jan 6, 2021 · 604660b · 604660b
1 parent 6ba4088
commit 604660b
Show file tree

Hide file tree

Showing 65 changed files with 502 additions and 208 deletions.
diff --git a/...pelines/common/evaluate-default-paths.yml → ...es/common/checkout-and-evaluate-paths.yml b/...pelines/common/evaluate-default-paths.yml → ...es/common/checkout-and-evaluate-paths.yml
@@ -1,9 +1,8 @@
-# Template to evaluate common paths in different pipelines.
 parameters:
   extraSubsets: ''
 
 jobs:
-- template: /eng/pipelines/common/evaluate-paths-job.yml
+- template: /eng/pipelines/common/checkout-job.yml
   parameters:
     paths:
     - subset: coreclr

diff --git a/eng/pipelines/common/checkout-job.yml b/eng/pipelines/common/checkout-job.yml
@@ -0,0 +1,62 @@
+### Check out job creating a git bundle and publishing it
+### into an Azure artifact for reuse by the subsequent build and test execution phases.
+### If paths is specified, we will create a job using evaluate-changed-paths.yml template
+### for each path specified.
+
+parameters:
+  # Object containing subset include and exclude paths in an array form.
+  # Scenarios:
+  #  1. exclude paths are specified
+  #     Will include all paths except the ones in the exclude list.
+  #  2. include paths are specified
+  #     Will only include paths specified in the list.
+  #  3. exclude + include:
+  #     1st we evaluate changes for all paths except ones in excluded list. If we can't find
+  #     any applicable changes like that, then we evaluate changes for incldued paths
+  #     if any of these two finds changes, then a variable will be set to true.
+  #  In order to consume this variable you need to reference it via: $[ dependencies.checkout.outputs['SetPathVars_<subset>.containschange'] ]
+  #
+  #  Array form example
+  #  paths:
+  #  - subset: coreclr
+  #    include:
+  #    - src/libraries/System.Private.CoreLib/*
+  #    exclude:
+  #    - src/libraries/*
+  #
+  #  This example will include ALL path changes except the ones under src/libraries/*!System.Private.CoreLib/*
+  paths: []
+
+jobs:
+- job: checkout
+  displayName: Checkout
+
+  pool:
+    vmImage: 'macOS-10.14'
+
+  steps:
+  - checkout: self
+    clean: true
+    fetchDepth: 20
+
+  - script: git bundle create $(Build.StagingDirectory)/Checkout.bundle HEAD
+    displayName: Create Checkout.bundle
+
+  - publish: $(Build.StagingDirectory)/Checkout.bundle
+    artifact: Checkout_bundle
+    displayName: Upload Checkout.bundle
+
+  - ${{ if and(ne(parameters.paths[0], ''), eq(variables['Build.Reason'], 'PullRequest')) }}:
+    - ${{ each path in parameters.paths }}:
+      - template: evaluate-changed-paths.yml
+        parameters:
+          subsetName: ${{ path.subset }} 
+          arguments:
+          # The commit that we're building is always a merge commit that is merging into the target branch.
+          # So the first parent of the commit is on the target branch and the second parent is on the source branch.
+          - --difftarget HEAD^1
+          - --subset ${{ path.subset }}
+          - ${{ if ne(path.include[0], '') }}:
+            - --includepaths '${{ join('+', path.include) }}'
+          - ${{ if ne(path.exclude[0], '') }}:
+            - --excludepaths '${{ join('+', path.exclude) }}'
diff --git a/eng/pipelines/common/checkout-step.yml b/eng/pipelines/common/checkout-step.yml
@@ -0,0 +1,23 @@
+parameters:
+  useBundle: true
+  fetchDepth: 20
+
+steps:
+# Checkout from bundle
+- ${{ if eq(parameters.useBundle, true) }}:
+  - checkout: none
+
+  - download: current
+    artifact: Checkout_bundle
+    displayName: Download Checkout.bundle
+
+  - script: |
+      $(setScriptToEchoAndFailOnNonZero)
+      git clone $(Pipeline.Workspace)/Checkout_bundle/Checkout.bundle .
+      git remote set-url origin $(Build.Repository.Uri)
+    displayName: Clone the repository from Checkout.bundle
+
+- ${{ if eq(parameters.useBundle, false) }}:
+    - checkout: self
+      clean: true
+      fetchDepth: ${{ parameters.fetchDepth }}
diff --git a/eng/pipelines/common/evaluate-paths-job.yml b/eng/pipelines/common/evaluate-paths-job.yml
diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
@@ -16,7 +16,7 @@ parameters:
   condition: true
   useContinueOnErrorDuringBuild: false
   shouldContinueOnError: false
-  dependOnEvaluatePaths: false
+  useCheckoutBundle: false
   isOfficialBuild: false
   runtimeFlavor: 'coreclr'
   helixQueues: ''
@@ -40,8 +40,8 @@ jobs:
     workspace:
       clean: all
 
-    ${{ if eq(parameters.dependOnEvaluatePaths, true) }}:
-      dependsOn: evaluate_paths
+    ${{ if eq(parameters.useCheckoutBundle, true) }}:
+      dependsOn: checkout
 
     variables:
       - name: _osParameter
@@ -75,9 +75,9 @@ jobs:
         - ${{ variable }}
 
     steps:
-    - checkout: self
-      clean: true
-      fetchDepth: $(checkoutFetchDepth)
+    - template: /eng/pipelines/common/checkout-step.yml
+      parameters:
+        useBundle: ${{ parameters.useCheckoutBundle }}
 
     - ${{ if eq(parameters.isOfficialBuild, true) }}:
       - template: /eng/pipelines/common/restore-internal-tools.yml

diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml
@@ -23,7 +23,7 @@ parameters:
   runtimeFlavorDisplayName: 'CoreCLR'
   runtimeVariant: ''
   dependsOn: []
-  dependOnEvaluatePaths: false
+  useCheckoutBundle: false
 
 ### Build managed test components (native components are getting built as part
 ### of the the product build job).
@@ -48,7 +48,8 @@ jobs:
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     variables: ${{ parameters.variables }}
     pool: ${{ parameters.pool }}
-    dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
+    useCheckoutBundle: ${{ parameters.useCheckoutBundle }}
+
 
     # Test jobs should continue on error for internal builds
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:

diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml
@@ -23,7 +23,7 @@ parameters:
   runtimeFlavorDisplayName: 'CoreCLR'
   shouldContinueOnError: false
   dependsOn: []
-  dependOnEvaluatePaths: false
+  useCheckoutBundle: false
 
 ### Test run job
 
@@ -47,7 +47,7 @@ jobs:
     runtimeVariant: ${{ parameters.runtimeVariant }}
     pool: ${{ parameters.pool }}
     condition: ${{ parameters.condition }}
-    dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
+    useCheckoutBundle: ${{ parameters.useCheckoutBundle }}
 
     # Test jobs should continue on error for internal builds
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:

diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml
@@ -4,9 +4,9 @@ variables:
 - name: isFullMatrix
   value: ${{ and(eq(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}
 
-# We only run evaluate paths on runtime and runtime-staging pipelines on PRs
-- name: dependOnEvaluatePaths
-  value: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging')) }}
+# We only use checkout bundle on PRs
+- name: useCheckoutBundle
+  value: ${{ eq(variables['Build.Reason'], 'PullRequest') }}
 - name: debugOnPrReleaseOnRolling
   ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     value: Release

diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml
@@ -21,7 +21,7 @@ jobs:
     shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }}
 
     # keep in sync with /eng/pipelines/common/variables.yml
-    dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging')) }}
+    useCheckoutBundle: ${{ eq(variables['Build.Reason'], 'PullRequest') }}
 
     variables:
       # Disable component governance in our CI builds. These builds are not shipping nor

diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml
@@ -24,8 +24,17 @@ trigger:
     - eng/pipelines/libraries/*
     - eng/pipelines/runtime.yml
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
 
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
+
 #
 # Debug builds
 #

diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml
@@ -8,7 +8,15 @@ schedules:
     - master
   always: true
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/crossgen2-composite.yml b/eng/pipelines/coreclr/crossgen2-composite.yml
@@ -8,7 +8,15 @@ schedules:
     - master
   always: true
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/crossgen2-gcstress.yml b/eng/pipelines/coreclr/crossgen2-gcstress.yml
@@ -8,7 +8,15 @@ schedules:
     - master
   always: true
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/crossgen2-outerloop.yml b/eng/pipelines/coreclr/crossgen2-outerloop.yml
@@ -8,7 +8,15 @@ schedules:
     - master
   always: true
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/crossgen2.yml b/eng/pipelines/coreclr/crossgen2.yml
@@ -8,7 +8,15 @@ schedules:
     - master
   always: true
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/gc-longrunning.yml b/eng/pipelines/coreclr/gc-longrunning.yml
@@ -8,7 +8,15 @@ schedules:
     - master
   always: true
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/gc-simulator.yml b/eng/pipelines/coreclr/gc-simulator.yml
@@ -8,7 +8,15 @@ schedules:
     - master
   always: true
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/gcstress-extra.yml b/eng/pipelines/coreclr/gcstress-extra.yml
@@ -8,7 +8,15 @@ schedules:
     - master
   always: true
 
+variables:
+  - template: /eng/pipelines/common/variables.yml
+
 jobs:
+#
+# Checkout repository
+#
+- ${{ if eq(variables.useCheckoutBundle, true) }}:
+  - template: /eng/pipelines/common/checkout-job.yml
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters: