CI: Speed up CircleCI by folding build workflow downstream (#4426)

algorand · Aug 19, 2022 · feab018 · feab018
1 parent 0b9580b
commit feab018
Showing 1 changed file with 36 additions and 93 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -64,8 +64,8 @@ workflows:
   version: 2
   "circleci_build_and_test":
     jobs:
-      - build:
-          name: << matrix.platform >>_build
+      - test:
+          name: << matrix.platform >>_test
           matrix: &matrix-default
             parameters:
               platform: ["amd64", "arm64"]
@@ -75,8 +75,8 @@ workflows:
                 - /rel\/.*/
                 - << pipeline.parameters.valid_nightly_branch >>
 
-      - build_nightly:
-          name: << matrix.platform >>_build_nightly
+      - test_nightly:
+          name: << matrix.platform >>_test_nightly
           matrix: &matrix-nightly
             parameters:
               platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"]
@@ -87,64 +87,49 @@ workflows:
                 - << pipeline.parameters.valid_nightly_branch >>
           context: slack-secrets
 
-      - test:
-          name: << matrix.platform >>_test
-          matrix:
-            <<: *matrix-default
-          requires:
-            - << matrix.platform >>_build
-
-      - test_nightly:
-          name: << matrix.platform >>_test_nightly
-          matrix:
-            <<: *matrix-nightly
-          requires:
-            - << matrix.platform >>_build_nightly
-          context: slack-secrets
-
       - integration:
           name: << matrix.platform >>_integration
           matrix:
             <<: *matrix-default
-          requires:
-            - << matrix.platform >>_build
+          filters:
+            <<: *filters-default
 
       - integration_nightly:
           name: << matrix.platform >>_integration_nightly
           matrix:
             <<: *matrix-nightly
-          requires:
-            - << matrix.platform >>_build_nightly
+          filters:
+            <<: *filters-nightly
           context: slack-secrets
 
       - e2e_expect:
           name: << matrix.platform >>_e2e_expect
           matrix:
             <<: *matrix-default
-          requires:
-            - << matrix.platform >>_build
+          filters:
+            <<: *filters-default
 
       - e2e_expect_nightly:
           name: << matrix.platform >>_e2e_expect_nightly
           matrix:
             <<: *matrix-nightly
-          requires:
-            - << matrix.platform >>_build_nightly
+          filters:
+            <<: *filters-nightly
           context: slack-secrets
 
       - e2e_subs:
           name: << matrix.platform >>_e2e_subs
           matrix:
             <<: *matrix-default
-          requires:
-            - << matrix.platform >>_build
+          filters:
+            <<: *filters-default
 
       - e2e_subs_nightly:
           name: << matrix.platform >>_e2e_subs_nightly
           matrix:
             <<: *matrix-nightly
-          requires:
-            - << matrix.platform >>_build_nightly
+          filters:
+            <<: *filters-nightly
           context:
             - slack-secrets
             - aws-secrets
@@ -226,12 +211,20 @@ commands:
             $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool unzip autoconf automake
 
   generic_build:
-    description: Run basic build and store in workspace for re-use by different architectures
+    description: >
+      Run basic build.
+
+      If command execution time increases _appreciably_, revisit CI topology:
+      * Historically, the command executes _quickly_ (< 3m with resource class >= medium).
+      * Consequently, it's faster to embed the command in a combined build + test workflow rather than independent build and test workflows.
     parameters:
       build_dir:
         type: string
         default: << pipeline.parameters.build_dir >>
     steps:
+      - prepare_build_dir
+      - checkout
+      - prepare_go
       - restore_libsodium
       - restore_cache:
           keys:
@@ -260,13 +253,6 @@ commands:
           key: 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}'
           paths:
             - tmp/go-cache
-      - persist_to_workspace:
-          root: << parameters.build_dir >>
-          paths:
-            - project
-            - go
-            - gimme
-            - .gimme
 
   cache_libsodium:
     description: Cache libsodium for build
@@ -313,17 +299,10 @@ commands:
         type: string
         default: << pipeline.parameters.result_path >>
     steps:
-      - attach_workspace:
-          at: << parameters.build_dir >>
       - run: |
           mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}
           touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml
           touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json
-      - restore_cache:
-          keys:
-            - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}'
-            - 'go-cache-v3-{{ arch }}-{{ .Branch }}-'
-            - 'go-cache-v3-{{ arch }}-'
       - run:
           name: Run build tests
           no_output_timeout: << parameters.no_output_timeout >>
@@ -385,8 +364,6 @@ commands:
         type: string
         default: << pipeline.parameters.result_path >>
     steps:
-      - attach_workspace:
-          at: << parameters.build_dir >>
       - run: |
           mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}
           touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml
@@ -492,33 +469,6 @@ commands:
                     scripts/travis/test_release.sh
 
 jobs:
-  build:
-    parameters:
-      platform:
-        type: string
-    executor: << parameters.platform >>_medium
-    working_directory: << pipeline.parameters.build_dir >>/project
-    steps:
-      - prepare_build_dir
-      - checkout
-      - prepare_go
-      - generic_build
-
-  build_nightly:
-    parameters:
-      platform:
-        type: string
-    executor: << parameters.platform >>_medium
-    working_directory: << pipeline.parameters.build_dir >>/project
-    steps:
-      - prepare_build_dir
-      - checkout
-      - prepare_go
-      - generic_build
-      - slack/notify: &slack-fail-event
-          event: fail
-          template: basic_fail_1
-
   test:
     parameters:
       platform:
@@ -527,8 +477,7 @@ jobs:
     working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     steps:
-      - prepare_build_dir
-      - prepare_go
+      - generic_build
       - generic_test:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_test
@@ -543,15 +492,15 @@ jobs:
     working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     steps:
-      - prepare_build_dir
-      - prepare_go
+      - generic_build
       - generic_test:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_test_nightly
           no_output_timeout: 45m
       - upload_coverage
-      - slack/notify:
-          <<: *slack-fail-event
+      - slack/notify: &slack-fail-event
+          event: fail
+          template: basic_fail_1
 
   integration:
     parameters:
@@ -563,8 +512,7 @@ jobs:
     environment:
       E2E_TEST_FILTER: "GO"
     steps:
-      - prepare_build_dir
-      - prepare_go
+      - generic_build
       - generic_integration:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_integration
@@ -580,8 +528,7 @@ jobs:
     environment:
       E2E_TEST_FILTER: "GO"
     steps:
-      - prepare_build_dir
-      - prepare_go
+      - generic_build
       - generic_integration:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_integration_nightly
@@ -599,8 +546,7 @@ jobs:
     environment:
       E2E_TEST_FILTER: "EXPECT"
     steps:
-      - prepare_build_dir
-      - prepare_go
+      - generic_build
       - generic_integration:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_e2e_expect
@@ -616,8 +562,7 @@ jobs:
     environment:
       E2E_TEST_FILTER: "EXPECT"
     steps:
-      - prepare_build_dir
-      - prepare_go
+      - generic_build
       - generic_integration:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform>>_e2e_expect_nightly
@@ -634,8 +579,7 @@ jobs:
     environment:
       E2E_TEST_FILTER: "SCRIPTS"
     steps:
-      - prepare_build_dir
-      - prepare_go
+      - generic_build
       - generic_integration:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_e2e_subs
@@ -654,8 +598,7 @@ jobs:
       # one of the platforms in the matrix.
       CI_KEEP_TEMP_PLATFORM: "amd64"
     steps:
-      - prepare_build_dir
-      - prepare_go
+      - generic_build
       - generic_integration:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_e2e_subs_nightly