From 58b2da3b26879c81463610b7d9ecb03ce109e13a Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Wed, 17 Aug 2022 22:46:53 -0400 Subject: [PATCH 01/13] Attempt to consolidate build into test workflows --- .circleci/config.yml | 118 +++++++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 39 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e04193f4a0..71a3550ffe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,16 +66,16 @@ workflows: jobs: - codegen_verification - - build: - name: << matrix.platform >>_build - matrix: &matrix-default - parameters: - platform: ["amd64", "arm64"] - filters: &filters-default - branches: - ignore: - - /rel\/.*/ - - << pipeline.parameters.valid_nightly_branch >> +# - build: +# name: << matrix.platform >>_build +# matrix: &matrix-default +# parameters: +# platform: ["amd64", "arm64"] +# filters: &filters-default +# branches: +# ignore: +# - /rel\/.*/ +# - << pipeline.parameters.valid_nightly_branch >> - build_nightly: name: << matrix.platform >>_build_nightly @@ -91,10 +91,9 @@ workflows: - test: name: << matrix.platform >>_test - matrix: - <<: *matrix-default - requires: - - << matrix.platform >>_build + matrix: &matrix-default + parameters: + platform: ["amd64", "arm64"] - test_nightly: name: << matrix.platform >>_test_nightly @@ -106,10 +105,11 @@ workflows: - integration: name: << matrix.platform >>_integration - matrix: - <<: *matrix-default - requires: - - << matrix.platform >>_build + matrix: &matrix-default + parameters: + platform: ["amd64", "arm64"] +# requires: +# - << matrix.platform >>_build - integration_nightly: name: << matrix.platform >>_integration_nightly @@ -121,10 +121,11 @@ workflows: - e2e_expect: name: << matrix.platform >>_e2e_expect - matrix: - <<: *matrix-default - requires: - - << matrix.platform >>_build + matrix: &matrix-default + parameters: + platform: ["amd64", "arm64"] +# requires: +# - << matrix.platform >>_build - e2e_expect_nightly: name: << matrix.platform >>_e2e_expect_nightly @@ -136,10 +137,11 @@ workflows: - e2e_subs: name: << matrix.platform >>_e2e_subs - matrix: - <<: *matrix-default - requires: - - << matrix.platform >>_build + matrix: &matrix-default + parameters: + platform: ["amd64", "arm64"] +# requires: +# - << matrix.platform >>_build - e2e_subs_nightly: name: << matrix.platform >>_e2e_subs_nightly @@ -228,6 +230,42 @@ commands: export msys2+=" -msys2 -c "\"\$@"\" --" $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool unzip autoconf automake + generic_build_consolidated: + description: TBD + parameters: + build_dir: + type: string + default: << pipeline.parameters.build_dir >> + steps: + - restore_libsodium + - restore_cache: + keys: + - 'go-mod-1.17.9-v3-{{ arch }}-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}' + - restore_cache: + keys: + - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' + - 'go-cache-v3-{{ arch }}-{{ .Branch }}-' + - 'go-cache-v3-{{ arch }}-' + - run: + name: scripts/travis/build.sh --make_debug + command: | + export PATH=$(echo "$PATH" | sed -e "s|:${HOME}/\.go_workspace/bin||g" | sed -e 's|:/usr/local/go/bin||g') + export GOPATH="<< parameters.build_dir >>/go" + export ALGORAND_DEADLOCK=enable + export GIMME_INSTALL_DIR=<< parameters.build_dir >> + export GIMME_ENV_PREFIX=<< parameters.build_dir >>/.gimme/envs + export GIMME_VERSION_PREFIX=<< parameters.build_dir >>/.gimme/versions + scripts/travis/build.sh --make_debug + - cache_libsodium + - save_cache: + key: 'go-mod-1.17.9-v3-{{ arch }}-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}' + paths: + - << parameters.build_dir >>/go/pkg/mod + - save_cache: + key: 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' + paths: + - tmp/go-cache + generic_build: description: Run basic build and store in workspace for re-use by different architectures parameters: @@ -316,17 +354,17 @@ commands: type: string default: << pipeline.parameters.result_path >> steps: - - attach_workspace: - at: << parameters.build_dir >> - - run: | - mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX} - touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml - touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json - - restore_cache: - keys: - - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' - - 'go-cache-v3-{{ arch }}-{{ .Branch }}-' - - 'go-cache-v3-{{ arch }}-' +# - attach_workspace: +# at: << parameters.build_dir >> +# - run: | +# mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX} +# touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml +# touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json +# - restore_cache: +# keys: +# - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' +# - 'go-cache-v3-{{ arch }}-{{ .Branch }}-' +# - 'go-cache-v3-{{ arch }}-' - run: name: Run build tests no_output_timeout: << parameters.no_output_timeout >> @@ -388,8 +426,8 @@ commands: type: string default: << pipeline.parameters.result_path >> steps: - - attach_workspace: - at: << parameters.build_dir >> +# - attach_workspace: +# at: << parameters.build_dir >> - run: | mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX} touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml @@ -528,6 +566,7 @@ jobs: working_directory: << pipeline.parameters.build_dir >>/project parallelism: 4 steps: + - generic_build_consolidated - prepare_build_dir - prepare_go - generic_test: @@ -564,6 +603,7 @@ jobs: environment: E2E_TEST_FILTER: "GO" steps: + - generic_build_consolidated - prepare_build_dir - prepare_go - generic_integration: From ead4161dbc042cb3a4db319bf38c9a6fe7045373 Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Wed, 17 Aug 2022 22:51:48 -0400 Subject: [PATCH 02/13] Try to fix the build --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 71a3550ffe..b5dcd6af02 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,6 +94,7 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] + filters: &filters-default - test_nightly: name: << matrix.platform >>_test_nightly @@ -108,6 +109,7 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] + filters: &filters-default # requires: # - << matrix.platform >>_build @@ -124,6 +126,7 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] + filters: &filters-default # requires: # - << matrix.platform >>_build @@ -140,6 +143,7 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] + filters: &filters-default # requires: # - << matrix.platform >>_build From 118fdc257f41263ead65efdf65efb1f3ed55d875 Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Wed, 17 Aug 2022 22:54:02 -0400 Subject: [PATCH 03/13] Revert "Try to fix the build" This reverts commit ead4161dbc042cb3a4db319bf38c9a6fe7045373. --- .circleci/config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5dcd6af02..71a3550ffe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,7 +94,6 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] - filters: &filters-default - test_nightly: name: << matrix.platform >>_test_nightly @@ -109,7 +108,6 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] - filters: &filters-default # requires: # - << matrix.platform >>_build @@ -126,7 +124,6 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] - filters: &filters-default # requires: # - << matrix.platform >>_build @@ -143,7 +140,6 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] - filters: &filters-default # requires: # - << matrix.platform >>_build From 28e94d7bfd4bdff0c8b2d000ab1363ef15cbd349 Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Wed, 17 Aug 2022 22:55:59 -0400 Subject: [PATCH 04/13] Try to fix build steps --- .circleci/config.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 71a3550ffe..5d5deb33bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -566,9 +566,10 @@ jobs: working_directory: << pipeline.parameters.build_dir >>/project parallelism: 4 steps: - - generic_build_consolidated - prepare_build_dir + - checkout - prepare_go + - generic_build_consolidated - generic_test: platform: << parameters.platform >> result_subdir: << parameters.platform >>_test @@ -603,9 +604,10 @@ jobs: environment: E2E_TEST_FILTER: "GO" steps: - - generic_build_consolidated - prepare_build_dir + - checkout - prepare_go + - generic_build_consolidated - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform >>_integration @@ -641,7 +643,9 @@ jobs: E2E_TEST_FILTER: "EXPECT" steps: - prepare_build_dir + - checkout - prepare_go + - generic_build_consolidated - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform >>_e2e_expect @@ -676,7 +680,9 @@ jobs: E2E_TEST_FILTER: "SCRIPTS" steps: - prepare_build_dir + - checkout - prepare_go + - generic_build_consolidated - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform >>_e2e_subs From f3e35e24080e3b451037867a002d2608c6eaf71e Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Wed, 17 Aug 2022 23:04:00 -0400 Subject: [PATCH 05/13] Restore dir creation --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5d5deb33bb..ce74631b63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -356,10 +356,10 @@ commands: steps: # - attach_workspace: # at: << parameters.build_dir >> -# - run: | -# mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX} -# touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml -# touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json + - run: | + mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX} + touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml + touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json # - restore_cache: # keys: # - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' From 2ffc7a55261c63afb13cd28fce0b19d8d45a22d9 Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Wed, 17 Aug 2022 23:31:10 -0400 Subject: [PATCH 06/13] Increase parallelism --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce74631b63..84fc0a6ab2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -564,7 +564,7 @@ jobs: type: string executor: << parameters.platform >>_medium working_directory: << pipeline.parameters.build_dir >>/project - parallelism: 4 + parallelism: 32 steps: - prepare_build_dir - checkout @@ -598,9 +598,9 @@ jobs: parameters: platform: type: string - executor: << parameters.platform >>_large + executor: << parameters.platform >>_medium working_directory: << pipeline.parameters.build_dir >>/project - parallelism: 2 + parallelism: 8 environment: E2E_TEST_FILTER: "GO" steps: @@ -638,7 +638,7 @@ jobs: type: string executor: << parameters.platform >>_medium working_directory: << pipeline.parameters.build_dir >>/project - parallelism: 2 + parallelism: 10 environment: E2E_TEST_FILTER: "EXPECT" steps: From bd838f92a391f138d1507df25ef67fbce8a0707b Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Thu, 18 Aug 2022 09:59:34 -0400 Subject: [PATCH 07/13] More integration parallelism --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39e95d2189..d1af4e155e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -599,7 +599,7 @@ jobs: type: string executor: << parameters.platform >>_medium working_directory: << pipeline.parameters.build_dir >>/project - parallelism: 8 + parallelism: 16 environment: E2E_TEST_FILTER: "GO" steps: From f23630f3d663aa1014aee7a45b815187036abf89 Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Thu, 18 Aug 2022 10:48:58 -0400 Subject: [PATCH 08/13] Checkpoint working config --- .circleci/config.yml | 190 ++++++++++++++++--------------------------- 1 file changed, 68 insertions(+), 122 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d1af4e155e..15b64e973c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,41 +64,27 @@ workflows: version: 2 "circleci_build_and_test": jobs: -# - build: -# name: << matrix.platform >>_build -# matrix: &matrix-default -# parameters: -# platform: ["amd64", "arm64"] -# filters: &filters-default -# branches: -# ignore: -# - /rel\/.*/ -# - << pipeline.parameters.valid_nightly_branch >> - - - build_nightly: - name: << matrix.platform >>_build_nightly - matrix: &matrix-nightly - parameters: - platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"] - filters: &filters-nightly - branches: - only: - - /rel\/.*/ - - << pipeline.parameters.valid_nightly_branch >> - context: slack-secrets - - test: name: << matrix.platform >>_test matrix: &matrix-default parameters: platform: ["amd64", "arm64"] + filters: &filters-default + branches: + ignore: + - /rel\/.*/ + - << pipeline.parameters.valid_nightly_branch >> - test_nightly: name: << matrix.platform >>_test_nightly - matrix: - <<: *matrix-nightly - requires: - - << matrix.platform >>_build_nightly + matrix: &matrix-nightly + parameters: + platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"] + filters: &filters-nightly + branches: + only: + - /rel\/.*/ + - << pipeline.parameters.valid_nightly_branch >> context: slack-secrets - integration: @@ -106,15 +92,22 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] -# requires: -# - << matrix.platform >>_build + filters: &filters-default + branches: + ignore: + - /rel\/.*/ + - << pipeline.parameters.valid_nightly_branch >> - integration_nightly: name: << matrix.platform >>_integration_nightly - matrix: - <<: *matrix-nightly - requires: - - << matrix.platform >>_build_nightly + matrix: &matrix-nightly + parameters: + platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"] + filters: &filters-nightly + branches: + only: + - /rel\/.*/ + - << pipeline.parameters.valid_nightly_branch >> context: slack-secrets - e2e_expect: @@ -122,15 +115,22 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] -# requires: -# - << matrix.platform >>_build + filters: &filters-default + branches: + ignore: + - /rel\/.*/ + - << pipeline.parameters.valid_nightly_branch >> - e2e_expect_nightly: name: << matrix.platform >>_e2e_expect_nightly - matrix: - <<: *matrix-nightly - requires: - - << matrix.platform >>_build_nightly + matrix: &matrix-nightly + parameters: + platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"] + filters: &filters-nightly + branches: + only: + - /rel\/.*/ + - << pipeline.parameters.valid_nightly_branch >> context: slack-secrets - e2e_subs: @@ -138,15 +138,22 @@ workflows: matrix: &matrix-default parameters: platform: ["amd64", "arm64"] -# requires: -# - << matrix.platform >>_build + filters: &filters-default + branches: + ignore: + - /rel\/.*/ + - << pipeline.parameters.valid_nightly_branch >> - e2e_subs_nightly: name: << matrix.platform >>_e2e_subs_nightly - matrix: - <<: *matrix-nightly - requires: - - << matrix.platform >>_build_nightly + matrix: &matrix-nightly + parameters: + platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"] + filters: &filters-nightly + branches: + only: + - /rel\/.*/ + - << pipeline.parameters.valid_nightly_branch >> context: - slack-secrets - aws-secrets @@ -227,44 +234,8 @@ commands: export msys2+=" -msys2 -c "\"\$@"\" --" $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool unzip autoconf automake - generic_build_consolidated: - description: TBD - parameters: - build_dir: - type: string - default: << pipeline.parameters.build_dir >> - steps: - - restore_libsodium - - restore_cache: - keys: - - 'go-mod-1.17.9-v3-{{ arch }}-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}' - - restore_cache: - keys: - - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' - - 'go-cache-v3-{{ arch }}-{{ .Branch }}-' - - 'go-cache-v3-{{ arch }}-' - - run: - name: scripts/travis/build.sh --make_debug - command: | - export PATH=$(echo "$PATH" | sed -e "s|:${HOME}/\.go_workspace/bin||g" | sed -e 's|:/usr/local/go/bin||g') - export GOPATH="<< parameters.build_dir >>/go" - export ALGORAND_DEADLOCK=enable - export GIMME_INSTALL_DIR=<< parameters.build_dir >> - export GIMME_ENV_PREFIX=<< parameters.build_dir >>/.gimme/envs - export GIMME_VERSION_PREFIX=<< parameters.build_dir >>/.gimme/versions - scripts/travis/build.sh --make_debug - - cache_libsodium - - save_cache: - key: 'go-mod-1.17.9-v3-{{ arch }}-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}' - paths: - - << parameters.build_dir >>/go/pkg/mod - - save_cache: - key: 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' - paths: - - tmp/go-cache - generic_build: - description: Run basic build and store in workspace for re-use by different architectures + description: Run basic build parameters: build_dir: type: string @@ -298,13 +269,6 @@ commands: key: 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' paths: - tmp/go-cache - - persist_to_workspace: - root: << parameters.build_dir >> - paths: - - project - - go - - gimme - - .gimme cache_libsodium: description: Cache libsodium for build @@ -530,33 +494,6 @@ commands: scripts/travis/test_release.sh jobs: - build: - parameters: - platform: - type: string - executor: << parameters.platform >>_medium - working_directory: << pipeline.parameters.build_dir >>/project - steps: - - prepare_build_dir - - checkout - - prepare_go - - generic_build - - build_nightly: - parameters: - platform: - type: string - executor: << parameters.platform >>_medium - working_directory: << pipeline.parameters.build_dir >>/project - steps: - - prepare_build_dir - - checkout - - prepare_go - - generic_build - - slack/notify: &slack-fail-event - event: fail - template: basic_fail_1 - test: parameters: platform: @@ -568,7 +505,7 @@ jobs: - prepare_build_dir - checkout - prepare_go - - generic_build_consolidated + - generic_build - generic_test: platform: << parameters.platform >> result_subdir: << parameters.platform >>_test @@ -584,14 +521,17 @@ jobs: parallelism: 4 steps: - prepare_build_dir + - checkout - prepare_go + - generic_build - generic_test: platform: << parameters.platform >> result_subdir: << parameters.platform >>_test_nightly no_output_timeout: 45m - upload_coverage - - slack/notify: - <<: *slack-fail-event + - slack/notify: &slack-fail-event + event: fail + template: basic_fail_1 integration: parameters: @@ -606,7 +546,7 @@ jobs: - prepare_build_dir - checkout - prepare_go - - generic_build_consolidated + - generic_build - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform >>_integration @@ -623,7 +563,9 @@ jobs: E2E_TEST_FILTER: "GO" steps: - prepare_build_dir + - checkout - prepare_go + - generic_build - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform >>_integration_nightly @@ -644,7 +586,7 @@ jobs: - prepare_build_dir - checkout - prepare_go - - generic_build_consolidated + - generic_build - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform >>_e2e_expect @@ -661,7 +603,9 @@ jobs: E2E_TEST_FILTER: "EXPECT" steps: - prepare_build_dir + - checkout - prepare_go + - generic_build - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform>>_e2e_expect_nightly @@ -681,7 +625,7 @@ jobs: - prepare_build_dir - checkout - prepare_go - - generic_build_consolidated + - generic_build - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform >>_e2e_subs @@ -701,7 +645,9 @@ jobs: CI_KEEP_TEMP_PLATFORM: "amd64" steps: - prepare_build_dir + - checkout - prepare_go + - generic_build - generic_integration: platform: << parameters.platform >> result_subdir: << parameters.platform >>_e2e_subs_nightly From c70b961b5bb0491a586b300ae3cff629e6fe5f6b Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Thu, 18 Aug 2022 10:54:32 -0400 Subject: [PATCH 09/13] Checkpoint matrix and filters alias usage --- .circleci/config.yml | 72 +++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 48 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 15b64e973c..d0f21294aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,71 +89,47 @@ workflows: - integration: name: << matrix.platform >>_integration - matrix: &matrix-default - parameters: - platform: ["amd64", "arm64"] - filters: &filters-default - branches: - ignore: - - /rel\/.*/ - - << pipeline.parameters.valid_nightly_branch >> + matrix: + <<: *matrix-default + filters: + <<: *filters-default - integration_nightly: name: << matrix.platform >>_integration_nightly - matrix: &matrix-nightly - parameters: - platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"] - filters: &filters-nightly - branches: - only: - - /rel\/.*/ - - << pipeline.parameters.valid_nightly_branch >> + matrix: + <<: *matrix-nightly + filters: + <<: *filters-nightly context: slack-secrets - e2e_expect: name: << matrix.platform >>_e2e_expect - matrix: &matrix-default - parameters: - platform: ["amd64", "arm64"] - filters: &filters-default - branches: - ignore: - - /rel\/.*/ - - << pipeline.parameters.valid_nightly_branch >> + matrix: + <<: *matrix-default + filters: + <<: *filters-default - e2e_expect_nightly: name: << matrix.platform >>_e2e_expect_nightly - matrix: &matrix-nightly - parameters: - platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"] - filters: &filters-nightly - branches: - only: - - /rel\/.*/ - - << pipeline.parameters.valid_nightly_branch >> + matrix: + <<: *matrix-nightly + filters: + <<: *filters-nightly context: slack-secrets - e2e_subs: name: << matrix.platform >>_e2e_subs - matrix: &matrix-default - parameters: - platform: ["amd64", "arm64"] - filters: &filters-default - branches: - ignore: - - /rel\/.*/ - - << pipeline.parameters.valid_nightly_branch >> + matrix: + <<: *matrix-default + filters: + <<: *filters-default - e2e_subs_nightly: name: << matrix.platform >>_e2e_subs_nightly - matrix: &matrix-nightly - parameters: - platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"] - filters: &filters-nightly - branches: - only: - - /rel\/.*/ - - << pipeline.parameters.valid_nightly_branch >> + matrix: + <<: *matrix-nightly + filters: + <<: *filters-nightly context: - slack-secrets - aws-secrets From 6f9b989ff63482ae41f0ed0aacc28a60a625cdf5 Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Thu, 18 Aug 2022 10:56:51 -0400 Subject: [PATCH 10/13] Delete commented out steps --- .circleci/config.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0f21294aa..05dd8a6f09 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -291,17 +291,10 @@ commands: type: string default: << pipeline.parameters.result_path >> steps: -# - attach_workspace: -# at: << parameters.build_dir >> - run: | mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX} touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json -# - restore_cache: -# keys: -# - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}' -# - 'go-cache-v3-{{ arch }}-{{ .Branch }}-' -# - 'go-cache-v3-{{ arch }}-' - run: name: Run build tests no_output_timeout: << parameters.no_output_timeout >> @@ -363,8 +356,6 @@ commands: type: string default: << pipeline.parameters.result_path >> steps: -# - attach_workspace: -# at: << parameters.build_dir >> - run: | mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX} touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml From 6f2cfc98055d1e237f3cb310c49bfe69577745de Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Thu, 18 Aug 2022 11:14:24 -0400 Subject: [PATCH 11/13] Reset to as was parallelism and executors --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 05dd8a6f09..02371066a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -467,7 +467,7 @@ jobs: type: string executor: << parameters.platform >>_medium working_directory: << pipeline.parameters.build_dir >>/project - parallelism: 32 + parallelism: 4 steps: - prepare_build_dir - checkout @@ -504,9 +504,9 @@ jobs: parameters: platform: type: string - executor: << parameters.platform >>_medium + executor: << parameters.platform >>_large working_directory: << pipeline.parameters.build_dir >>/project - parallelism: 16 + parallelism: 2 environment: E2E_TEST_FILTER: "GO" steps: @@ -546,7 +546,7 @@ jobs: type: string executor: << parameters.platform >>_medium working_directory: << pipeline.parameters.build_dir >>/project - parallelism: 10 + parallelism: 2 environment: E2E_TEST_FILTER: "EXPECT" steps: From cbb44866801b572235be78fe183fe41643ba3907 Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Fri, 19 Aug 2022 09:49:53 -0400 Subject: [PATCH 12/13] Add explanatory topology comment --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 02371066a7..011f124fbe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -211,7 +211,12 @@ commands: $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool unzip autoconf automake generic_build: - description: Run basic build + description: > + Run basic build. + + If command execution time increases _appreciably_, revisit CI topology: + * Historically, the command executes _quickly_ (< 3m with resource class >= medium). + * Consequently, it's faster to embed the command in a combined build + test workflow rather than independent build and test workflows. parameters: build_dir: type: string From eca634d92c2586b9188d4f63eeb0cf80c8b08a79 Mon Sep 17 00:00:00 2001 From: michaeldiamant Date: Fri, 19 Aug 2022 09:54:09 -0400 Subject: [PATCH 13/13] Consolidate build pre-requisite steps into generic_build command --- .circleci/config.yml | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 011f124fbe..d4f9805b7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,6 +222,9 @@ commands: type: string default: << pipeline.parameters.build_dir >> steps: + - prepare_build_dir + - checkout + - prepare_go - restore_libsodium - restore_cache: keys: @@ -474,9 +477,6 @@ jobs: working_directory: << pipeline.parameters.build_dir >>/project parallelism: 4 steps: - - prepare_build_dir - - checkout - - prepare_go - generic_build - generic_test: platform: << parameters.platform >> @@ -492,9 +492,6 @@ jobs: working_directory: << pipeline.parameters.build_dir >>/project parallelism: 4 steps: - - prepare_build_dir - - checkout - - prepare_go - generic_build - generic_test: platform: << parameters.platform >> @@ -515,9 +512,6 @@ jobs: environment: E2E_TEST_FILTER: "GO" steps: - - prepare_build_dir - - checkout - - prepare_go - generic_build - generic_integration: platform: << parameters.platform >> @@ -534,9 +528,6 @@ jobs: environment: E2E_TEST_FILTER: "GO" steps: - - prepare_build_dir - - checkout - - prepare_go - generic_build - generic_integration: platform: << parameters.platform >> @@ -555,9 +546,6 @@ jobs: environment: E2E_TEST_FILTER: "EXPECT" steps: - - prepare_build_dir - - checkout - - prepare_go - generic_build - generic_integration: platform: << parameters.platform >> @@ -574,9 +562,6 @@ jobs: environment: E2E_TEST_FILTER: "EXPECT" steps: - - prepare_build_dir - - checkout - - prepare_go - generic_build - generic_integration: platform: << parameters.platform >> @@ -594,9 +579,6 @@ jobs: environment: E2E_TEST_FILTER: "SCRIPTS" steps: - - prepare_build_dir - - checkout - - prepare_go - generic_build - generic_integration: platform: << parameters.platform >> @@ -616,9 +598,6 @@ jobs: # one of the platforms in the matrix. CI_KEEP_TEMP_PLATFORM: "amd64" steps: - - prepare_build_dir - - checkout - - prepare_go - generic_build - generic_integration: platform: << parameters.platform >>