Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI: Speed up CircleCI by folding build workflow downstream #4426

Merged
merged 14 commits into from
Aug 19, 2022
Merged
113 changes: 36 additions & 77 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ workflows:
version: 2
"circleci_build_and_test":
jobs:
- build:
name: << matrix.platform >>_build
- test:
name: << matrix.platform >>_test
matrix: &matrix-default
parameters:
platform: ["amd64", "arm64"]
Expand All @@ -75,8 +75,8 @@ workflows:
- /rel\/.*/
- << pipeline.parameters.valid_nightly_branch >>

- build_nightly:
name: << matrix.platform >>_build_nightly
- test_nightly:
name: << matrix.platform >>_test_nightly
matrix: &matrix-nightly
parameters:
platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"]
Expand All @@ -87,64 +87,49 @@ workflows:
- << pipeline.parameters.valid_nightly_branch >>
context: slack-secrets

- test:
name: << matrix.platform >>_test
matrix:
<<: *matrix-default
requires:
- << matrix.platform >>_build

- test_nightly:
name: << matrix.platform >>_test_nightly
matrix:
<<: *matrix-nightly
requires:
- << matrix.platform >>_build_nightly
context: slack-secrets

- integration:
name: << matrix.platform >>_integration
matrix:
<<: *matrix-default
requires:
- << matrix.platform >>_build
filters:
<<: *filters-default

- integration_nightly:
name: << matrix.platform >>_integration_nightly
matrix:
<<: *matrix-nightly
requires:
- << matrix.platform >>_build_nightly
filters:
<<: *filters-nightly
context: slack-secrets

- e2e_expect:
name: << matrix.platform >>_e2e_expect
matrix:
<<: *matrix-default
requires:
- << matrix.platform >>_build
filters:
<<: *filters-default

- e2e_expect_nightly:
name: << matrix.platform >>_e2e_expect_nightly
matrix:
<<: *matrix-nightly
requires:
- << matrix.platform >>_build_nightly
filters:
<<: *filters-nightly
context: slack-secrets

- e2e_subs:
name: << matrix.platform >>_e2e_subs
matrix:
<<: *matrix-default
requires:
- << matrix.platform >>_build
filters:
<<: *filters-default

- e2e_subs_nightly:
name: << matrix.platform >>_e2e_subs_nightly
matrix:
<<: *matrix-nightly
requires:
- << matrix.platform >>_build_nightly
filters:
<<: *filters-nightly
context:
- slack-secrets
- aws-secrets
Expand Down Expand Up @@ -226,7 +211,7 @@ commands:
$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool unzip autoconf automake

generic_build:
description: Run basic build and store in workspace for re-use by different architectures
description: Run basic build
jannotti marked this conversation as resolved.
Show resolved Hide resolved
parameters:
build_dir:
type: string
Expand Down Expand Up @@ -260,13 +245,6 @@ commands:
key: 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}'
paths:
- tmp/go-cache
- persist_to_workspace:
root: << parameters.build_dir >>
paths:
- project
- go
- gimme
- .gimme

cache_libsodium:
description: Cache libsodium for build
Expand Down Expand Up @@ -313,17 +291,10 @@ commands:
type: string
default: << pipeline.parameters.result_path >>
steps:
- attach_workspace:
at: << parameters.build_dir >>
- run: |
mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}
touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml
touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json
- restore_cache:
keys:
- 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}'
- 'go-cache-v3-{{ arch }}-{{ .Branch }}-'
- 'go-cache-v3-{{ arch }}-'
- run:
name: Run build tests
no_output_timeout: << parameters.no_output_timeout >>
Expand Down Expand Up @@ -385,8 +356,6 @@ commands:
type: string
default: << pipeline.parameters.result_path >>
steps:
- attach_workspace:
at: << parameters.build_dir >>
- run: |
mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}
touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml
Expand Down Expand Up @@ -492,33 +461,6 @@ commands:
scripts/travis/test_release.sh

jobs:
build:
parameters:
platform:
type: string
executor: << parameters.platform >>_medium
working_directory: << pipeline.parameters.build_dir >>/project
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build

build_nightly:
parameters:
platform:
type: string
executor: << parameters.platform >>_medium
working_directory: << pipeline.parameters.build_dir >>/project
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
- slack/notify: &slack-fail-event
event: fail
template: basic_fail_1

test:
parameters:
platform:
Expand All @@ -528,7 +470,9 @@ jobs:
parallelism: 4
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
jannotti marked this conversation as resolved.
Show resolved Hide resolved
- generic_test:
platform: << parameters.platform >>
result_subdir: << parameters.platform >>_test
Expand All @@ -544,14 +488,17 @@ jobs:
parallelism: 4
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
- generic_test:
platform: << parameters.platform >>
result_subdir: << parameters.platform >>_test_nightly
no_output_timeout: 45m
- upload_coverage
- slack/notify:
<<: *slack-fail-event
- slack/notify: &slack-fail-event
event: fail
template: basic_fail_1

integration:
parameters:
Expand All @@ -564,7 +511,9 @@ jobs:
E2E_TEST_FILTER: "GO"
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
- generic_integration:
platform: << parameters.platform >>
result_subdir: << parameters.platform >>_integration
Expand All @@ -581,7 +530,9 @@ jobs:
E2E_TEST_FILTER: "GO"
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
- generic_integration:
platform: << parameters.platform >>
result_subdir: << parameters.platform >>_integration_nightly
Expand All @@ -600,7 +551,9 @@ jobs:
E2E_TEST_FILTER: "EXPECT"
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
- generic_integration:
platform: << parameters.platform >>
result_subdir: << parameters.platform >>_e2e_expect
Expand All @@ -617,7 +570,9 @@ jobs:
E2E_TEST_FILTER: "EXPECT"
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
- generic_integration:
platform: << parameters.platform >>
result_subdir: << parameters.platform>>_e2e_expect_nightly
Expand All @@ -635,7 +590,9 @@ jobs:
E2E_TEST_FILTER: "SCRIPTS"
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
- generic_integration:
platform: << parameters.platform >>
result_subdir: << parameters.platform >>_e2e_subs
Expand All @@ -655,7 +612,9 @@ jobs:
CI_KEEP_TEMP_PLATFORM: "amd64"
steps:
- prepare_build_dir
- checkout
- prepare_go
- generic_build
- generic_integration:
platform: << parameters.platform >>
result_subdir: << parameters.platform >>_e2e_subs_nightly
Expand Down