Atree storage migration #29583
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - 'auto-cadence-upgrade/**' | |
| - staging | |
| - trying | |
| - 'feature/**' | |
| - 'v[0-9]+.[0-9]+' | |
| pull_request: | |
| branches: | |
| - master* | |
| - 'auto-cadence-upgrade/**' | |
| - 'feature/**' | |
| - 'v[0-9]+.[0-9]+' | |
| merge_group: | |
| branches: | |
| - master | |
| env: | |
| GO_VERSION: "1.20" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| golangci: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| dir: [./, ./integration/, ./insecure/] | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Run go generate | |
| run: go generate | |
| working-directory: ${{ matrix.dir }} | |
| - name: Run golangci-lint | |
| uses: golangci/golangci-lint-action@v3 | |
| with: | |
| # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. | |
| version: v1.54 | |
| args: -v | |
| working-directory: ${{ matrix.dir }} | |
| # https://github.com/golangci/golangci-lint-action/issues/244 | |
| skip-cache: true | |
| tidy: | |
| name: Tidy | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Run tidy | |
| run: make tidy | |
| - name: code sanity check | |
| run: make code-sanity-check | |
| create-dynamic-test-matrix: | |
| name: Create Dynamic Test Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Set Test Matrix | |
| id: set-test-matrix | |
| run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine/access engine/collection engine/common engine/consensus engine/execution/ingestion:buildjet-8vcpu-ubuntu-2204 engine/execution/computation engine/execution engine/verification engine:buildjet-4vcpu-ubuntu-2204 fvm ledger module/dkg module:buildjet-4vcpu-ubuntu-2204 network/alsp network/test/cohort1:buildjet-16vcpu-ubuntu-2204 network/test/cohort2:buildjet-4vcpu-ubuntu-2204 network/p2p/connection network/p2p/node:buildjet-4vcpu-ubuntu-2204 network/p2p/scoring network/p2p network state storage utils | |
| unit-test: | |
| name: Unit Tests (${{ matrix.targets.name }}) | |
| needs: create-dynamic-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}} | |
| ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 | |
| runs-on: ${{ matrix.targets.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Setup tests (${{ matrix.targets.name }}) | |
| run: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools | |
| - name: Run tests (${{ matrix.targets.name }}) | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.txt | |
| flags: unittests | |
| name: codecov-umbrella | |
| unit-test-modules: | |
| name: Unit Tests (Modules) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: insecure | |
| setup: install-tools | |
| retries: 5 | |
| race: 0 | |
| runner: buildjet-4vcpu-ubuntu-2204 | |
| - name: integration | |
| setup: install-tools | |
| retries: 5 | |
| race: 0 | |
| runner: buildjet-4vcpu-ubuntu-2204 | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Setup tests (${{ matrix.name }}) | |
| run: make ${{ matrix.setup }} | |
| - name: Run tests (${{ matrix.name }}) | |
| env: | |
| RACE_DETECTOR: ${{ matrix.race }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: ${{ matrix.retries }} | |
| # run test target inside each module's root | |
| command: VERBOSE=1 make -C ${{ matrix.name }} test | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.txt | |
| flags: unittests | |
| name: codecov-umbrella | |
| docker-build: | |
| name: Docker Build | |
| runs-on: buildjet-16vcpu-ubuntu-2204 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| with: | |
| # all tags are needed for integration tests | |
| fetch-depth: 0 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Docker build | |
| run: make docker-build-flow docker-build-flow-corrupt | |
| - name: Save Docker images | |
| run: | | |
| docker save \ | |
| gcr.io/flow-container-registry/access:latest \ | |
| gcr.io/flow-container-registry/collection:latest \ | |
| gcr.io/flow-container-registry/consensus:latest \ | |
| gcr.io/flow-container-registry/execution:latest \ | |
| gcr.io/flow-container-registry/ghost:latest \ | |
| gcr.io/flow-container-registry/observer:latest \ | |
| gcr.io/flow-container-registry/verification:latest \ | |
| gcr.io/flow-container-registry/access-corrupted:latest \ | |
| gcr.io/flow-container-registry/execution-corrupted:latest \ | |
| gcr.io/flow-container-registry/verification-corrupted:latest > flow-docker-images.tar | |
| - name: Cache Docker images | |
| uses: actions/cache@v3 | |
| with: | |
| path: flow-docker-images.tar | |
| # use the workflow run id as part of the cache key to ensure these docker images will only be used for a single workflow run | |
| key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }} | |
| integration-test: | |
| name: Integration Tests | |
| needs: docker-build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Access Cohort1 Integration Tests | |
| make: make -C integration access-cohort1-tests | |
| runner: buildjet-4vcpu-ubuntu-2204 | |
| - name: Access Cohort2 Integration Tests | |
| make: make -C integration access-cohort2-tests | |
| runner: ubuntu-latest | |
| - name: Access Cohort3 Integration Tests | |
| make: make -C integration access-cohort3-tests | |
| runner: ubuntu-latest | |
| # test suite has single test which is flaky and needs to be fixed - reminder here to put it back when it's fixed | |
| # - name: BFT (Framework) Integration Tests | |
| # make: make -C integration bft-framework-tests | |
| # runner: ubuntu-latest | |
| - name: BFT (Protocol) Integration Tests | |
| make: make -C integration bft-protocol-tests | |
| runner: buildjet-8vcpu-ubuntu-2204 | |
| - name: BFT (Gossipsub) Integration Tests | |
| make: make -C integration bft-gossipsub-tests | |
| runner: ubuntu-latest | |
| - name: Collection Integration Tests | |
| make: make -C integration collection-tests | |
| runner: ubuntu-latest | |
| - name: Consensus Integration Tests | |
| make: make -C integration consensus-tests | |
| runner: ubuntu-latest | |
| - name: Epoch Cohort1 Integration Tests | |
| make: make -C integration epochs-cohort1-tests | |
| runner: buildjet-8vcpu-ubuntu-2204 | |
| - name: Epoch Cohort2 Integration Tests | |
| make: make -C integration epochs-cohort2-tests | |
| runner: buildjet-4vcpu-ubuntu-2204 | |
| - name: Execution Integration Tests | |
| make: make -C integration execution-tests | |
| runner: ubuntu-latest | |
| - name: Ghost Integration Tests | |
| make: make -C integration ghost-tests | |
| runner: ubuntu-latest | |
| - name: MVP Integration Tests | |
| make: make -C integration mvp-tests | |
| runner: ubuntu-latest | |
| - name: Network Integration Tests | |
| make: make -C integration network-tests | |
| runner: ubuntu-latest | |
| - name: Verification Integration Tests | |
| make: make -C integration verification-tests | |
| runner: ubuntu-latest | |
| - name: Upgrade Integration Tests | |
| make: make -C integration upgrades-tests | |
| runner: ubuntu-latest | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| with: | |
| # all tags are needed for integration tests | |
| fetch-depth: 0 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Load cached Docker images | |
| uses: actions/cache@v3 | |
| with: | |
| path: flow-docker-images.tar | |
| # use the same cache key as the docker-build job | |
| key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }} | |
| - name: Load Docker images | |
| run: docker load -i flow-docker-images.tar | |
| - name: Run tests (${{ matrix.name }}) | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 ${{ matrix.make }} |