From 0ac04950c3fe0cc3e9041b426333b6d5dec5f0c6 Mon Sep 17 00:00:00 2001 From: Daniel Wedul Date: Thu, 6 Oct 2022 14:55:56 -0600 Subject: [PATCH 1/8] [1125]: Remove badgerdb from the matrices in the sims jobs. --- .github/workflows/sims.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index 41c178a099..2f64e9d1f7 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -206,7 +206,7 @@ jobs: # The test-sim-multi-seed-short test should take 30-40 minutes. # TODO: Add the "after-import" test once it's fixed: https://github.com/provenance-io/provenance/issues/754 test: ["import-export", "multi-seed-short"] - db-backend: ["goleveldb", "cleveldb", "badgerdb"] + db-backend: ["goleveldb", "cleveldb"] # Not putting "macos-latest" in this because: # a) They take longer than on ubuntu to begin with. # b) We're limited to 5 Mac runners at once so they get queued up, taking even longer to finish all of them. @@ -279,7 +279,7 @@ jobs: # The test-sim-simple test is pretty quick and should be able to identify glaring problems. # The test-sim-benchmark is handy to have for each db type. test: ["simple", "benchmark"] - db-backend: ["goleveldb", "cleveldb", "badgerdb"] + db-backend: ["goleveldb", "cleveldb"] os: ["ubuntu-latest", "macos-latest"] runs-on: ${{ matrix.os }} steps: From 29d447392599d8f6b0b1ae6d251aab1854dcc12d Mon Sep 17 00:00:00 2001 From: Daniel Wedul Date: Mon, 10 Oct 2022 14:25:50 -0600 Subject: [PATCH 2/8] [1125]: Remove cleveldb and mac stuff from the sims workflow. --- .github/workflows/sims.yml | 143 ++----------------------------------- 1 file changed, 6 insertions(+), 137 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index 2f64e9d1f7..30422015b9 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -38,35 +38,22 @@ jobs: id: def-vars run: | echo "::set-output name=file-prefix::sim-test-${GITHUB_SHA:0:7}-${GITHUB_RUN_ATTEMPT}" - echo "::set-output name=db-cache-key-hash::${{ hashFiles('scripts/cleveldb_build_and_install.sh') }}" echo "::set-output name=go-cache-key-hash::${{ hashFiles('go.sum') }}" outputs: go-version: 1.18 should-run: ${{ env.GIT_DIFF }} file-prefix: ${{ steps.def-vars.outputs.file-prefix }} - db-cache-key-suffix: sims-db3-${{ steps.def-vars.outputs.db-cache-key-hash }} - # In Order: - # * The leveldb repo tarball - # * The directory extracted from the leveldb tarball - db-cache-path: | - leveldb*.tar.gz - leveldb-* go-cache-key-suffix: sims-go3-${{ steps.def-vars.outputs.go-cache-key-hash }} # In Order: # * Go binary directory # * Go module directory # * Go build cache (Linux) - # * Go build cache (Mac) go-cache-path: | ~/go/bin ~/go/pkg/mod ~/.cache/go-build - ~/Library/Caches/go-build build-linux: - # Note: Can't use a matrix to combine build-linux and build-mac because they'd both have the same job id and that would cause - # problems below with the "needs" directive on the tests. Basically, as soon as the first one finished, the tests - # would kick off for both OSes even though one might not be ready yet. needs: setup if: needs.setup.outputs.should-run runs-on: ubuntu-latest @@ -77,16 +64,8 @@ jobs: echo " go-version: [${{ needs.setup.outputs.go-version }}]" echo " should-run: [${{ needs.setup.outputs.should-run }}]" echo " file-prefix: [${{ needs.setup.outputs.file-prefix }}]" - echo "db-cache-key-suffix: [${{ needs.setup.outputs.db-cache-key-suffix }}]" - echo " db-cache-path: [${{ needs.setup.outputs.db-cache-path }}]" echo "go-cache-key-suffix: [${{ needs.setup.outputs.go-cache-key-suffix }}]" echo " go-cache-path: [${{ needs.setup.outputs.go-cache-path }}]" - - uses: actions/cache@v3 - name: Load db cache - id: db-cache-setup - with: - key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} - path: ${{ needs.setup.outputs.db-cache-path }} - uses: actions/cache@v3 name: Load go cache id: go-cache-setup @@ -99,86 +78,6 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev wget curl build-essential cmake gcc sqlite3 - - name: Build cleveldb - if: steps.db-cache-setup.outputs.cache-hit != 'true' - run: | - export CLEVELDB_DO_BUILD='true' - export CLEVELDB_DO_INSTALL='false' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - - name: Install cleveldb - run: | - export CLEVELDB_DO_BUILD='false' - export CLEVELDB_DO_INSTALL='true' - export CLEVELDB_SUDO='true' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - - uses: actions/setup-go@v3 - with: - go-version: ${{ needs.setup.outputs.go-version }} - - name: Display go version - run: go version - - name: Install runsim - if: steps.go-cache-setup.outputs.cache-hit != 'true' - run: export GO111MODULE="on" && go get github.com/cosmos/tools/cmd/runsim@v1.0.0 - - name: Update provwasm contract - run: make download-smart-contracts - - name: Build provenanced - run: make build - - name: Provenanced version - run: build/provenanced version --long - - build-mac: - # Note: Can't use a matrix to combine build-linux and build-mac because they'd both have the same job id and that would cause - # problems below with the "needs" directive on the tests. Basically, as soon as the first one finished, the tests - # would kick off for both OSes even though one might not be ready yet. - needs: setup - if: needs.setup.outputs.should-run - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - name: Output setup - run: | - echo " go-version: [${{ needs.setup.outputs.go-version }}]" - echo " should-run: [${{ needs.setup.outputs.should-run }}]" - echo " file-prefix: [${{ needs.setup.outputs.file-prefix }}]" - echo "db-cache-key-suffix: [${{ needs.setup.outputs.db-cache-key-suffix }}]" - echo " db-cache-path: [${{ needs.setup.outputs.db-cache-path }}]" - echo "go-cache-key-suffix: [${{ needs.setup.outputs.go-cache-key-suffix }}]" - echo " go-cache-path: [${{ needs.setup.outputs.go-cache-path }}]" - - uses: actions/cache@v3 - name: Load db cache - id: db-cache-setup - with: - key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} - path: ${{ needs.setup.outputs.db-cache-path }} - - uses: actions/cache@v3 - name: Load go cache - id: go-cache-setup - with: - key: ${{ runner.os }}-${{ needs.setup.outputs.go-cache-key-suffix }} - path: ${{ needs.setup.outputs.go-cache-path }} - - name: Setup build environment - run: | - brew install cmake sqlite3 - - name: Build cleveldb - if: steps.db-cache-setup.outputs.cache-hit != 'true' - # As of 2022-06-13 (it was a Monday), the macos-latest runners do not have the nproc command. - # But have 3 cores according to https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources - # The linux runners have nproc, so we don't need to set the _JOBS in that job. - run: | - export CLEVELDB_JOBS=3 - export CLEVELDB_DO_BUILD='true' - export CLEVELDB_DO_INSTALL='false' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - - name: Install cleveldb - run: | - export CLEVELDB_DO_BUILD='false' - export CLEVELDB_DO_INSTALL='true' - export CLEVELDB_SUDO='true' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - uses: actions/setup-go@v3 with: go-version: ${{ needs.setup.outputs.go-version }} @@ -206,20 +105,11 @@ jobs: # The test-sim-multi-seed-short test should take 30-40 minutes. # TODO: Add the "after-import" test once it's fixed: https://github.com/provenance-io/provenance/issues/754 test: ["import-export", "multi-seed-short"] - db-backend: ["goleveldb", "cleveldb"] - # Not putting "macos-latest" in this because: - # a) They take longer than on ubuntu to begin with. - # b) We're limited to 5 Mac runners at once so they get queued up, taking even longer to finish all of them. - # c) Most of the devs use a mac, so it's less likely that a Mac-specific bug goes unnoticed without these tests. + db-backend: ["goleveldb"] os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 - name: Load db cache - with: - key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} - path: ${{ needs.setup.outputs.db-cache-path }} - uses: actions/cache@v3 name: Load go cache with: @@ -228,14 +118,6 @@ jobs: - name: Define test-logs id: test-logs run: echo "::set-output name=test-logs::${{ needs.setup.outputs.file-prefix }}-${{ matrix.test }}-${{ matrix.db-backend }}-${{ matrix.os }}" - - name: Install cleveldb - if: matrix.db-backend == 'cleveldb' - run: | - export CLEVELDB_DO_BUILD='false' - export CLEVELDB_DO_INSTALL='true' - export CLEVELDB_SUDO='true' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - uses: actions/setup-go@v3 with: go-version: ${{ needs.setup.outputs.go-version }} @@ -271,7 +153,7 @@ jobs: go-test-multi-db: # These are tests that use go test to run (see sims.mk), and that we want to test using different database backends. - needs: [setup, build-linux, build-mac] + needs: [setup, build-linux] if: needs.setup.outputs.should-run strategy: fail-fast: false @@ -279,16 +161,11 @@ jobs: # The test-sim-simple test is pretty quick and should be able to identify glaring problems. # The test-sim-benchmark is handy to have for each db type. test: ["simple", "benchmark"] - db-backend: ["goleveldb", "cleveldb"] - os: ["ubuntu-latest", "macos-latest"] + db-backend: ["goleveldb"] + os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 - name: Load db cache - with: - key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} - path: ${{ needs.setup.outputs.db-cache-path }} - uses: actions/cache@v3 name: Load go cache with: @@ -297,14 +174,6 @@ jobs: - name: Define test-logs id: test-logs run: echo "::set-output name=test-logs::${{ needs.setup.outputs.file-prefix }}-${{ matrix.test }}-${{ matrix.db-backend }}-${{ matrix.os }}" - - name: Install cleveldb - if: matrix.db-backend == 'cleveldb' - run: | - export CLEVELDB_DO_BUILD='false' - export CLEVELDB_DO_INSTALL='true' - export CLEVELDB_SUDO='true' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - uses: actions/setup-go@v3 with: go-version: ${{ needs.setup.outputs.go-version }} @@ -326,7 +195,7 @@ jobs: go-test-single-db: # These are tests that use go test to run (see sims.mk), and that we don't care about testing using different database backends. - needs: [setup, build-linux, build-mac] + needs: [setup, build-linux] if: needs.setup.outputs.should-run strategy: fail-fast: false @@ -334,7 +203,7 @@ jobs: # The test-sim-nondeterminism test hard-codes the db backend to use memdb. # The test-sim-benchmark-invariants test can use different db backends, but to save resources, is down here. test: ["nondeterminism", "benchmark-invariants"] - os: ["ubuntu-latest", "macos-latest"] + os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 From 14cad2fe6e52f85c6534836c43d47d5b6c7318f4 Mon Sep 17 00:00:00 2001 From: Daniel Wedul Date: Mon, 10 Oct 2022 14:27:07 -0600 Subject: [PATCH 3/8] [754]: Re-enable the now fixed after-import sim test. --- .github/workflows/sims.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index 30422015b9..2c1b054017 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -100,11 +100,7 @@ jobs: strategy: fail-fast: false matrix: - # The test-sim-import-export test can take up to two hours. - # The test-sim-after-import test can take even longer. - # The test-sim-multi-seed-short test should take 30-40 minutes. - # TODO: Add the "after-import" test once it's fixed: https://github.com/provenance-io/provenance/issues/754 - test: ["import-export", "multi-seed-short"] + test: ["import-export", "multi-seed-short", "after-import"] db-backend: ["goleveldb"] os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} From 0eb825c3fe44dc50d70ce059243ceda7149d0a4a Mon Sep 17 00:00:00 2001 From: Daniel Wedul Date: Mon, 10 Oct 2022 14:36:27 -0600 Subject: [PATCH 4/8] Revert "[1125]: Remove cleveldb and mac stuff from the sims workflow." This reverts commit 29d447392599d8f6b0b1ae6d251aab1854dcc12d. # Conflicts: # .github/workflows/sims.yml --- .github/workflows/sims.yml | 147 +++++++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index 2c1b054017..b5b1270f86 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -38,22 +38,35 @@ jobs: id: def-vars run: | echo "::set-output name=file-prefix::sim-test-${GITHUB_SHA:0:7}-${GITHUB_RUN_ATTEMPT}" + echo "::set-output name=db-cache-key-hash::${{ hashFiles('scripts/cleveldb_build_and_install.sh') }}" echo "::set-output name=go-cache-key-hash::${{ hashFiles('go.sum') }}" outputs: go-version: 1.18 should-run: ${{ env.GIT_DIFF }} file-prefix: ${{ steps.def-vars.outputs.file-prefix }} + db-cache-key-suffix: sims-db3-${{ steps.def-vars.outputs.db-cache-key-hash }} + # In Order: + # * The leveldb repo tarball + # * The directory extracted from the leveldb tarball + db-cache-path: | + leveldb*.tar.gz + leveldb-* go-cache-key-suffix: sims-go3-${{ steps.def-vars.outputs.go-cache-key-hash }} # In Order: # * Go binary directory # * Go module directory # * Go build cache (Linux) + # * Go build cache (Mac) go-cache-path: | ~/go/bin ~/go/pkg/mod ~/.cache/go-build + ~/Library/Caches/go-build build-linux: + # Note: Can't use a matrix to combine build-linux and build-mac because they'd both have the same job id and that would cause + # problems below with the "needs" directive on the tests. Basically, as soon as the first one finished, the tests + # would kick off for both OSes even though one might not be ready yet. needs: setup if: needs.setup.outputs.should-run runs-on: ubuntu-latest @@ -64,8 +77,16 @@ jobs: echo " go-version: [${{ needs.setup.outputs.go-version }}]" echo " should-run: [${{ needs.setup.outputs.should-run }}]" echo " file-prefix: [${{ needs.setup.outputs.file-prefix }}]" + echo "db-cache-key-suffix: [${{ needs.setup.outputs.db-cache-key-suffix }}]" + echo " db-cache-path: [${{ needs.setup.outputs.db-cache-path }}]" echo "go-cache-key-suffix: [${{ needs.setup.outputs.go-cache-key-suffix }}]" echo " go-cache-path: [${{ needs.setup.outputs.go-cache-path }}]" + - uses: actions/cache@v3 + name: Load db cache + id: db-cache-setup + with: + key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} + path: ${{ needs.setup.outputs.db-cache-path }} - uses: actions/cache@v3 name: Load go cache id: go-cache-setup @@ -78,6 +99,86 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev wget curl build-essential cmake gcc sqlite3 + - name: Build cleveldb + if: steps.db-cache-setup.outputs.cache-hit != 'true' + run: | + export CLEVELDB_DO_BUILD='true' + export CLEVELDB_DO_INSTALL='false' + export CLEVELDB_DO_CLEANUP='false' + make cleveldb + - name: Install cleveldb + run: | + export CLEVELDB_DO_BUILD='false' + export CLEVELDB_DO_INSTALL='true' + export CLEVELDB_SUDO='true' + export CLEVELDB_DO_CLEANUP='false' + make cleveldb + - uses: actions/setup-go@v3 + with: + go-version: ${{ needs.setup.outputs.go-version }} + - name: Display go version + run: go version + - name: Install runsim + if: steps.go-cache-setup.outputs.cache-hit != 'true' + run: export GO111MODULE="on" && go get github.com/cosmos/tools/cmd/runsim@v1.0.0 + - name: Update provwasm contract + run: make download-smart-contracts + - name: Build provenanced + run: make build + - name: Provenanced version + run: build/provenanced version --long + + build-mac: + # Note: Can't use a matrix to combine build-linux and build-mac because they'd both have the same job id and that would cause + # problems below with the "needs" directive on the tests. Basically, as soon as the first one finished, the tests + # would kick off for both OSes even though one might not be ready yet. + needs: setup + if: needs.setup.outputs.should-run + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Output setup + run: | + echo " go-version: [${{ needs.setup.outputs.go-version }}]" + echo " should-run: [${{ needs.setup.outputs.should-run }}]" + echo " file-prefix: [${{ needs.setup.outputs.file-prefix }}]" + echo "db-cache-key-suffix: [${{ needs.setup.outputs.db-cache-key-suffix }}]" + echo " db-cache-path: [${{ needs.setup.outputs.db-cache-path }}]" + echo "go-cache-key-suffix: [${{ needs.setup.outputs.go-cache-key-suffix }}]" + echo " go-cache-path: [${{ needs.setup.outputs.go-cache-path }}]" + - uses: actions/cache@v3 + name: Load db cache + id: db-cache-setup + with: + key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} + path: ${{ needs.setup.outputs.db-cache-path }} + - uses: actions/cache@v3 + name: Load go cache + id: go-cache-setup + with: + key: ${{ runner.os }}-${{ needs.setup.outputs.go-cache-key-suffix }} + path: ${{ needs.setup.outputs.go-cache-path }} + - name: Setup build environment + run: | + brew install cmake sqlite3 + - name: Build cleveldb + if: steps.db-cache-setup.outputs.cache-hit != 'true' + # As of 2022-06-13 (it was a Monday), the macos-latest runners do not have the nproc command. + # But have 3 cores according to https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + # The linux runners have nproc, so we don't need to set the _JOBS in that job. + run: | + export CLEVELDB_JOBS=3 + export CLEVELDB_DO_BUILD='true' + export CLEVELDB_DO_INSTALL='false' + export CLEVELDB_DO_CLEANUP='false' + make cleveldb + - name: Install cleveldb + run: | + export CLEVELDB_DO_BUILD='false' + export CLEVELDB_DO_INSTALL='true' + export CLEVELDB_SUDO='true' + export CLEVELDB_DO_CLEANUP='false' + make cleveldb - uses: actions/setup-go@v3 with: go-version: ${{ needs.setup.outputs.go-version }} @@ -100,12 +201,25 @@ jobs: strategy: fail-fast: false matrix: + # The test-sim-import-export test can take up to two hours. + # The test-sim-after-import test can take even longer. + # The test-sim-multi-seed-short test should take 30-40 minutes. + # TODO: Add the "after-import" test once it's fixed: https://github.com/provenance-io/provenance/issues/754 test: ["import-export", "multi-seed-short", "after-import"] - db-backend: ["goleveldb"] + db-backend: ["goleveldb", "cleveldb"] + # Not putting "macos-latest" in this because: + # a) They take longer than on ubuntu to begin with. + # b) We're limited to 5 Mac runners at once so they get queued up, taking even longer to finish all of them. + # c) Most of the devs use a mac, so it's less likely that a Mac-specific bug goes unnoticed without these tests. os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 + - uses: actions/cache@v3 + name: Load db cache + with: + key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} + path: ${{ needs.setup.outputs.db-cache-path }} - uses: actions/cache@v3 name: Load go cache with: @@ -114,6 +228,14 @@ jobs: - name: Define test-logs id: test-logs run: echo "::set-output name=test-logs::${{ needs.setup.outputs.file-prefix }}-${{ matrix.test }}-${{ matrix.db-backend }}-${{ matrix.os }}" + - name: Install cleveldb + if: matrix.db-backend == 'cleveldb' + run: | + export CLEVELDB_DO_BUILD='false' + export CLEVELDB_DO_INSTALL='true' + export CLEVELDB_SUDO='true' + export CLEVELDB_DO_CLEANUP='false' + make cleveldb - uses: actions/setup-go@v3 with: go-version: ${{ needs.setup.outputs.go-version }} @@ -149,7 +271,7 @@ jobs: go-test-multi-db: # These are tests that use go test to run (see sims.mk), and that we want to test using different database backends. - needs: [setup, build-linux] + needs: [setup, build-linux, build-mac] if: needs.setup.outputs.should-run strategy: fail-fast: false @@ -157,11 +279,16 @@ jobs: # The test-sim-simple test is pretty quick and should be able to identify glaring problems. # The test-sim-benchmark is handy to have for each db type. test: ["simple", "benchmark"] - db-backend: ["goleveldb"] - os: ["ubuntu-latest"] + db-backend: ["goleveldb", "cleveldb"] + os: ["ubuntu-latest", "macos-latest"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 + - uses: actions/cache@v3 + name: Load db cache + with: + key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} + path: ${{ needs.setup.outputs.db-cache-path }} - uses: actions/cache@v3 name: Load go cache with: @@ -170,6 +297,14 @@ jobs: - name: Define test-logs id: test-logs run: echo "::set-output name=test-logs::${{ needs.setup.outputs.file-prefix }}-${{ matrix.test }}-${{ matrix.db-backend }}-${{ matrix.os }}" + - name: Install cleveldb + if: matrix.db-backend == 'cleveldb' + run: | + export CLEVELDB_DO_BUILD='false' + export CLEVELDB_DO_INSTALL='true' + export CLEVELDB_SUDO='true' + export CLEVELDB_DO_CLEANUP='false' + make cleveldb - uses: actions/setup-go@v3 with: go-version: ${{ needs.setup.outputs.go-version }} @@ -191,7 +326,7 @@ jobs: go-test-single-db: # These are tests that use go test to run (see sims.mk), and that we don't care about testing using different database backends. - needs: [setup, build-linux] + needs: [setup, build-linux, build-mac] if: needs.setup.outputs.should-run strategy: fail-fast: false @@ -199,7 +334,7 @@ jobs: # The test-sim-nondeterminism test hard-codes the db backend to use memdb. # The test-sim-benchmark-invariants test can use different db backends, but to save resources, is down here. test: ["nondeterminism", "benchmark-invariants"] - os: ["ubuntu-latest"] + os: ["ubuntu-latest", "macos-latest"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 From 1751bd600cd9b0ebda9cc8b0c209ec44a387d2ed Mon Sep 17 00:00:00 2001 From: Daniel Wedul Date: Mon, 10 Oct 2022 14:37:42 -0600 Subject: [PATCH 5/8] [1125]: Remove the mac stuff from the sims workflow. --- .github/workflows/sims.yml | 83 ++------------------------------------ 1 file changed, 4 insertions(+), 79 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index b5b1270f86..bf4d5133c2 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -56,17 +56,12 @@ jobs: # * Go binary directory # * Go module directory # * Go build cache (Linux) - # * Go build cache (Mac) go-cache-path: | ~/go/bin ~/go/pkg/mod ~/.cache/go-build - ~/Library/Caches/go-build build-linux: - # Note: Can't use a matrix to combine build-linux and build-mac because they'd both have the same job id and that would cause - # problems below with the "needs" directive on the tests. Basically, as soon as the first one finished, the tests - # would kick off for both OSes even though one might not be ready yet. needs: setup if: needs.setup.outputs.should-run runs-on: ubuntu-latest @@ -128,72 +123,6 @@ jobs: - name: Provenanced version run: build/provenanced version --long - build-mac: - # Note: Can't use a matrix to combine build-linux and build-mac because they'd both have the same job id and that would cause - # problems below with the "needs" directive on the tests. Basically, as soon as the first one finished, the tests - # would kick off for both OSes even though one might not be ready yet. - needs: setup - if: needs.setup.outputs.should-run - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - name: Output setup - run: | - echo " go-version: [${{ needs.setup.outputs.go-version }}]" - echo " should-run: [${{ needs.setup.outputs.should-run }}]" - echo " file-prefix: [${{ needs.setup.outputs.file-prefix }}]" - echo "db-cache-key-suffix: [${{ needs.setup.outputs.db-cache-key-suffix }}]" - echo " db-cache-path: [${{ needs.setup.outputs.db-cache-path }}]" - echo "go-cache-key-suffix: [${{ needs.setup.outputs.go-cache-key-suffix }}]" - echo " go-cache-path: [${{ needs.setup.outputs.go-cache-path }}]" - - uses: actions/cache@v3 - name: Load db cache - id: db-cache-setup - with: - key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} - path: ${{ needs.setup.outputs.db-cache-path }} - - uses: actions/cache@v3 - name: Load go cache - id: go-cache-setup - with: - key: ${{ runner.os }}-${{ needs.setup.outputs.go-cache-key-suffix }} - path: ${{ needs.setup.outputs.go-cache-path }} - - name: Setup build environment - run: | - brew install cmake sqlite3 - - name: Build cleveldb - if: steps.db-cache-setup.outputs.cache-hit != 'true' - # As of 2022-06-13 (it was a Monday), the macos-latest runners do not have the nproc command. - # But have 3 cores according to https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources - # The linux runners have nproc, so we don't need to set the _JOBS in that job. - run: | - export CLEVELDB_JOBS=3 - export CLEVELDB_DO_BUILD='true' - export CLEVELDB_DO_INSTALL='false' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - - name: Install cleveldb - run: | - export CLEVELDB_DO_BUILD='false' - export CLEVELDB_DO_INSTALL='true' - export CLEVELDB_SUDO='true' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - - uses: actions/setup-go@v3 - with: - go-version: ${{ needs.setup.outputs.go-version }} - - name: Display go version - run: go version - - name: Install runsim - if: steps.go-cache-setup.outputs.cache-hit != 'true' - run: export GO111MODULE="on" && go get github.com/cosmos/tools/cmd/runsim@v1.0.0 - - name: Update provwasm contract - run: make download-smart-contracts - - name: Build provenanced - run: make build - - name: Provenanced version - run: build/provenanced version --long - runsim: # These tests are the ones that use the runsim program (see sims.mk). needs: [setup, build-linux] @@ -207,10 +136,6 @@ jobs: # TODO: Add the "after-import" test once it's fixed: https://github.com/provenance-io/provenance/issues/754 test: ["import-export", "multi-seed-short", "after-import"] db-backend: ["goleveldb", "cleveldb"] - # Not putting "macos-latest" in this because: - # a) They take longer than on ubuntu to begin with. - # b) We're limited to 5 Mac runners at once so they get queued up, taking even longer to finish all of them. - # c) Most of the devs use a mac, so it's less likely that a Mac-specific bug goes unnoticed without these tests. os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: @@ -271,7 +196,7 @@ jobs: go-test-multi-db: # These are tests that use go test to run (see sims.mk), and that we want to test using different database backends. - needs: [setup, build-linux, build-mac] + needs: [setup, build-linux] if: needs.setup.outputs.should-run strategy: fail-fast: false @@ -280,7 +205,7 @@ jobs: # The test-sim-benchmark is handy to have for each db type. test: ["simple", "benchmark"] db-backend: ["goleveldb", "cleveldb"] - os: ["ubuntu-latest", "macos-latest"] + os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -326,7 +251,7 @@ jobs: go-test-single-db: # These are tests that use go test to run (see sims.mk), and that we don't care about testing using different database backends. - needs: [setup, build-linux, build-mac] + needs: [setup, build-linux] if: needs.setup.outputs.should-run strategy: fail-fast: false @@ -334,7 +259,7 @@ jobs: # The test-sim-nondeterminism test hard-codes the db backend to use memdb. # The test-sim-benchmark-invariants test can use different db backends, but to save resources, is down here. test: ["nondeterminism", "benchmark-invariants"] - os: ["ubuntu-latest", "macos-latest"] + os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 From 88d2f573fbeebab6e1395ea5a606dba3c3f90ced Mon Sep 17 00:00:00 2001 From: Daniel Wedul Date: Mon, 10 Oct 2022 14:39:59 -0600 Subject: [PATCH 6/8] [1125]: Remove cleveldb from the runsim matrix since those but leave it in the go-test matrix. The runsims tests take significantly longer than the go tests (40 minutes vs 8 minutes). This way, we at least still have a little coverage with cleveldb. --- .github/workflows/sims.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index bf4d5133c2..054900adc0 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -135,7 +135,7 @@ jobs: # The test-sim-multi-seed-short test should take 30-40 minutes. # TODO: Add the "after-import" test once it's fixed: https://github.com/provenance-io/provenance/issues/754 test: ["import-export", "multi-seed-short", "after-import"] - db-backend: ["goleveldb", "cleveldb"] + db-backend: ["goleveldb"] os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: @@ -153,14 +153,6 @@ jobs: - name: Define test-logs id: test-logs run: echo "::set-output name=test-logs::${{ needs.setup.outputs.file-prefix }}-${{ matrix.test }}-${{ matrix.db-backend }}-${{ matrix.os }}" - - name: Install cleveldb - if: matrix.db-backend == 'cleveldb' - run: | - export CLEVELDB_DO_BUILD='false' - export CLEVELDB_DO_INSTALL='true' - export CLEVELDB_SUDO='true' - export CLEVELDB_DO_CLEANUP='false' - make cleveldb - uses: actions/setup-go@v3 with: go-version: ${{ needs.setup.outputs.go-version }} From 68a585745ec4b53698980e5f2bcda4af18ef6ffd Mon Sep 17 00:00:00 2001 From: Daniel Wedul Date: Mon, 10 Oct 2022 14:42:24 -0600 Subject: [PATCH 7/8] [754]: Remove the todo about re-adding the after-import test. --- .github/workflows/sims.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index 054900adc0..68313b98ff 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -130,10 +130,6 @@ jobs: strategy: fail-fast: false matrix: - # The test-sim-import-export test can take up to two hours. - # The test-sim-after-import test can take even longer. - # The test-sim-multi-seed-short test should take 30-40 minutes. - # TODO: Add the "after-import" test once it's fixed: https://github.com/provenance-io/provenance/issues/754 test: ["import-export", "multi-seed-short", "after-import"] db-backend: ["goleveldb"] os: ["ubuntu-latest"] From 1f81c882894daa22d646b2601f59808bfe663a74 Mon Sep 17 00:00:00 2001 From: Daniel Wedul Date: Mon, 10 Oct 2022 14:55:06 -0600 Subject: [PATCH 8/8] [1125]: No need to reload the db cache in the runsim tests. --- .github/workflows/sims.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index 68313b98ff..bb40967a57 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -136,11 +136,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 - name: Load db cache - with: - key: ${{ runner.os }}-${{ needs.setup.outputs.db-cache-key-suffix }} - path: ${{ needs.setup.outputs.db-cache-path }} - uses: actions/cache@v3 name: Load go cache with: