[VM] tests: add --expected-test-amount

fix yaml

.github/workflows/vm-pr.yml

@@ -11,14 +11,79 @@ defaults:
     working-directory: packages/vm
 
 jobs:
+  test-vm-api:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+      - uses: actions/checkout@v1
+
+      - name: Dependency cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          key: VM-${{ runner.os }}-12-${{ hashFiles('**/package-lock.json') }}
+          path: '**/node_modules'
+
+      # Installs root dependencies, ignoring Bootstrap All script.
+      # Bootstraps the current package only
+      - run: npm install --ignore-scripts && npm run bootstrap:vm
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: ${{github.workspace}}
+
+      # Builds current package and the ones it depends from.
+      - run: npm run build:vm
+        working-directory: ${{github.workspace}}
+
+      - run: npm run coverage
+
+      - uses: codecov/codecov-action@v1
+        with:
+          file: ${{ env.cwd }}/coverage/lcov.info
+          flags: vm
+
+      - run: npm run test:API
+      - run: npm run test:API:browser
+      - run: npm run lint
+
+  test-vm-state:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        fork: ['Berlin', 'MuirGlacier', 'Petersburg', 'Istanbul', 'Byzantium', 'SpuriousDragon', 'TangerineWhistle', 'Homestead', 'Chainstart']
+      fail-fast: false
+    steps:
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+      - uses: actions/checkout@v1
+
+      - name: Dependency cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          key: VM-${{ runner.os }}-12-${{ hashFiles('**/package-lock.json') }}
+          path: '**/node_modules'
+
+      # Installs root dependencies, ignoring Bootstrap All script.
+      # Bootstraps the current package only
+      - run: npm install --ignore-scripts && npm run bootstrap:vm
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: ${{github.workspace}}
+
+      # Builds current package and the ones it depends from.
+      - run: npm run build:vm
+        working-directory: ${{github.workspace}}
+
+      - run: npm run test:state -- --fork=${{ matrix.fork }}
+
   test-vm-blockchain:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # This is the most fair division among 1 directory vs. everything else
-        args: ['--excludeDir=stTimeConsuming', '--dir=GeneralStateTests/stTimeConsuming']
-        # Run specific fork tests
-        fork: ['Chainstart', 'Homestead', 'SpuriousDragon', 'TangerineWhistle', 'Byzantium', 'Constantinople', 'Petersburg', 'Istanbul']
+        # Args to pass to the tester. Note that some have splitted the slow tests and only running those: these are only on forks where that is applicable (see PR #489 for numbers on these)
+        args: ['--fork=Chainstart --expected-test-amount=4385', '--fork=Homestead --expected-test-amount=6997', '--fork=Petersburg --excludeDir=stTimeConsuming --expected-test-amount=17174', '--fork=Petersburg --dir=GeneralStateTests/stTimeConsuming --expected-test-amount=15561', '--fork=Istanbul --excludeDir=stTimeConsuming --expected-test-amount=19817', '--fork=Istanbul --dir=GeneralStateTests/stTimeConsuming --expected-test-amount=15561', '--fork=Berlin --expected-test-amount=33']
       fail-fast: false
     steps:
       - uses: actions/setup-node@v1
@@ -43,4 +108,57 @@ jobs:
       - run: npm run build:vm
         working-directory: ${{github.workspace}}
 
-      - run: npm run test:blockchain -- ${{ matrix.args }} --fork=${{ matrix.fork }}
+      - run: npm run test:blockchain -- ${{ matrix.args }}
+
+  vm-benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+      - uses: actions/checkout@v2
+
+      - name: Dependency cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          key: VM-${{ runner.os }}-${{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }}
+          path: '**/node_modules'
+
+      # Installs root dependencies, ignoring Bootstrap All script.
+      # Bootstraps the current package only
+      - run: npm install --ignore-scripts && npm run bootstrap:vm
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: ${{github.workspace}}
+
+      # Builds current package and the ones it depends from.
+      - run: npm run build:vm
+        working-directory: ${{github.workspace}}
+
+      - run: npm run build:benchmarks
+        working-directory: ${{ env.cwd }}
+
+      - run: npm run benchmarks -- 10 | tee output.txt
+        working-directory: ${{ env.cwd }}
+
+      # Run git stash in case github-action-benchmark has trouble switching to gh-pages branch due to differing package-locks
+      - run: git stash
+
+      - name: Compare benchmarks
+        uses: rhysd/github-action-benchmark@v1
+        with:
+          tool: 'benchmarkjs'
+          # Where the output from the benchmark tool is stored
+          output-file-path: ${{ env.cwd }}/output.txt
+          # Enable alert commit comment
+          comment-on-alert: true
+          # Always leave a commit comment comparing the current benchmark with previous
+          comment-always: true
+          # GitHub API token to make a commit comment
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Push and deploy to GitHub pages branch automatically (if on master)
+          auto-push: 'false'
+
+      # Re-apply git stash to prepare for saving back to cache.
+      # Avoids exit code 1 by checking if there are changes to be stashed first
+      - run: STASH_LIST=`git stash list` && [ ! -z $STASH_LIST ] && git stash apply || echo "No files to stash-apply. Skipping…"

packages/vm/tests/tester.js

@@ -26,6 +26,7 @@ const Common = require('@ethereumjs/common').default
  * --gas: only run this state test if the transaction has this gasLimit
  * --value: only run this state test if the transaction has this call value
  * --debug: enable BlockchainTests debugger (compares post state against the expected post state)
+ * --expected-test-amount: (optional) if present, check after tests are ran if at least this amount of tests have passed (inclusive)
  */
 
 async function runTests() {
@@ -171,6 +172,10 @@ async function runTests() {
         }
       }
 
+      if (argv['expected-test-amount']) {
+        t.ok(t.assertCount >= argv['expected-test-amount'], "expected " + argv['expected-test-amount'] + " checks but only got " + t.assertCount)
+      }
+
       t.end()
 
     })