Vmtests valid

JSONSchema/bc-filler-schema.json

@@ -328,4 +328,4 @@
             "type": "object"
         }
     }
-}
+}

JSONSchema/bc-schema.json

@@ -314,4 +314,4 @@
             "type": "object"
         }
     }
-}
+}

JSONSchema/definitions.json

@@ -317,4 +317,4 @@
             "type": "array"
         }
     }
-}
+}

JSONSchema/st-filler-schema.json

@@ -257,4 +257,4 @@
         }
     },
     "type": "object"
-}
+}

JSONSchema/st-schema.json

@@ -258,4 +258,4 @@
         }
     },
     "type": "object"
-}
+}

JSONSchema/vm-filler-schema.json

@@ -159,4 +159,4 @@
         }
     },
     "type": "object"
-}
+}

JSONSchema/vm-schema.json

@@ -138,4 +138,4 @@
         }
     },
     "type": "object"
-}
+}

Makefile

@@ -1,8 +1,9 @@
 ETHEREUM_TEST_PATH=$(CURDIR)
 export ETHEREUM_TEST_PATH
 
+# TODO: stop stripping out stEWASMTests from gs_tests
 tx_tests:=$(wildcard TransactionTests/*)
-gs_tests:=$(wildcard GeneralStateTests/*)
+gs_tests:=$(filter-out %stEWASMTests, $(wildcard GeneralStateTests/*))
 bc_tests:=$(wildcard BlockchainTests/*)
 vm_tests:=$(wildcard VMTests/*)
 all_tests:=$(gs_tests) $(bc_tests) $(vm_tests)
@@ -17,14 +18,32 @@ all_schemas:=$(wildcard JSONSchema/*.json)
 
 # Testset sanitation
 
-sani: $(all_schemas:=.format) $(vm_fillers:=.format) $(vm_tests:=.format)
+sani: sani-schema sani-vm sani-gs sani-tx sani-bc
+
+sani-schema: $(all_schemas:=.format)
+
+sani-vm: $(vm_tests:=.format) $(vm_fillers:=.format) \
+         $(vm_tests:=.valid)  $(vm_fillers:=.valid)  \
+         $(vm_tests:=.filled)
+
+# TODO: enable $(gs_fillers:=.valid) $(gs_tests:=.format) $(gs_fillers:=.format)
+sani-gs: $(gs_tests:=.valid) \
+         $(gs_tests:=.filled)
+
+# TODO: enable $(tx_tests:=.format) $(tx_fillers:=.format) $(tx_tests:=.valid) $(tx_fillers:=.valid)
+sani-tx: $(tx_tests:=.filled)
+
+# TODO: enable $(bc_tests:=.format) $(bc_fillers:=.format) $(bc_tests:=.filled)
+sani-bc: $(bc_tests:=.valid)  $(bc_fillers:=.valid)
 
 %.format:
 	python3 test.py format ./$*
 	git diff --quiet --exit-code &>/dev/null
 
-%.sani:
-	python3 test.py validate    ./$*
+%.valid:
+	python3 test.py validate ./$*
+
+%.filled:
 	python3 test.py checkFilled ./$*
 
 # Test running command
@@ -41,5 +60,5 @@ fill-tests:=$(all-tests:=.fill)
 fill: $(fill-tests)
 
 %.fill:
-	testeth -t $* -- --filltests --verbosity 2
+	testeth -t $* -- --filltests --verbosity 2 --all
 	python3 test.py format ./$*

README.md

@@ -37,6 +37,51 @@ All files should be of the form:
 
 Arrays are allowed, but don't use them for sets of properties - only use them for data that is clearly a continuous contiguous sequence of values.
 
+Test Set Sanitation
+-------------------
+
+### Checkers
+
+Several basic checks against the test-set are performed to enusre that they have been filled and are formatted correctly.
+Currently, there are three types of checks that we can perform:
+
+-   `make TEST_PREFIX.format`: check that the JSON is formatted correctly.
+-   `make TEST_PREFIX.valid`: check that the JSON files are valid against the JSON schemas in `./JSONSchema`.
+-   `make TEST_PREFIX.filled`: check that the JSON tests are filled with the correct source hashes against the fillers.
+
+The constant `TEST_PREFIX` is a path prefix to the test-set you're interested in performing the checks on.
+For instance:
+
+-   `make ./src/VMTestsFiller/vmArithmeticTest.format` will check that all JSON files in `./src/VMTestsFiller/vmArithmeticTest` are formatted correctly.
+-   `make ./src.valid` will check that all the JSON files in `./src` are valid against the JSON schemas in `./JSONSchema`.
+-   `make ./BlockchainTests.filled` will check that the source hashes in the JSON tests in `./BlockchainTests` are the same as the hashes of the fillers in `./src/BlockchainTestsFiller`.
+
+These checks are all performed by the file `./test.py`, which can be invoked on individual files as well.
+Run `./test.py` with no arguments for help.
+
+### Sanitizers
+
+The above checkers are packaged together into sanitizers for each test-suite, marking which testsuites are passing which testers.
+See the `TODO`s in the `Makefile` to see which checkers are enabled for which test-suites.
+
+-   `make sani`: will run all passing sanitizers on all passing testsuites.
+-   `make sani-TESTNAME`: will run just the passing sanitizers for the given testsuite.
+    `TESTNAME` can be one of:
+
+    -   `vm`: VMTests and VMTestsFiller
+    -   `gs`: GeneralStateTests and GeneralStateTestsFiller
+    -   `bc`: BlockchainTests and BlockchainTestsFiller
+    -   `tx`: TransactionTests and TransactionTestsFiller
+
+### Runners/Fillers
+
+The tests can also be run/filled with the same `Makefile`.
+
+-   `make run`: will use `testeth` to all the test-suites.
+-   `make fill`: will use `testeth` to fill all the test-suites.
+-   `make TEST_PREFIX.run`: runs a single testsuite.
+-   `make TEST_PREFIX.fill`: fills and formats a single testsuite.
+
 FAQ
 ---
 

VMTests/vmArithmeticTest/add0.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/add0Filler.json",
-            "sourceHash": "e8e12ee9a80d43ce4a13957f3d852b93ddc9fbb4722fbb922a0dee6c6de18de8"
+            "sourceHash": "8ffcccd596ff216a8304f822e302bd7b2abe3f7f7616928a2a0385d2cfdf01ac"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/add1.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/add1Filler.json",
-            "sourceHash": "a23c9e4a53bd51d64aaf5a2ad8c2e7c31c457d68370be0b7253d2124a823540d"
+            "sourceHash": "36b138942544cd189bff05ff1694917ed4cef48760592536b2829f3e90d5850a"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/add2.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/add2Filler.json",
-            "sourceHash": "50066a5c2c6857bc9dffa863694a7c858eb2054e9becf45bb70dcf24341c18ba"
+            "sourceHash": "c17fbe9e92877d5a60743ca4c172bba1060e20b08f9ee00ca0ccb6b72bac897f"
         },
         "callcreates": [],
         "env": {
@@ -45,4 +45,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/add3.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/add3Filler.json",
-            "sourceHash": "cbfef553d8e13dd1f27e5fc3d2e4e5510cc9e545a21658a05481bcf667e6477f"
+            "sourceHash": "d64685dfa395fd7dc69e5883eaedaeece8217daff0ad658a083dc2f2fc870ef1"
         },
         "callcreates": [],
         "env": {
@@ -45,4 +45,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/add4.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/add4Filler.json",
-            "sourceHash": "b5f7f278016c8ae538453bc3e8a364f64bf3de646c0e1f26f4db3d8c2f40cd99"
+            "sourceHash": "3b04e90121702949a56e656f4f895f5de3aed278fd3530eec4c52c378f41dcfa"
         },
         "callcreates": [],
         "env": {
@@ -45,4 +45,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod0.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod0Filler.json",
-            "sourceHash": "23a828d76496714224cf09ae32468851cccbc63de0b8205787f17f3f529d9e86"
+            "sourceHash": "aba7a646dee5e90acf0c2257a986ff99d044f057de21667f2acec868f7d4b8eb"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod1.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod1Filler.json",
-            "sourceHash": "21940beb8d531f2e841328da98111184c51a9b379083efecc69a7e242125f4b7"
+            "sourceHash": "edf8071d0533dd82ff2fb34b4caab47db62aa7d918004d82630cd2b37ddcbc90"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod1_overflow2.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod1_overflow2Filler.json",
-            "sourceHash": "7f4c24f9191abfd44da5fde6159b78eef7fd8152e4fc6a36e5b32a2f27f89c8c"
+            "sourceHash": "9029d29f6ad3ad30d1cae39fe98723bb400f9fab4562bd7898a0168a0f353a3f"
         },
         "callcreates": [],
         "env": {
@@ -45,4 +45,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod1_overflow3.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod1_overflow3Filler.json",
-            "sourceHash": "910ff6466be088b829f03cd6243b9bd256262a0b2affbcdeae19c20c2f98d415"
+            "sourceHash": "f51942005cbb182f759380c7c7fa43d9666d8ea095cc9ce0538e04c2ffc2dd03"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod1_overflow4.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod1_overflow4Filler.json",
-            "sourceHash": "26ef2f1c62d0543640a61c4550ad1c4451679a0a71c6c95a707077789d6bff69"
+            "sourceHash": "59184840928cfdb750aab06b0a42c30d4dc99a713f315239b1b6596ec9ef92c2"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod1_overflowDiff.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod1_overflowDiffFiller.json",
-            "sourceHash": "2c556344c4be4c97acd8cf3582513de03ef2410d879a5be5957956748b24cd31"
+            "sourceHash": "4836017c5969c3cc9e6fee8a3da1e2d81454d2880d10fadd146cdb8ca8b57273"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod2.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod2Filler.json",
-            "sourceHash": "a18442325ceba36a1b1ce67e58339c9efe0a0ed0d101e047132db15960f7d833"
+            "sourceHash": "22a6b0713565860ede4858c04c85d823cc24829962730303aeb7bacf48e6e2af"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod2_0.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod2_0Filler.json",
-            "sourceHash": "b9e563c96bb3d3e715cd941a5d43ede67095b47c395934f0cdc3dd5fcc26077a"
+            "sourceHash": "dc557aacebaa46b88f072e79ac09df8a576c1551f7ce3381bc22aa33a5fdfa7b"
         },
         "callcreates": [],
         "env": {
@@ -45,4 +45,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod2_1.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod2_1Filler.json",
-            "sourceHash": "4c8b184bddd8148361dfbf0f8bf3f4390e1636cebfe8c047235bbd6301fa5845"
+            "sourceHash": "f5227cd573f551a66a89ca6ef0075d999fa0dfcd1b6cc162deba46db3fc957ae"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}

VMTests/vmArithmeticTest/addmod3.json

@@ -5,7 +5,7 @@
             "filledwith": "testeth 1.4.0.dev0-56+commit.71414ae3",
             "lllcversion": "Version: 0.4.25-develop.2018.5.30+commit.0a1a8bfb.Linux.g++",
             "source": "src/VMTestsFiller/vmArithmeticTest/addmod3Filler.json",
-            "sourceHash": "fcafebf571b5f29d85245017c5b9a0c13d04cbfc373aef6e72ddec05eca4300f"
+            "sourceHash": "98f42c19c4c02edb27ddbe15d425fa031f3b79998b3fcfce32ecbb070dca20f7"
         },
         "callcreates": [],
         "env": {
@@ -47,4 +47,4 @@
             }
         }
     }
-}
+}