From 9228ec0193a518c37d63ee742dc31d3a28979ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Fri, 29 May 2020 18:57:54 +0200 Subject: [PATCH 1/3] test: Move fizzy-spectests smoketests files --- circle.yml | 6 +++--- .../spectests/default}/smoketest.0.wasm | Bin .../spectests/default}/smoketest.1.wasm | Bin .../spectests/default}/smoketest.10.wasm | Bin .../spectests/default}/smoketest.11.wasm | Bin .../spectests/default}/smoketest.12.wasm | Bin .../spectests/default}/smoketest.13.wat | 0 .../spectests/default}/smoketest.14.wat | 0 .../spectests/default}/smoketest.2.wasm | Bin .../spectests/default}/smoketest.3.wasm | 0 .../spectests/default}/smoketest.4.wasm | Bin .../spectests/default}/smoketest.5.wasm | Bin .../spectests/default}/smoketest.6.wasm | Bin .../spectests/default}/smoketest.7.wasm | Bin .../spectests/default}/smoketest.8.wasm | Bin .../spectests/default}/smoketest.9.wasm | Bin .../spectests/default}/smoketest.json | 0 .../spectests/default}/smoketest.wast | 0 .../spectests/failures}/failures.0.wasm | Bin .../spectests/failures}/failures.1.wasm | Bin .../spectests/failures}/failures.10.wasm | Bin .../spectests/failures}/failures.11.wasm | Bin .../spectests/failures}/failures.12.wasm | Bin .../spectests/failures}/failures.13.wasm | Bin .../spectests/failures}/failures.14.wasm | Bin .../spectests/failures}/failures.2.wasm | Bin .../spectests/failures}/failures.3.wasm | Bin .../spectests/failures}/failures.4.wasm | Bin .../spectests/failures}/failures.5.wasm | Bin .../spectests/failures}/failures.6.wasm | 0 .../spectests/failures}/failures.7.wasm | Bin .../spectests/failures}/failures.8.wasm | Bin .../spectests/failures}/failures.9.wasm | 0 .../spectests/failures}/failures.json | 0 .../spectests/failures}/failures.wast | 0 35 files changed, 3 insertions(+), 3 deletions(-) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.0.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.1.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.10.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.11.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.12.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.13.wat (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.14.wat (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.2.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.3.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.4.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.5.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.6.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.7.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.8.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.9.wasm (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.json (100%) rename test/{spectests/smoketest => smoketests/spectests/default}/smoketest.wast (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.0.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.1.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.10.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.11.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.12.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.13.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.14.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.2.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.3.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.4.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.5.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.6.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.7.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.8.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.9.wasm (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.json (100%) rename test/{spectests/smoketest-failures => smoketests/spectests/failures}/failures.wast (100%) diff --git a/circle.yml b/circle.yml index f70517f9f..833e33e2a 100644 --- a/circle.yml +++ b/circle.yml @@ -79,19 +79,19 @@ commands: working_directory: ~/build environment: LLVM_PROFILE_FILE: spectests-validation.profraw - command: bin/fizzy-spectests ~/project/test/spectests/smoketest --skip-validation + command: bin/fizzy-spectests ~/project/test/smoketests/spectests/default --skip-validation - run: name: "Run smoketest with fizzy-spectests" working_directory: ~/build environment: LLVM_PROFILE_FILE: spectests-skipvalidation.profraw - command: bin/fizzy-spectests ~/project/test/spectests/smoketest + command: bin/fizzy-spectests ~/project/test/smoketests/spectests/default - run: name: "Run smoketest failures with fizzy-spectests" working_directory: ~/build environment: LLVM_PROFILE_FILE: spectests-failures.profraw - command: bin/fizzy-spectests ~/project/test/spectests/smoketest-failures || true + command: bin/fizzy-spectests ~/project/test/smoketests/spectests/failures || true benchmark: description: "Run benchmarks" diff --git a/test/spectests/smoketest/smoketest.0.wasm b/test/smoketests/spectests/default/smoketest.0.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.0.wasm rename to test/smoketests/spectests/default/smoketest.0.wasm diff --git a/test/spectests/smoketest/smoketest.1.wasm b/test/smoketests/spectests/default/smoketest.1.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.1.wasm rename to test/smoketests/spectests/default/smoketest.1.wasm diff --git a/test/spectests/smoketest/smoketest.10.wasm b/test/smoketests/spectests/default/smoketest.10.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.10.wasm rename to test/smoketests/spectests/default/smoketest.10.wasm diff --git a/test/spectests/smoketest/smoketest.11.wasm b/test/smoketests/spectests/default/smoketest.11.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.11.wasm rename to test/smoketests/spectests/default/smoketest.11.wasm diff --git a/test/spectests/smoketest/smoketest.12.wasm b/test/smoketests/spectests/default/smoketest.12.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.12.wasm rename to test/smoketests/spectests/default/smoketest.12.wasm diff --git a/test/spectests/smoketest/smoketest.13.wat b/test/smoketests/spectests/default/smoketest.13.wat similarity index 100% rename from test/spectests/smoketest/smoketest.13.wat rename to test/smoketests/spectests/default/smoketest.13.wat diff --git a/test/spectests/smoketest/smoketest.14.wat b/test/smoketests/spectests/default/smoketest.14.wat similarity index 100% rename from test/spectests/smoketest/smoketest.14.wat rename to test/smoketests/spectests/default/smoketest.14.wat diff --git a/test/spectests/smoketest/smoketest.2.wasm b/test/smoketests/spectests/default/smoketest.2.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.2.wasm rename to test/smoketests/spectests/default/smoketest.2.wasm diff --git a/test/spectests/smoketest/smoketest.3.wasm b/test/smoketests/spectests/default/smoketest.3.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.3.wasm rename to test/smoketests/spectests/default/smoketest.3.wasm diff --git a/test/spectests/smoketest/smoketest.4.wasm b/test/smoketests/spectests/default/smoketest.4.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.4.wasm rename to test/smoketests/spectests/default/smoketest.4.wasm diff --git a/test/spectests/smoketest/smoketest.5.wasm b/test/smoketests/spectests/default/smoketest.5.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.5.wasm rename to test/smoketests/spectests/default/smoketest.5.wasm diff --git a/test/spectests/smoketest/smoketest.6.wasm b/test/smoketests/spectests/default/smoketest.6.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.6.wasm rename to test/smoketests/spectests/default/smoketest.6.wasm diff --git a/test/spectests/smoketest/smoketest.7.wasm b/test/smoketests/spectests/default/smoketest.7.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.7.wasm rename to test/smoketests/spectests/default/smoketest.7.wasm diff --git a/test/spectests/smoketest/smoketest.8.wasm b/test/smoketests/spectests/default/smoketest.8.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.8.wasm rename to test/smoketests/spectests/default/smoketest.8.wasm diff --git a/test/spectests/smoketest/smoketest.9.wasm b/test/smoketests/spectests/default/smoketest.9.wasm similarity index 100% rename from test/spectests/smoketest/smoketest.9.wasm rename to test/smoketests/spectests/default/smoketest.9.wasm diff --git a/test/spectests/smoketest/smoketest.json b/test/smoketests/spectests/default/smoketest.json similarity index 100% rename from test/spectests/smoketest/smoketest.json rename to test/smoketests/spectests/default/smoketest.json diff --git a/test/spectests/smoketest/smoketest.wast b/test/smoketests/spectests/default/smoketest.wast similarity index 100% rename from test/spectests/smoketest/smoketest.wast rename to test/smoketests/spectests/default/smoketest.wast diff --git a/test/spectests/smoketest-failures/failures.0.wasm b/test/smoketests/spectests/failures/failures.0.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.0.wasm rename to test/smoketests/spectests/failures/failures.0.wasm diff --git a/test/spectests/smoketest-failures/failures.1.wasm b/test/smoketests/spectests/failures/failures.1.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.1.wasm rename to test/smoketests/spectests/failures/failures.1.wasm diff --git a/test/spectests/smoketest-failures/failures.10.wasm b/test/smoketests/spectests/failures/failures.10.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.10.wasm rename to test/smoketests/spectests/failures/failures.10.wasm diff --git a/test/spectests/smoketest-failures/failures.11.wasm b/test/smoketests/spectests/failures/failures.11.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.11.wasm rename to test/smoketests/spectests/failures/failures.11.wasm diff --git a/test/spectests/smoketest-failures/failures.12.wasm b/test/smoketests/spectests/failures/failures.12.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.12.wasm rename to test/smoketests/spectests/failures/failures.12.wasm diff --git a/test/spectests/smoketest-failures/failures.13.wasm b/test/smoketests/spectests/failures/failures.13.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.13.wasm rename to test/smoketests/spectests/failures/failures.13.wasm diff --git a/test/spectests/smoketest-failures/failures.14.wasm b/test/smoketests/spectests/failures/failures.14.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.14.wasm rename to test/smoketests/spectests/failures/failures.14.wasm diff --git a/test/spectests/smoketest-failures/failures.2.wasm b/test/smoketests/spectests/failures/failures.2.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.2.wasm rename to test/smoketests/spectests/failures/failures.2.wasm diff --git a/test/spectests/smoketest-failures/failures.3.wasm b/test/smoketests/spectests/failures/failures.3.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.3.wasm rename to test/smoketests/spectests/failures/failures.3.wasm diff --git a/test/spectests/smoketest-failures/failures.4.wasm b/test/smoketests/spectests/failures/failures.4.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.4.wasm rename to test/smoketests/spectests/failures/failures.4.wasm diff --git a/test/spectests/smoketest-failures/failures.5.wasm b/test/smoketests/spectests/failures/failures.5.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.5.wasm rename to test/smoketests/spectests/failures/failures.5.wasm diff --git a/test/spectests/smoketest-failures/failures.6.wasm b/test/smoketests/spectests/failures/failures.6.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.6.wasm rename to test/smoketests/spectests/failures/failures.6.wasm diff --git a/test/spectests/smoketest-failures/failures.7.wasm b/test/smoketests/spectests/failures/failures.7.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.7.wasm rename to test/smoketests/spectests/failures/failures.7.wasm diff --git a/test/spectests/smoketest-failures/failures.8.wasm b/test/smoketests/spectests/failures/failures.8.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.8.wasm rename to test/smoketests/spectests/failures/failures.8.wasm diff --git a/test/spectests/smoketest-failures/failures.9.wasm b/test/smoketests/spectests/failures/failures.9.wasm similarity index 100% rename from test/spectests/smoketest-failures/failures.9.wasm rename to test/smoketests/spectests/failures/failures.9.wasm diff --git a/test/spectests/smoketest-failures/failures.json b/test/smoketests/spectests/failures/failures.json similarity index 100% rename from test/spectests/smoketest-failures/failures.json rename to test/smoketests/spectests/failures/failures.json diff --git a/test/spectests/smoketest-failures/failures.wast b/test/smoketests/spectests/failures/failures.wast similarity index 100% rename from test/spectests/smoketest-failures/failures.wast rename to test/smoketests/spectests/failures/failures.wast From 9cb5eba6c48fb996a2fe1e4a8bc8484c7eb57cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Sat, 30 May 2020 10:18:27 +0200 Subject: [PATCH 2/3] test: Add fizzy-spectests smoke tests to ctest --- test/smoketests/CMakeLists.txt | 1 + test/smoketests/spectests/CMakeLists.txt | 43 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 test/smoketests/spectests/CMakeLists.txt diff --git a/test/smoketests/CMakeLists.txt b/test/smoketests/CMakeLists.txt index 102a69ea1..c1608cc39 100644 --- a/test/smoketests/CMakeLists.txt +++ b/test/smoketests/CMakeLists.txt @@ -3,3 +3,4 @@ # SPDX-License-Identifier: Apache-2.0 add_subdirectory(benchmarks) +add_subdirectory(spectests) diff --git a/test/smoketests/spectests/CMakeLists.txt b/test/smoketests/spectests/CMakeLists.txt new file mode 100644 index 000000000..cd10a6d42 --- /dev/null +++ b/test/smoketests/spectests/CMakeLists.txt @@ -0,0 +1,43 @@ +# Fizzy: A fast WebAssembly interpreter +# Copyright 2019-2020 The Fizzy Authors. +# SPDX-License-Identifier: Apache-2.0 + +add_test( + NAME fizzy/smoketests/spectests/default + COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/default +) +set_tests_properties( + fizzy/smoketests/spectests/default + PROPERTIES + PASS_REGULAR_EXPRESSION "PASSED 23, FAILED 0, SKIPPED 7" +) + +add_test( + NAME fizzy/smoketests/spectests/skipvalidation + COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/default --skip-validation +) +set_tests_properties( + fizzy/smoketests/spectests/skipvalidation + PROPERTIES + PASS_REGULAR_EXPRESSION "PASSED 22, FAILED 0, SKIPPED 8" +) + +add_test( + NAME fizzy/smoketests/spectests/failures + COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/failures +) +set_tests_properties( + fizzy/smoketests/spectests/failures + PROPERTIES + PASS_REGULAR_EXPRESSION "PASSED 4, FAILED 18, SKIPPED 2" +) + + +# Dump coverage data to distinct files (otherwise file will be overwritten). +set_tests_properties( + fizzy/smoketests/spectests/default + fizzy/smoketests/spectests/skipvalidation + fizzy/smoketests/spectests/failures + PROPERTIES + ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/spectests-%p.profraw +) From 309dfe6002a7b2c532e61672cd786c5acb6b32bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Tue, 2 Jun 2020 19:45:19 +0200 Subject: [PATCH 3/3] ci: Run smoke tests via ctest --- circle.yml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/circle.yml b/circle.yml index 833e33e2a..010ed8daf 100644 --- a/circle.yml +++ b/circle.yml @@ -71,27 +71,18 @@ commands: description: "Test" steps: - run: - name: "Test" + name: "Run unit tests" working_directory: ~/build - command: ctest -R ${TESTS_FILTER:-'.*'} -j4 --schedule-random --output-on-failure + command: ctest -R unittests -j4 --schedule-random --output-on-failure - run: - name: "Run smoketest with fizzy-spectests --skip-validation" + name: "Run smoke tests" working_directory: ~/build - environment: - LLVM_PROFILE_FILE: spectests-validation.profraw - command: bin/fizzy-spectests ~/project/test/smoketests/spectests/default --skip-validation + command: ctest -R smoketests -j4 --schedule-random --output-on-failure - run: - name: "Run smoketest with fizzy-spectests" + name: "Run other tests" working_directory: ~/build - environment: - LLVM_PROFILE_FILE: spectests-skipvalidation.profraw - command: bin/fizzy-spectests ~/project/test/smoketests/spectests/default - - run: - name: "Run smoketest failures with fizzy-spectests" - working_directory: ~/build - environment: - LLVM_PROFILE_FILE: spectests-failures.profraw - command: bin/fizzy-spectests ~/project/test/smoketests/spectests/failures || true + command: ctest -E 'unittests|smoketests' -j4 --schedule-random --output-on-failure + benchmark: description: "Run benchmarks"