Merge pull request #364 from wasmx/smoketests

Add smoketests to ctest

circle.yml

@@ -71,27 +71,18 @@ commands:
     description: "Test"
     steps:
     - run:
-        name: "Test"
+        name: "Run unit tests"
         working_directory: ~/build
-        command: ctest -R ${TESTS_FILTER:-'.*'} -j4 --schedule-random --output-on-failure
+        command: ctest -R unittests -j4 --schedule-random --output-on-failure
     - run:
-        name: "Run smoketest with fizzy-spectests --skip-validation"
+        name: "Run smoke tests"
         working_directory: ~/build
-        environment:
-          LLVM_PROFILE_FILE: spectests-validation.profraw
-        command: bin/fizzy-spectests ~/project/test/spectests/smoketest --skip-validation
+        command: ctest -R smoketests -j4 --schedule-random --output-on-failure
     - run:
-        name: "Run smoketest with fizzy-spectests"
+        name: "Run other tests"
         working_directory: ~/build
-        environment:
-          LLVM_PROFILE_FILE: spectests-skipvalidation.profraw
-        command: bin/fizzy-spectests ~/project/test/spectests/smoketest
-    - run:
-        name: "Run smoketest failures with fizzy-spectests"
-        working_directory: ~/build
-        environment:
-          LLVM_PROFILE_FILE: spectests-failures.profraw
-        command: bin/fizzy-spectests ~/project/test/spectests/smoketest-failures || true
+        command: ctest -E 'unittests|smoketests' -j4 --schedule-random --output-on-failure
+
 
   benchmark:
     description: "Run benchmarks"

test/smoketests/CMakeLists.txt

@@ -3,3 +3,4 @@
 # SPDX-License-Identifier: Apache-2.0
 
 add_subdirectory(benchmarks)
+add_subdirectory(spectests)

test/smoketests/spectests/CMakeLists.txt

@@ -0,0 +1,43 @@
+# Fizzy: A fast WebAssembly interpreter
+# Copyright 2019-2020 The Fizzy Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+add_test(
+    NAME fizzy/smoketests/spectests/default
+    COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/default
+)
+set_tests_properties(
+    fizzy/smoketests/spectests/default
+    PROPERTIES
+    PASS_REGULAR_EXPRESSION "PASSED 23, FAILED 0, SKIPPED 7"
+)
+
+add_test(
+    NAME fizzy/smoketests/spectests/skipvalidation
+    COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/default --skip-validation
+)
+set_tests_properties(
+    fizzy/smoketests/spectests/skipvalidation
+    PROPERTIES
+    PASS_REGULAR_EXPRESSION "PASSED 22, FAILED 0, SKIPPED 8"
+)
+
+add_test(
+    NAME fizzy/smoketests/spectests/failures
+    COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/failures
+)
+set_tests_properties(
+    fizzy/smoketests/spectests/failures
+    PROPERTIES
+    PASS_REGULAR_EXPRESSION "PASSED 4, FAILED 18, SKIPPED 2"
+)
+
+
+# Dump coverage data to distinct files (otherwise file will be overwritten).
+set_tests_properties(
+    fizzy/smoketests/spectests/default
+    fizzy/smoketests/spectests/skipvalidation
+    fizzy/smoketests/spectests/failures
+    PROPERTIES
+    ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/spectests-%p.profraw
+)