Skip to content

Commit

Permalink
Purge winbug from regression tests
Browse files Browse the repository at this point in the history
We no longer use or need this label for all tests have been made to pass
on Windows (when we expect them to).
  • Loading branch information
tautschnig committed Jul 10, 2024
1 parent b3f1e71 commit 06a3e16
Show file tree
Hide file tree
Showing 4 changed files with 7 additions and 32 deletions.
8 changes: 0 additions & 8 deletions regression/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,3 @@ This folder contains the CProver regression test-suite.
These tests are known to not work with Z3 (the version running on our CI).
- `thorough-z3-smt-backend`:
These tests are too slow to be run in CI with Z3.

### Platform

- `winbug`:
These tests are currently known to be failing on Windows,
but passing on other platforms. https://github.com/diffblue/cbmc/pull/5572
will address one part thereof; the remaining ones are C++ tests that fail on
both Windows and MacOS for our lack of C++-11 support.
14 changes: 4 additions & 10 deletions regression/book-examples/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,32 +1,26 @@
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
set(gcc_only -X gcc-only)
set(gcc_only_string "-X;gcc-only;")
set(exclude_win_broken_tests -X winbug)
# We add the string at the end of the exclusion list, so it must not
# have the `;` at the end or it bugs out.
set(exclude_win_broken_tests_string "-X;winbug")
set(gcc_only_string "-X;gcc-only")
else()
set(gcc_only "")
set(gcc_only_string "")
set(exclude_win_broken_tests "")
set(exclude_win_broken_tests_string "")
endif()

add_test_pl_tests(
"$<TARGET_FILE:cbmc> --validate-goto-model --validate-ssa-equation" -X smt-backend ${gcc_only} ${exclude_win_broken_tests}
"$<TARGET_FILE:cbmc> --validate-goto-model --validate-ssa-equation" -X smt-backend ${gcc_only}
)

add_test_pl_profile(
"book-examples-paths-lifo"
"$<TARGET_FILE:cbmc> --paths lifo"
"-C;-X;thorough-paths;-X;smt-backend;-X;paths-lifo-expected-failure;${gcc_only_string}-s;paths-lifo;${exclude_win_broken_tests_string}"
"-C;-X;thorough-paths;-X;smt-backend;-X;paths-lifo-expected-failure;${gcc_only_string}-s;paths-lifo"
"CORE"
)

add_test_pl_profile(
"book-examples-cprover-smt2"
"$<TARGET_FILE:cbmc> --cprover-smt2"
"-C;-X;broken-smt-backend;-X;thorough-smt-backend;-X;broken-cprover-smt-backend;-X;thorough-cprover-smt-backend;${gcc_only_string}-s;cprover-smt2;${exclude_win_broken_tests_string}"
"-C;-X;broken-smt-backend;-X;thorough-smt-backend;-X;broken-cprover-smt-backend;-X;thorough-cprover-smt-backend;${gcc_only_string}-s;cprover-smt2"
"CORE"
)

Expand Down
12 changes: 3 additions & 9 deletions regression/cbmc/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,32 +1,26 @@
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
set(gcc_only -X gcc-only)
set(gcc_only_string "-X;gcc-only;")
set(exclude_win_broken_tests -X winbug)
# We add the string at the end of the exclusion list, so it must not
# have the `;` at the end or it bugs out.
set(exclude_win_broken_tests_string "-X;winbug")
else()
set(gcc_only "")
set(gcc_only_string "")
set(exclude_win_broken_tests "")
set(exclude_win_broken_tests_string "")
endif()

add_test_pl_tests(
"$<TARGET_FILE:cbmc> --validate-goto-model --validate-ssa-equation" -X smt-backend ${gcc_only} ${exclude_win_broken_tests}
"$<TARGET_FILE:cbmc> --validate-goto-model --validate-ssa-equation" -X smt-backend ${gcc_only}
)

add_test_pl_profile(
"cbmc-paths-lifo"
"$<TARGET_FILE:cbmc> --paths lifo"
"-C;-X;thorough-paths;-X;smt-backend;-X;paths-lifo-expected-failure;${gcc_only_string}-s;paths-lifo;${exclude_win_broken_tests_string}"
"-C;-X;thorough-paths;-X;smt-backend;-X;paths-lifo-expected-failure;${gcc_only_string}-s;paths-lifo"
"CORE"
)

add_test_pl_profile(
"cbmc-cprover-smt2"
"$<TARGET_FILE:cbmc> --cprover-smt2"
"-C;-X;broken-smt-backend;-X;thorough-smt-backend;-X;broken-cprover-smt-backend;-X;thorough-cprover-smt-backend;${gcc_only_string}-s;cprover-smt2;${exclude_win_broken_tests_string}"
"-C;-X;broken-smt-backend;-X;thorough-smt-backend;-X;broken-cprover-smt-backend;-X;thorough-cprover-smt-backend;${gcc_only_string}-s;cprover-smt2"
"CORE"
)

Expand Down
5 changes: 0 additions & 5 deletions regression/cbmc/export-symex-ready-goto/test-bad-usage.desc
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,3 @@ test.c
--
Ensure that --export-symex-ready-goto exported.symex-ready.goto terminates
with error when incorrectly used.

The reason why we use `winbug` is that it fails on certain windows system
probably due to different interpretation of "". Not a bug, but we're using
that label to remain consistent with other parts of the codebase, and not
to unnecessarily introduce new ones for simple use cases.

0 comments on commit 06a3e16

Please sign in to comment.