Merge 'tests: support boost::test decorators and tolerate failures in…

… test_spawn_input' from Kefu Chai

* testing/entry_point: do not support Boost <= 1.58
* testing: set test name and file in constructor
* testing: do not use global vector for collecting tests
* testing: add boost unit test decorator support
* tests: do not fail spawn_test if up to 3 tests fail

See also scylladb#1320

Signed-off-by: Kefu Chai <kefu.chai@scylladb.com>

Closes scylladb#1498

* github.com:scylladb/seastar:
  tests: do not fail spawn_test if less or equal to 3 tests fail
  testing: add boost unit test decorator support
  testing: do not use global vector for collecting tests
  testing: set line number for each test
  testing: set test name and file in constructor
  testing/entry_point: do not support Boost <= 1.58

include/seastar/testing/seastar_test.hh

@@ -30,23 +30,33 @@
 #include <seastar/util/std-compat.hh>
 #include <seastar/testing/entry_point.hh>
 
+#define SEASTAR_TEST_INVOKE(func, ...) func(__VA_ARGS__)
+
+namespace boost::unit_test::decorator {
+
+class collector_t;
+
+}
+
 namespace seastar {
 
 namespace testing {
 
 class seastar_test {
+    const std::string _test_file;
 public:
-    seastar_test();
+    seastar_test(const char* test_name, const char* test_file, int test_line);
+    seastar_test(const char* test_name, const char* test_file, int test_line,
+                 boost::unit_test::decorator::collector_t& decorators);
     virtual ~seastar_test() {}
-    virtual const char* get_test_file() const = 0;
-    virtual const char* get_name() const = 0;
-    virtual int get_expected_failures() const { return 0; }
+    const std::string& get_test_file() const {
+        return _test_file;
+    }
+    static const std::string& get_name();
     virtual future<> run_test_case() const = 0;
     void run();
 };
 
-const std::vector<seastar_test*>& known_tests();
-
 // BOOST_REQUIRE_EXCEPTION predicates
 namespace exception_predicate {
 

include/seastar/testing/test_case.hh

@@ -22,15 +22,35 @@
 
 #pragma once
 
+#include <boost/preprocessor/control/iif.hpp>
+#include <boost/preprocessor/comparison/equal.hpp>
+#include <boost/preprocessor/variadic/size.hpp>
+
 #include <seastar/core/future.hh>
 
 #include <seastar/testing/seastar_test.hh>
 
-#define SEASTAR_TEST_CASE(name) \
-    struct name : public seastar::testing::seastar_test { \
-        const char* get_test_file() const override { return __FILE__; } \
-        const char* get_name() const override { return #name; } \
-        seastar::future<> run_test_case() const override; \
-    }; \
-    static const name name ## _instance; /* NOLINT(cert-err58-cpp) */ \
+#define SEASTAR_TEST_CASE_WITH_DECO(name, decorators)               \
+    struct name : public seastar::testing::seastar_test {           \
+        using seastar::testing::seastar_test::seastar_test;         \
+        seastar::future<> run_test_case() const override;           \
+    };                                                              \
+    static const name name ## _instance(                            \
+        #name,                                                      \
+        __FILE__,                                                   \
+        __LINE__,                                                   \
+        decorators); /* NOLINT(cert-err58-cpp) */                   \
     seastar::future<> name::run_test_case() const
+
+#define SEASTAR_TEST_CASE_WITHOUT_DECO(name)                        \
+    SEASTAR_TEST_CASE_WITH_DECO(                                    \
+        name,                                                       \
+        boost::unit_test::decorator::collector_t::instance())
+
+#define SEASTAR_TEST_CASE(...)                                      \
+    SEASTAR_TEST_INVOKE(                                            \
+        BOOST_PP_IIF(                                               \
+            BOOST_PP_EQUAL(BOOST_PP_VARIADIC_SIZE(__VA_ARGS__), 1), \
+            SEASTAR_TEST_CASE_WITHOUT_DECO,                         \
+            SEASTAR_TEST_CASE_WITH_DECO),                           \
+        __VA_ARGS__)

include/seastar/testing/thread_test_case.hh

@@ -25,22 +25,38 @@
 #include <seastar/core/future.hh>
 #include <seastar/core/thread.hh>
 
+#include <boost/preprocessor/control/iif.hpp>
+#include <boost/preprocessor/comparison/equal.hpp>
+#include <boost/preprocessor/variadic/size.hpp>
+
 #include <seastar/testing/seastar_test.hh>
 
-#define SEASTAR_THREAD_TEST_CASE_EXPECTED_FAILURES(name, failures) \
-    struct name : public seastar::testing::seastar_test { \
-        const char* get_test_file() const override { return __FILE__; } \
-        const char* get_name() const override { return #name; } \
-        int get_expected_failures() const override { return failures; } \
-        seastar::future<> run_test_case() const override { \
-            return seastar::async([this] { \
-                do_run_test_case(); \
-            }); \
-        } \
-        void do_run_test_case() const; \
-    }; \
-    static const name name ## _instance; /* NOLINT(cert-err58-cpp) */ \
+#define SEASTAR_THREAD_TEST_CASE_WITH_DECO(name, decorators) \
+    struct name : public seastar::testing::seastar_test {    \
+        using seastar::testing::seastar_test::seastar_test;  \
+        seastar::future<> run_test_case() const override {   \
+            return seastar::async([this] {                   \
+                do_run_test_case();                          \
+            });                                              \
+        }                                                    \
+        void do_run_test_case() const;                       \
+    };                                                       \
+    static const name name ## _instance(                     \
+        #name,                                               \
+        __FILE__,                                            \
+        __LINE__,                                            \
+        decorators); /* NOLINT(cert-err58-cpp) */            \
     void name::do_run_test_case() const
 
-#define SEASTAR_THREAD_TEST_CASE(name) \
-    SEASTAR_THREAD_TEST_CASE_EXPECTED_FAILURES(name, 0)
+#define SEASTAR_THREAD_TEST_CASE_WITHOUT_DECO(name)          \
+    SEASTAR_THREAD_TEST_CASE_WITH_DECO(                      \
+        name,                                                \
+        boost::unit_test::decorator::collector_t::instance())
+
+#define SEASTAR_THREAD_TEST_CASE(...)                               \
+    SEASTAR_TEST_INVOKE(                                            \
+        BOOST_PP_IIF(                                               \
+            BOOST_PP_EQUAL(BOOST_PP_VARIADIC_SIZE(__VA_ARGS__), 1), \
+            SEASTAR_THREAD_TEST_CASE_WITHOUT_DECO,                  \
+            SEASTAR_THREAD_TEST_CASE_WITH_DECO),                    \
+        __VA_ARGS__)

src/testing/entry_point.cc

@@ -29,21 +29,7 @@ namespace seastar {
 namespace testing {
 
 static bool init_unit_test_suite() {
-    const auto& tests = known_tests();
     auto&& ts = boost::unit_test::framework::master_test_suite();
-    ts.p_name.set(tests.size() ? (tests)[0]->get_test_file() : "seastar-tests");
-
-    for (seastar_test* test : tests) {
-#if BOOST_VERSION > 105800
-        ts.add(boost::unit_test::make_test_case([test] { test->run(); }, test->get_name(),
-                                                test->get_test_file(), 0),
-                                                test->get_expected_failures(), 0);
-#else
-        ts.add(boost::unit_test::make_test_case([test] { test->run(); }, test->get_name()),
-                                                test->get_expected_failures(), 0);
-#endif
-    }
-
     return global_test_runner().start(ts.argc, ts.argv);
 }
 

src/testing/seastar_test.cc

@@ -52,25 +52,21 @@ void seastar_test::run() {
     });
 }
 
-// We store a pointer because tests are registered from dynamic initializers,
-// so we must ensure that 'tests' is initialized before any dynamic initializer.
-// I use a primitive type, which is guaranteed to be initialized before any
-// dynamic initializer and lazily allocate the factor.
-
-static std::vector<seastar_test*>* tests = nullptr;
-
-const std::vector<seastar_test*>& known_tests() {
-    if (!tests) {
-        throw std::runtime_error("No tests registered");
-    }
-    return *tests;
+seastar_test::seastar_test(const char* test_name, const char* test_file, int test_line)
+    : seastar_test(test_name, test_file, test_line, boost::unit_test::decorator::collector_t::instance()) {}
+
+seastar_test::seastar_test(const char* test_name, const char* test_file, int test_line,
+                           boost::unit_test::decorator::collector_t& decorators)
+    : _test_file{test_file} {
+    auto test = boost::unit_test::make_test_case([this] { run(); }, test_name, test_file, test_line);
+    decorators.store_in(*test);
+    decorators.reset();
+    boost::unit_test::framework::current_auto_test_suite().add(test);
 }
 
-seastar_test::seastar_test() {
-    if (!tests) {
-        tests = new std::vector<seastar_test*>();
-    }
-    tests->push_back(this);
+const std::string& seastar_test::get_name() {
+    const auto& current_test = boost::unit_test::framework::current_test_unit();
+    return current_test.p_name.get();
 }
 
 namespace exception_predicate {

tests/unit/spawn_test.cc

@@ -92,7 +92,7 @@ SEASTAR_TEST_CASE(test_spawn_echo) {
     });
 }
 
-SEASTAR_TEST_CASE(test_spawn_input) {
+SEASTAR_TEST_CASE(test_spawn_input, *boost::unit_test::expected_failures(3)) {
     static const sstring text = "hello world\n";
     return spawn_process("/bin/cat").then([] (auto process) {
         auto stdin = process.stdin();
@@ -101,13 +101,12 @@ SEASTAR_TEST_CASE(test_spawn_input) {
             return stdin.write(text).then([&stdin] {
                 return stdin.flush();
             }).handle_exception_type([] (std::system_error& e) {
-                testlog.error("failed to write to stdin: {}", e);
-                return make_exception_future<>(std::move(e));
+                BOOST_TEST_ERROR(fmt::format("failed to write to stdin: {}", e));
             }).then([&stdout] {
                 return stdout.read_exactly(text.size());
             }).handle_exception_type([] (std::system_error& e) {
-                testlog.error("failed to read from stdout: {}", e);
-                return make_exception_future<temporary_buffer<char>>(std::move(e));
+                BOOST_TEST_ERROR(fmt::format("failed to read from stdout: {}", e));
+                return make_ready_future<temporary_buffer<char>>();
             }).then([] (temporary_buffer<char> echo) {
                 BOOST_CHECK_EQUAL(sstring(echo.get(), echo.size()), text);
             }).finally([&p] {

tests/unit/thread_test.cc

@@ -147,7 +147,7 @@ SEASTAR_TEST_CASE(test_asan_false_positive) {
 }
 #endif
 
-SEASTAR_THREAD_TEST_CASE_EXPECTED_FAILURES(abc, 2) {
+SEASTAR_THREAD_TEST_CASE(abc, *boost::unit_test::expected_failures(2)) {
     BOOST_TEST(false);
     BOOST_TEST(false);
 }
@@ -172,13 +172,13 @@ SEASTAR_TEST_CASE(test_thread_custom_stack_size) {
 // detect_stack_use_after_return=1 from the environment.
 #if defined(SEASTAR_THREAD_STACK_GUARDS) && defined(__x86_64__) && !defined(SEASTAR_ASAN_ENABLED)
 struct test_thread_custom_stack_size_failure : public seastar::testing::seastar_test {
-    const char* get_test_file() const override { return __FILE__; }
-    const char* get_name() const override { return "test_thread_custom_stack_size_failure"; }
-    int get_expected_failures() const override { return 0; } \
+    using seastar::testing::seastar_test::seastar_test;
     seastar::future<> run_test_case() const override;
 };
 
-static test_thread_custom_stack_size_failure test_thread_custom_stack_size_failure_instance;
+static test_thread_custom_stack_size_failure test_thread_custom_stack_size_failure_instance(
+    "test_thread_custom_stack_size_failure",
+    __FILE__, __LINE__);
 static thread_local volatile bool stack_guard_bypassed = false;
 
 static int get_mprotect_flags(void* ctx) {