Add support for test cases to declare tags (labels)

This change allows for test cases to declare any number of associated
"tags". The tags are specified after the test case name string as a
list of bracketed tags, mimicking Catch2's syntax.

The LoadTests.cmake script has been modified to apply test case's declared
tags as CTest labels. This also gives us the ability to apply test
fixture requirements granularly on only tests that declare their
requirement (via a tag).

Author: vector-of-bool

build/cmake/LoadTests.cmake

@@ -3,57 +3,68 @@
 # allowing CTest to control the execution, parallelization, and collection of
 # test results.
 
-if (NOT EXISTS "${TEST_LIBMONGOC_EXE}")
+if(NOT EXISTS "${TEST_LIBMONGOC_EXE}")
     # This will fail if 'test-libmongoc' is not compiled yet.
-    message (WARNING "The test executable ${TEST_LIBMONGOC_EXE} is not present. "
+    message(WARNING "The test executable ${TEST_LIBMONGOC_EXE} is not present. "
                      "Its tests will not be registered")
-    add_test (mongoc/not-found NOT_FOUND)
-    return ()
-endif ()
+    add_test(mongoc/not-found NOT_FOUND)
+    return()
+endif()
 
-# Get the list of tests
-execute_process (
-    COMMAND "${TEST_LIBMONGOC_EXE}" --list-tests --no-fork
-    OUTPUT_VARIABLE tests_out
+# Get the list of tests. This command emits CMake code that defines variables for
+# all test cases defined in the suite
+execute_process(
+    COMMAND "${TEST_LIBMONGOC_EXE}" --tests-cmake --no-fork
+    OUTPUT_VARIABLE tests_cmake
     WORKING_DIRECTORY "${SRC_ROOT}"
     RESULT_VARIABLE retc
     )
-if (retc)
+if(retc)
     # Failed to list the tests. That's bad.
-    message (FATAL_ERROR "Failed to run test-libmongoc to discover tests [${retc}]:\n${tests_out}")
-endif ()
+    message(FATAL_ERROR "Failed to run test-libmongoc to discover tests [${retc}]:\n${tests_out}")
+endif()
 
-# Split lines on newlines
-string (REPLACE "\n" ";" lines "${tests_out}")
+# Execute the code that defines the test case information
+cmake_language(EVAL CODE "${tests_cmake}")
 
-# TODO: Allow individual test cases to specify the fixtures they want.
-set (all_fixtures "mongoc/fixtures/fake_kms_provider_server")
-set (all_env
+# Define environment variables that are common to all test cases
+set(all_env
     TEST_KMS_PROVIDER_HOST=localhost:14987  # Refer: Fixtures.cmake
     )
 
-# Generate the test definitions
-foreach (line IN LISTS lines)
-    if (NOT line MATCHES "^/")
-        # Only generate if the line begins with `/`, which all tests should.
-        continue ()
-    endif ()
-    # The new test name is prefixed with 'mongoc'
-    set (test "mongoc${line}")
-    # Define the test. Use `--ctest-run` to tell it that CTest is in control.
-    add_test ("${test}" "${TEST_LIBMONGOC_EXE}" --ctest-run "${line}")
-    set_tests_properties ("${test}" PROPERTIES
+# The emitted code defines a list MONGOC_TESTS with the name of every test case
+# in the suite.
+foreach(casename IN LISTS MONGOC_TESTS)
+    set(name "mongoc${casename}")
+    # Run the program with --ctest-run to select only this one test case
+    add_test("${name}" "${TEST_LIBMONGOC_EXE}" --ctest-run "${casename}")
+    # The emitted code defines a TAGS list for every test case that it emits. We use
+    # these as the LABELS for the test case
+    set(labels "${MONGOC_TEST_${casename}_TAGS}")
+
+    # Find what test fixtures the test wants by inspecting labels. Each "uses:"
+    # label defines the names of the test fixtures that a particular case requires
+    set(fixtures "${labels}")
+    list(FILTER fixtures INCLUDE REGEX "^uses:")
+    list(TRANSFORM fixtures REPLACE "^uses:(.*)$" "mongoc/fixtures/\\1")
+
+    # Add a label for all test cases generated via this script so that they
+    # can be (de)selected separately:
+    list(APPEND labels test-libmongoc-generated)
+    # Set up the test:
+    set_tests_properties("${name}" PROPERTIES
         # test-libmongoc expects to execute in the root of the source directory
         WORKING_DIRECTORY "${SRC_ROOT}"
         # If a test emits '@@ctest-skipped@@', this tells us that the test is
         # skipped.
         SKIP_REGULAR_EXPRESSION "@@ctest-skipped@@"
         # 45 seconds of timeout on each test.
         TIMEOUT 45
-        FIXTURES_REQUIRED "${all_fixtures}"
+        # Common environment variables:
         ENVIRONMENT "${all_env}"
-        # Mark all tests generated from the executable, so they can be (de)selected
-        # for execution separately.
-        LABELS "test-libmongoc-generated"
-        )
-endforeach ()
+        # Apply the labels
+        LABELS "${labels}"
+        # Fixture requirements:
+        FIXTURES_REQUIRED "${fixtures}"
+    )
+endforeach()

src/libmongoc/tests/TestSuite.c

@@ -20,6 +20,7 @@
 #include <mongoc/mongoc.h>
 
 #include <bson/bson.h>
+#include <bson/bson_t.h>
 
 #include <mlib/str.h>
 
@@ -168,6 +169,8 @@ TestSuite_Init(TestSuite *suite, const char *name, int argc, char **argv)
          suite->flags |= TEST_HELPTEXT;
       } else if (0 == strcmp("--list-tests", argv[i])) {
          suite->flags |= TEST_LISTTESTS;
+      } else if (0 == strcmp("--tests-cmake", argv[i])) {
+         suite->flags |= TEST_TESTS_CMAKE;
       } else if ((0 == strcmp("-s", argv[i])) || (0 == strcmp("--silent", argv[i]))) {
          suite->silent = true;
       } else if ((0 == strcmp("--ctest-run", argv[i]))) {
@@ -280,19 +283,37 @@ TestSuite_AddLive(TestSuite *suite, /* IN */
 
 
 Test *
-_V_TestSuite_AddFull(TestSuite *suite, const char *name, TestFuncWC func, TestFuncDtor dtor, void *ctx, va_list ap)
+_V_TestSuite_AddFull(
+   TestSuite *suite, const char *name_and_tags, TestFuncWC func, TestFuncDtor dtor, void *ctx, va_list ap)
 {
-   if (suite->ctest_run.data && mstr_cmp(suite->ctest_run, !=, mstr_cstring(name))) {
+   // Split the name and tags around the first whitespace:
+   mstr_view name, tags;
+   mstr_split_around(mstr_cstring(name_and_tags), mstr_cstring(" "), &name, &tags);
+   // Trim extras:
+   tags = mstr_trim(tags);
+
+   if (suite->ctest_run.data && mstr_cmp(suite->ctest_run, !=, name)) {
+      // We are running CTest, and not running this particular test, so just skip registering it
       if (dtor) {
          dtor(ctx);
       }
       return NULL;
    }
 
    Test *test = TestVec_push(&suite->tests);
-   test->name = mstr_copy_cstring(name);
+   test->name = mstr_copy(name);
    test->func = func;
 
+   mstr_view tag, tail;
+   tail = tags;
+   while (tail.len) {
+      mlib_check(mstr_split_around(tail, mstr_cstring("["), NULL, &tag),
+                 because,
+                 "Expected an opening bracket for the next test tag");
+      mlib_check(mstr_split_around(tag, mstr_cstring("]"), &tag, &tail), because, "Expected a closing bracket for tag");
+      *mstr_vec_push(&test->tags) = mstr_copy(tag);
+   }
+
    CheckFunc check;
    while ((check = va_arg(ap, CheckFunc))) {
       *CheckFuncVec_push(&test->checks) = check;
@@ -334,13 +355,12 @@ TestSuite_AddWC(TestSuite *suite,  /* IN */
 }
 
 
-void
-_TestSuite_AddFull(TestSuite *suite,  /* IN */
-                   const char *name,  /* IN */
-                   TestFuncWC func,   /* IN */
-                   TestFuncDtor dtor, /* IN */
-                   void *ctx,
-                   ...) /* IN */
+void(TestSuite_AddFull)(TestSuite *suite,  /* IN */
+                        const char *name,  /* IN */
+                        TestFuncWC func,   /* IN */
+                        TestFuncDtor dtor, /* IN */
+                        void *ctx,
+                        ...) /* IN */
 {
    va_list ap;
 
@@ -640,6 +660,7 @@ TestSuite_PrintHelp(TestSuite *suite) /* IN */
           "Options:\n"
           "    -h, --help            Show this help menu.\n"
           "    --list-tests          Print list of available tests.\n"
+          "    --tests-cmake         Print CMake code that defines test information.\n"
           "    -f, --no-fork         Do not spawn a process per test (abort on "
           "first error).\n"
           "    -l, --match PATTERN   Run test by name, e.g. \"/Client/command\" or "
@@ -668,6 +689,18 @@ TestSuite_PrintTests(TestSuite *suite) /* IN */
    printf("\n");
 }
 
+static void
+TestSuite_PrintCMake(TestSuite *suite)
+{
+   printf("set(MONGOC_TESTS)\n");
+   mlib_vec_foreach (Test, t, suite->tests) {
+      printf("list(APPEND MONGOC_TESTS [[%.*s]])\n", MSTR_FMT(t->name));
+      printf("set(MONGOC_TEST_%.*s_TAGS)\n", MSTR_FMT(t->name));
+      mlib_vec_foreach (mstr, tag, t->tags) {
+         printf("list(APPEND MONGOC_TEST_%.*s_TAGS [[%.*s]])\n", MSTR_FMT(t->name), MSTR_FMT(*tag));
+      }
+   }
+}
 
 static void
 TestSuite_PrintJsonSystemHeader(FILE *stream)
@@ -989,7 +1022,11 @@ TestSuite_Run(TestSuite *suite) /* IN */
       TestSuite_PrintTests(suite);
    }
 
-   if ((suite->flags & TEST_HELPTEXT) || (suite->flags & TEST_LISTTESTS)) {
+   if (suite->flags & TEST_TESTS_CMAKE) {
+      TestSuite_PrintCMake(suite);
+   }
+
+   if ((suite->flags & TEST_HELPTEXT) || (suite->flags & TEST_LISTTESTS) || (suite->flags & TEST_TESTS_CMAKE)) {
       return 0;
    }
 

src/libmongoc/tests/TestSuite.h

@@ -96,6 +96,7 @@ bson_open(const char *filename, int flags, ...)
 #define TEST_DEBUGOUTPUT (1 << 3)
 #define TEST_TRACE (1 << 4)
 #define TEST_LISTTESTS (1 << 5)
+#define TEST_TESTS_CMAKE (1 << 6)
 
 
 #define CERT_CA CERT_TEST_DIR "/ca.pem"
@@ -671,6 +672,10 @@ struct Test {
     * @brief The C string that names the test case
     */
    mstr name;
+   /**
+    * @brief Set of tags that are associated with this test case
+    */
+   mstr_vec tags;
    /**
     * @brief The function that will be executed for the test case
     */
@@ -704,6 +709,7 @@ Test_Destroy(Test *t)
       t->dtor(t->ctx);
    }
    mstr_delete(t->name);
+   mstr_vec_destroy(&t->tags);
    CheckFuncVec_destroy(&t->checks);
 }
 
@@ -764,11 +770,10 @@ struct _TestFnCtx {
    TestFuncDtor dtor;
 };
 
-
 void
 TestSuite_Init(TestSuite *suite, const char *name, int argc, char **argv);
 void
-TestSuite_Add(TestSuite *suite, const char *name, TestFunc func);
+TestSuite_Add(TestSuite *suite, const char *name_and_tags, TestFunc func);
 int
 TestSuite_CheckLive(void);
 void
@@ -779,21 +784,22 @@ void
 _TestSuite_AddMockServerTest(TestSuite *suite, const char *name, TestFunc func, ...);
 #define TestSuite_AddMockServerTest(_suite, _name, ...) _TestSuite_AddMockServerTest(_suite, _name, __VA_ARGS__, NULL)
 void
-TestSuite_AddWC(TestSuite *suite, const char *name, TestFuncWC func, TestFuncDtor dtor, void *ctx);
+TestSuite_AddWC(TestSuite *suite, const char *name_and_tags, TestFuncWC func, TestFuncDtor dtor, void *ctx);
 Test *
-_V_TestSuite_AddFull(TestSuite *suite, const char *name, TestFuncWC func, TestFuncDtor dtor, void *ctx, va_list ap);
+_V_TestSuite_AddFull(
+   TestSuite *suite, const char *name_and_tags, TestFuncWC func, TestFuncDtor dtor, void *ctx, va_list ap);
 void
-_TestSuite_AddFull(TestSuite *suite, const char *name, TestFuncWC func, TestFuncDtor dtor, void *ctx, ...);
+TestSuite_AddFull(TestSuite *suite, const char *name_and_tags, TestFuncWC func, TestFuncDtor dtor, void *ctx, ...);
 void
 _TestSuite_TestFnCtxDtor(void *ctx);
 #define TestSuite_AddFull(_suite, _name, _func, _dtor, _ctx, ...) \
-   _TestSuite_AddFull(_suite, _name, _func, _dtor, _ctx, __VA_ARGS__, NULL)
-#define TestSuite_AddFullWithTestFn(_suite, _name, _func, _dtor, _test_fn, ...)                   \
-   do {                                                                                           \
-      TestFnCtx *ctx = bson_malloc(sizeof(TestFnCtx));                                            \
-      ctx->test_fn = (TestFunc)(_test_fn);                                                        \
-      ctx->dtor = _dtor;                                                                          \
-      _TestSuite_AddFull(_suite, _name, _func, _TestSuite_TestFnCtxDtor, ctx, __VA_ARGS__, NULL); \
+   (TestSuite_AddFull)(_suite, _name, _func, _dtor, _ctx, __VA_ARGS__, NULL)
+#define TestSuite_AddFullWithTestFn(_suite, _name, _func, _dtor, _test_fn, ...)                  \
+   do {                                                                                          \
+      TestFnCtx *ctx = bson_malloc(sizeof(TestFnCtx));                                           \
+      ctx->test_fn = (TestFunc)(_test_fn);                                                       \
+      ctx->dtor = _dtor;                                                                         \
+      TestSuite_AddFull(_suite, _name, _func, _TestSuite_TestFnCtxDtor, ctx, __VA_ARGS__, NULL); \
    } while (0)
 int
 TestSuite_Run(TestSuite *suite);

src/libmongoc/tests/test-mcd-azure-imds.c

@@ -107,5 +107,10 @@ test_mcd_azure_imds_install(TestSuite *suite)
 {
    TestSuite_Add(suite, "/azure/imds/http/parse", _test_oauth_parse);
    TestSuite_Add(suite, "/azure/imds/http/request", _test_http_req);
-   TestSuite_AddFull(suite, "/azure/imds/http/talk", _test_with_mock_server, NULL, NULL, have_mock_server_env);
+   TestSuite_AddFull(suite,
+                     "/azure/imds/http/talk [uses:fake_kms_provider_server]",
+                     _test_with_mock_server,
+                     NULL,
+                     NULL,
+                     have_mock_server_env);
 }