Let's rewrite the test harness

azure-devops/run-build.yml

@@ -63,7 +63,7 @@ jobs:
         cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=$(vcpkgLocation)\scripts\buildsystems\vcpkg.cmake ^
         -DVCPKG_TARGET_TRIPLET=${{ parameters.targetPlatform }}-windows -DCMAKE_CXX_COMPILER=cl ^
         -DCMAKE_BUILD_TYPE=Release -DLIT_FLAGS=$(litFlags) ^
-        -DCMAKE_CXX_FLAGS=/analyze ^
+        -DCMAKE_CXX_FLAGS=/analyze:autolog- ^
         -S $(Build.SourcesDirectory) -B $(buildOutputLocation)
         cmake --build $(buildOutputLocation)
       displayName: 'Build the STL'

tests/CMakeLists.txt

@@ -9,6 +9,8 @@ set(STL_TESTED_HEADERS_DIR "${STL_BUILD_ROOT}/inc")
 
 set(LLVM_PROJECT_SOURCE_DIR "${STL_SOURCE_DIR}/llvm-project" CACHE PATH
     "Location of the llvm-project source tree")
+set(LIBCXX_SOURCE_DIR "${LLVM_PROJECT_SOURCE_DIR}/libcxx" CACHE PATH
+    "Location of the libcxx source tree")
 
 add_subdirectory(libcxx)
 add_subdirectory(std)

tests/libcxx/CMakeLists.txt

@@ -3,14 +3,14 @@
 
 set(LIBCXX_ENVLST "${CMAKE_CURRENT_SOURCE_DIR}/usual_matrix.lst")
 set(LIBCXX_EXPECTED_RESULTS "${CMAKE_CURRENT_SOURCE_DIR}/expected_results.txt")
-set(LIBCXX_SOURCE_DIR "${LLVM_PROJECT_SOURCE_DIR}/libcxx" CACHE PATH
-    "Location of the libcxx source tree")
 set(LIBCXX_TEST_OUTPUT_DIR "${STL_TEST_OUTPUT_DIR}/libcxx")
 
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg)
 
 set(LIBCXX_LIT_CONFIG_MAP "map_config(\"${LIBCXX_SOURCE_DIR}/test/lit.cfg.py\", \"${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg\")\n")
+string(APPEND LIBCXX_LIT_CONFIG_MAP "map_config(\"${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg\", \"${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg\")\n")
+
 set_property(GLOBAL APPEND_STRING PROPERTY STL_LIT_CONFIG_MAP ${LIBCXX_LIT_CONFIG_MAP})
 set_property(GLOBAL APPEND PROPERTY STL_LIT_TEST_DIRS "${LIBCXX_SOURCE_DIR}/test/std")

tests/libcxx/lit.cfg

@@ -1,29 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from pathlib import Path
-import importlib
-import site
-
-site.addsitedir(str(Path(__file__).parents[1] / "utils"))
-site.addsitedir(str(Path(__file__).parent / "tests"))
-
-config.name = 'libc++'
-config.suffixes = {".pass.cpp", ".fail.cpp"}
-
-if config.test_source_root is None:
-    config.test_source_root = str(Path(__file__).parent)
-
-config.test_exec_root = getattr(config, 'test_exec_root', None)
-
-if not config.test_exec_root:
-    import tempfile
-    config.test_exec_root = tempfile.mkdtemp(prefix=config.name + '-testsuite-')
-    lit_config.note('Creating temporary directory for tests: %s' % config.test_exec_root)
-
-config_module_name = getattr(config, 'config_module_name', 'stl.test.config')
-config_module = importlib.import_module(config_module_name)
-
-configuration = config_module.Configuration(lit_config, config)
-configuration.configure()
-config.test_format = configuration.get_test_format()
+lit_config.fatal(
+    "You seem to be running Lit directly -- you should be running Lit through "
+    "<build>/tests/utils/stl-lit/stl-lit.py, which will ensure that the right Lit configuration "
+    "file is used.")

tests/libcxx/lit.site.cfg.in

@@ -1,25 +1,40 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+import os
 import site
 site.addsitedir("@STL_TEST_UTILS_DIR@")
+site.addsitedir(os.path.join("@LIBCXX_SOURCE_DIR@", "utils"))
+import stl.test.config
+import stl.test.features
+import stl.test.file_parsing
+import stl.test.format
+import stl.test.params
 
-config.configuration_variant       = "stl"
-config.cxx_archive_root            = "@CMAKE_ARCHIVE_OUTPUT_DIRECTORY@"
-config.cxx_headers                 = "@STL_TESTED_HEADERS_DIR@"
-config.cxx_library_root            = "@CMAKE_LIBRARY_OUTPUT_DIRECTORY@"
-config.cxx_runtime_root            = "@CMAKE_RUNTIME_OUTPUT_DIRECTORY@"
-config.envlst_path                 = "@LIBCXX_ENVLST@"
-config.expected_results_list_path  = "@LIBCXX_EXPECTED_RESULTS@"
-config.format_name                 = "LibcxxTestFormat"
-config.include_dirs                = ["@LIBCXX_SOURCE_DIR@/test/support"]
-config.libcxx_obj_root             = "@LIBCXX_TEST_OUTPUT_DIR@"
-config.msvc_toolset_libs_root      = "@TOOLSET_LIB@"
-config.stl_build_root              = "@STL_BUILD_ROOT@"
-config.stl_src_root                = "@STL_SOURCE_DIR@"
-config.target_arch                 = "@VCLIBS_TARGET_ARCHITECTURE@"
-config.test_exec_root              = "@LIBCXX_TEST_OUTPUT_DIR@"
-config.test_source_root            = "@LIBCXX_SOURCE_DIR@/test"
-config.test_subdirs                = ["@LIBCXX_SOURCE_DIR@/test/std"]
+config.envlst_path      = '@LIBCXX_ENVLST@'
+config.name             = 'libc++'
+config.suffixes         = ['[.]pass[.]cpp$', '[.]fail[.]cpp$']
+config.test_exec_root   = '@LIBCXX_TEST_OUTPUT_DIR@'
+config.test_format      = stl.test.format.LibcxxTestFormat()
+config.test_source_root = '@LIBCXX_SOURCE_DIR@/test'
 
-lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")
+lit_config.expected_results = getattr(lit_config, 'expected_results', dict())
+lit_config.include_dirs     = getattr(lit_config, 'include_dirs', dict())
+lit_config.library_dirs     = getattr(lit_config, 'library_dirs', dict())
+lit_config.test_subdirs     = getattr(lit_config, 'test_subdirs', dict())
+
+lit_config.expected_results[config.name] = stl.test.file_parsing.parse_result_file('@LIBCXX_EXPECTED_RESULTS@')
+lit_config.include_dirs[config.name]     = ['@STL_TESTED_HEADERS_DIR@', '@LIBCXX_SOURCE_DIR@/test/support']
+lit_config.library_dirs[config.name]     = ['@CMAKE_LIBRARY_OUTPUT_DIRECTORY@', '@TOOLSET_LIB@']
+lit_config.test_subdirs[config.name]     = ['@LIBCXX_SOURCE_DIR@/test/std']
+
+lit_config.cxx_runtime = '@CMAKE_RUNTIME_OUTPUT_DIRECTORY@'
+lit_config.target_arch = '@VCLIBS_TARGET_ARCHITECTURE@'
+
+# Add parameters and features to the config
+stl.test.config.configure(
+    stl.test.params.getDefaultParameters(config, lit_config),
+    stl.test.features.getDefaultFeatures(config, lit_config),
+    config,
+    lit_config
+)

tests/libcxx/usual_matrix.lst

@@ -5,5 +5,5 @@ RUNALL_INCLUDE ..\universal_prefix.lst
 RUNALL_CROSSLIST
 PM_CL="/EHsc /MTd /std:c++latest /permissive- /FImsvc_stdlib_force_include.h /wd4643"
 RUNALL_CROSSLIST
-PM_CL="/analyze /Zc:preprocessor"
+PM_CL="/analyze:autolog- /Zc:preprocessor"
 PM_COMPILER="clang-cl" PM_CL="-fno-ms-compatibility -fno-delayed-template-parsing"

tests/std/lit.cfg

@@ -1,29 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from pathlib import Path
-import importlib
-import site
-
-site.addsitedir(str(Path(__file__).parents[1] / "utils"))
-site.addsitedir(str(Path(__file__).parent / "tests"))
-
-config.name = 'std'
-config.suffixes.add('test.cpp')
-
-if config.test_source_root is None:
-    config.test_source_root = str(Path(__file__).parent)
-
-config.test_exec_root = getattr(config, 'test_exec_root', None)
-
-if not config.test_exec_root:
-    import tempfile
-    config.test_exec_root = tempfile.mkdtemp(prefix=config.name + '-testsuite-')
-    lit_config.note('Creating temporary directory for tests: %s' % config.test_exec_root)
-
-config_module_name = getattr(config, 'config_module_name', 'stl.test.config')
-config_module = importlib.import_module(config_module_name)
-
-configuration = config_module.Configuration(lit_config, config)
-configuration.configure()
-config.test_format = configuration.get_test_format()
+lit_config.fatal(
+    "You seem to be running Lit directly -- you should be running Lit through "
+    "<build>/tests/utils/stl-lit/stl-lit.py, which will ensure that the right Lit configuration "
+    "file is used.")

tests/std/lit.site.cfg.in

@@ -1,18 +1,42 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-config.cxx_archive_root            = "@CMAKE_ARCHIVE_OUTPUT_DIRECTORY@"
-config.cxx_headers                 = "@STL_TESTED_HEADERS_DIR@"
-config.cxx_library_root            = "@CMAKE_LIBRARY_OUTPUT_DIRECTORY@"
-config.cxx_runtime_root            = "@CMAKE_RUNTIME_OUTPUT_DIRECTORY@"
-config.expected_results_list_path  = "@STD_EXPECTED_RESULTS@"
-config.include_dirs                = ["@LIBCXX_SOURCE_DIR@/test/support", "@STL_SOURCE_DIR@/tests/std/include"]
-config.msvc_toolset_libs_root      = "@TOOLSET_LIB@"
-config.stl_build_root              = "@STL_BUILD_ROOT@"
-config.stl_src_root                = "@STL_SOURCE_DIR@"
-config.target_arch                 = "@VCLIBS_TARGET_ARCHITECTURE@"
-config.test_exec_root              = "@STD_TEST_OUTPUT_DIR@"
-config.test_subdirs_file           = "@STD_TEST_SUBDIRS_FILE@"
-config.test_subdirs_root           = "@STD_TEST_SUBDIRS_ROOT@"
+import os
+import site
+site.addsitedir("@STL_TEST_UTILS_DIR@")
+site.addsitedir(os.path.join("@LIBCXX_SOURCE_DIR@", "utils"))
+import stl.test.config
+import stl.test.features
+import stl.test.file_parsing
+import stl.test.format
+import stl.test.params
 
-lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")
+config.name           = 'std'
+config.suffixes       = ['test[.]cpp$', 'test[.]compile[.]pass[.]cpp$']
+config.test_exec_root = "@STD_TEST_OUTPUT_DIR@"
+config.test_format    = stl.test.format.STLTestFormat()
+
+lit_config.expected_results = getattr(lit_config, 'expected_results', dict())
+lit_config.include_dirs     = getattr(lit_config, 'include_dirs', dict())
+lit_config.library_dirs     = getattr(lit_config, 'library_dirs', dict())
+lit_config.test_subdirs     = getattr(lit_config, 'test_subdirs', dict())
+
+lit_config.expected_results[config.name] = stl.test.file_parsing.parse_result_file('@STD_EXPECTED_RESULTS@')
+lit_config.include_dirs[config.name]     = \
+    ['@STL_TESTED_HEADERS_DIR@', '@LIBCXX_SOURCE_DIR@/test/support', '@STL_SOURCE_DIR@/tests/std/include']
+lit_config.library_dirs[config.name]     = ['@CMAKE_LIBRARY_OUTPUT_DIRECTORY@', '@TOOLSET_LIB@']
+lit_config.test_subdirs[config.name]     = \
+    [os.path.normpath(os.path.join('@STD_TEST_SUBDIRS_ROOT@', path)) for \
+     path in stl.test.file_parsing.parse_commented_file('@STD_TEST_SUBDIRS_FILE@')]
+
+lit_config.cxx_headers = '@STL_TESTED_HEADERS_DIR@'
+lit_config.cxx_runtime = '@CMAKE_RUNTIME_OUTPUT_DIRECTORY@'
+lit_config.target_arch = '@VCLIBS_TARGET_ARCHITECTURE@'
+
+# Add parameters and features to the config
+stl.test.config.configure(
+    stl.test.params.getDefaultParameters(config, lit_config),
+    stl.test.features.getDefaultFeatures(config, lit_config),
+    config,
+    lit_config
+)

tests/std/tests/Dev09_056375_locale_cleanup/custom_format.py

@@ -1,42 +1,23 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from pathlib import Path
+import os
 
 from stl.test.format import STLTestFormat, TestStep
 
 
 class CustomTestFormat(STLTestFormat):
-    def getBuildSteps(self, test, lit_config, shared):
-        exe_source = Path(test.getSourcePath())
-        dll_source = exe_source.parent / 'TestDll.cpp'
-        shared.exec_dir = test.getExecDir()
-        output_base = test.getOutputBaseName()
-        output_dir = test.getOutputDir()
-        pass_var, fail_var = test.getPassFailResultCodes()
-        dll_output = output_dir / 'TestDll.DLL'
-
-        dll_compile_cmd, out_files, exec_file = \
-            test.cxx.executeBasedOnFlagsCmd([dll_source], output_dir,
-                                            shared.exec_dir, 'TestDll',
-                                            ['/Fe' + str(dll_output)],
-                                            [], ['/DLL'])
-
-        shared.dll_file = dll_output
-
-        yield TestStep(dll_compile_cmd, shared.exec_dir, [dll_source],
-                       test.cxx.compile_env)
-
-        exe_compile_cmd, out_files, shared.exec_file = \
-            test.cxx.executeBasedOnFlagsCmd([exe_source], output_dir,
-                                            shared.exec_dir, output_base,
-                                            [], [], [])
-
-        yield TestStep(exe_compile_cmd, shared.exec_dir, [exe_source],
-                       test.cxx.compile_env)
-
-    def getTestSteps(self, test, lit_config, shared):
-        if shared.exec_file is not None:
-            yield TestStep([str(shared.exec_file)], shared.exec_dir,
-                           [shared.exec_file, shared.dll_file],
-                           test.cxx.compile_env)
+    def getBuildSteps(self, test, litConfig, shared):
+        exeSource = test.getSourcePath()
+        dllSource = os.path.join(os.path.dirname(exeSource), 'TestDll.cpp')
+
+        outputDir, outputBase = test.getTempPaths()
+        dllOutput = os.path.join(outputDir, 'TestDll.DLL')
+
+        cmd = [test.cxx, dllSource, *test.flags, *test.compileFlags, '/Fe' + dllOutput,
+               '/link', '/DLL', *test.linkFlags]
+
+        yield TestStep(cmd, shared.execDir, shared.env, False)
+
+        for step in super().getBuildSteps(test, litConfig, shared):
+            yield step

tests/std/tests/Dev09_056375_locale_cleanup/lit.local.cfg

@@ -1,10 +1,9 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+import os
+import site
+site.addsitedir(os.path.dirname(os.path.dirname(__file__)))
 import Dev09_056375_locale_cleanup.custom_format
 
-config.test_format = \
-    Dev09_056375_locale_cleanup.custom_format.CustomTestFormat(config.test_format.cxx,
-                                                               config.test_format.execute_external,
-                                                               config.test_format.build_executor,
-                                                               config.test_format.test_executor)
+config.test_format = Dev09_056375_locale_cleanup.custom_format.CustomTestFormat()

tests/std/tests/Dev09_172666_tr1_tuple_odr/custom_format.py

@@ -1,23 +1,24 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from pathlib import Path
+import os
 
 from stl.test.format import STLTestFormat, TestStep
+from stl.test.tests import TestType
 
 
 class CustomTestFormat(STLTestFormat):
-    def getBuildSteps(self, test, lit_config, shared):
-        shared.exec_dir = test.getExecDir()
-        exe_source = Path(test.getSourcePath())
-        test2_source = exe_source.parent / 'test2.cpp'
-        output_base = test.getOutputBaseName()
-        output_dir = test.getOutputDir()
+    def getBuildSteps(self, test, litConfig, shared):
+        exeSource = test.getSourcePath()
+        test2Source = os.path.join(os.path.dirname(exeSource), 'test2.cpp')
 
-        cmd, out_files, shared.exec_file = \
-            test.cxx.executeBasedOnFlagsCmd([exe_source, test2_source],
-                                            output_dir, shared.exec_dir,
-                                            output_base, [], [], [])
+        outputDir, outputBase = test.getTempPaths()
 
-        yield TestStep(cmd, shared.exec_dir, [exe_source, test2_source],
-                       test.cxx.compile_env)
+        if TestType.COMPILE in test.testType:
+            cmd = [test.cxx, '/c', exeSource, test2Source, *test.flags, *test.compileFlags]
+        elif TestType.RUN in test.testType:
+            shared.execFile = outputBase + '.exe'
+            cmd = [test.cxx, exeSource, test2Source, *test.flags, *test.compileFlags, '/Fe' + shared.execFile,
+                   '/link', *test.linkFlags]
+
+        yield TestStep(cmd, shared.execDir, shared.env, False)

tests/std/tests/Dev09_172666_tr1_tuple_odr/lit.local.cfg

@@ -1,10 +1,9 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+import os
+import site
+site.addsitedir(os.path.dirname(os.path.dirname(__file__)))
 import Dev09_172666_tr1_tuple_odr.custom_format
 
-config.test_format = \
-    Dev09_172666_tr1_tuple_odr.custom_format.CustomTestFormat(config.test_format.cxx,
-                                                              config.test_format.execute_external,
-                                                              config.test_format.build_executor,
-                                                              config.test_format.test_executor)
+config.test_format = Dev09_172666_tr1_tuple_odr.custom_format.CustomTestFormat()

tests/std/tests/GH_000545_include_compare/custom_format.py

@@ -1,28 +1,27 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from pathlib import Path
 import os
 
 from stl.test.format import STLTestFormat, TestStep
+from stl.test.tests import TestType
 
 
 class CustomTestFormat(STLTestFormat):
-    def getBuildSteps(self, test, lit_config, shared):
-        shared.exec_dir = test.getExecDir()
-        output_base = test.getOutputBaseName()
-        output_dir = test.getOutputDir()
-        exe_source_dir = Path(test.getSourcePath()).parent
+    def getBuildSteps(self, test, litConfig, shared):
+        exeSourceDir = os.path.dirname(test.getSourcePath())
+        _, outputBase = test.getTempPaths()
 
-        source_files = []
-        for filename in os.listdir(exe_source_dir):
+        sourceFiles = []
+        for filename in os.listdir(exeSourceDir):
             if filename.endswith('.cpp'):
-                source_files.append(exe_source_dir / filename)
+                sourceFiles.append(os.path.join(exeSourceDir, filename))
 
-        cmd, out_files, shared.exec_file = \
-            test.cxx.executeBasedOnFlagsCmd(source_files, output_dir,
-                                            shared.exec_dir, output_base,
-                                            [], [], [])
+        if TestType.COMPILE in test.testType:
+            cmd = [test.cxx, '/c', *sourceFiles, *test.flags, *test.compileFlags]
+        elif TestType.RUN in test.testType:
+            shared.execFile = outputBase + '.exe'
+            cmd = [test.cxx, *sourceFiles, *test.flags, *test.compileFlags, '/Fe' + shared.execFile,
+                   '/link', *test.linkFlags]
 
-        yield TestStep(cmd, shared.exec_dir, source_files,
-                       test.cxx.compile_env)
+        yield TestStep(cmd, shared.execDir, shared.env, False)