Add tests using lit as a test harness

.gitignore

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+__pycache__/
 .vs/
 .vscode/
 /build/

CMakeLists.txt

@@ -87,3 +87,7 @@ get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\
 set(TOOLSET_LIB "${TOOLSET_ROOT_DIR}/lib/${VCLIBS_X86_OR_X64}")
 
 add_subdirectory(stl)
+
+# Since we don't configure any of the tests with CMake, there is no downside to enabling testing unconditionally.
+enable_testing()
+add_subdirectory(tests)

azure-devops/run-build.yml

@@ -3,6 +3,7 @@
 
 jobs:
 - job: ${{ parameters.targetPlatform }}
+  timeoutInMinutes: 360
   pool:
     name: StlBuild-2020-03-24
 
@@ -24,15 +25,16 @@ jobs:
       condition: and(ne(variables.CACHE_RESTORED, 'true'), contains('${{ parameters.targetPlatform }}', 'arm'))
       timeoutInMinutes: 10
       inputs:
-        vcpkgArguments: 'boost-build:x86-windows'
+        vcpkgArguments: 'boost-build'
         vcpkgDirectory: '$(vcpkgLocation)'
+        vcpkgTriplet: 'x86-windows'
     - task: run-vcpkg@0
       displayName: 'Run vcpkg'
       timeoutInMinutes: 10
       inputs:
         vcpkgArguments: 'boost-math'
         vcpkgDirectory: '$(vcpkgLocation)'
-        vcpkgTriplet: ${{ parameters.targetPlatform }}-windows
+        vcpkgTriplet: '${{ parameters.targetPlatform }}-windows'
     - task: run-cmake@0
       displayName: 'Build Support Tools'
       timeoutInMinutes: 2
@@ -62,7 +64,40 @@ jobs:
       displayName: 'Build the STL'
       timeoutInMinutes: 10
       inputs:
-        cmakeListsTxtPath: 'CMakeSettings.json'
-        useVcpkgToolchainFile: true
-        configurationRegexFilter: '.*${{ parameters.targetPlatform }}.*'
+        cmakeListsOrSettingsJson: 'CMakeListsTxtAdvanced'
+        cmakeListsTxtPath: '$(Build.SourcesDirectory)/CMakeLists.txt'
         buildDirectory: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }}
+        useVcpkgToolchainFile: true
+        cmakeAppendedArgs: '-G Ninja -DENABLE_XUNIT_OUTPUT=TRUE'
+    - task: PowerShell@2
+      displayName: 'Run Tests'
+      timeoutInMinutes: 120
+      condition: in('${{ parameters.targetPlatform }}', 'x64', 'x86')
+      inputs:
+        workingDirectory: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }}
+        targetType: inline
+        script: |
+          Add-Content -Path ./run-tests.cmd -Value "set PATH=$(split-path (get-command ctest).source);%PATH%"
+          Add-Content -Path ./run-tests.cmd -Value "set PATH=$(split-path (get-command clang-cl).source);%PATH%"
+          $currentDir = "$((Get-Item -Path "./").FullName)"
+          Add-Content -Path ./run-tests.cmd -Value "cd $currentDir"
+          Add-Content -Path ./run-tests.cmd -Value "ctest -V"
+          $(vcpkgLocation)/vcpkg.exe env --triplet ${{ parameters.targetPlatform }}-windows "$currentDir/run-tests.cmd"
+    - task: PublishTestResults@2
+      displayName: 'Publish libcxx Tests'
+      timeoutInMinutes: 10
+      condition: in('${{ parameters.targetPlatform }}', 'x64', 'x86')
+      inputs:
+        searchFolder: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }}
+        testResultsFormat: JUnit
+        testResultsFiles: '**/libcxx.test.xml'
+        testRunTitle: 'libcxx-${{ parameters.targetPlatform }}'
+    - task: PublishTestResults@2
+      displayName: 'Publish std Tests'
+      timeoutInMinutes: 10
+      condition: in('${{ parameters.targetPlatform }}', 'x64', 'x86')
+      inputs:
+        searchFolder: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }}
+        testResultsFormat: JUnit
+        testResultsFiles: '**/std.test.xml'
+        testRunTitle: 'std-${{ parameters.targetPlatform }}'

tests/CMakeLists.txt

@@ -0,0 +1,39 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set(STL_BUILD_ROOT "${PROJECT_BINARY_DIR}/out")
+set(STL_SOURCE_DIR "${PROJECT_SOURCE_DIR}")
+set(STL_TEST_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}")
+set(STL_TEST_UTILS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/utils")
+set(STL_TESTED_HEADERS_DIR "${STL_BUILD_ROOT}/inc")
+
+set(LLVM_PROJECT_SOURCE_DIR "${STL_SOURCE_DIR}/llvm-project" CACHE PATH
+    "Location of the llvm-project source tree")
+# This being a cache variable serves the dual purpose of letting the user point
+# to different out-of-tree LLVM components and also override the set() call
+# in llvm-lit's CMakeLists.txt.
+set(LLVM_SOURCE_DIR "${LLVM_PROJECT_SOURCE_DIR}/llvm" CACHE PATH
+    "Location of the llvm source tree")
+set(LLVM_LIT_SOURCE_DIR "${LLVM_SOURCE_DIR}/utils/llvm-lit")
+set(LLVM_LIT_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/llvm-lit")
+
+# We need to define this here as opposed to the libcxx CMakeLists.txt as
+# the std testsuite also needs to know where libcxx lives
+set(LIBCXX_SOURCE_DIR "${LLVM_PROJECT_SOURCE_DIR}/libcxx")
+
+list(APPEND CMAKE_MODULE_PATH "${LLVM_SOURCE_DIR}/cmake/modules")
+include(AddLLVM) # for get_llvm_lit_path needed in llvm-lit's CMakeLists.txt
+
+# The result of the first call to get_llvm_lit_path is cached
+get_llvm_lit_path(LIT_BASE_DIR LIT_FILE_NAME)
+set(LLVM_LIT_LOCATION "${LIT_BASE_DIR}/${LIT_FILE_NAME}")
+
+set(Python_FIND_STRATEGY VERSION)
+find_package(Python3)
+
+add_subdirectory(std)
+add_subdirectory(libcxx)
+
+# Add the llvm-lit subdirectory last so all the test directories have had a
+# chance to add to the config map.
+add_subdirectory(${LLVM_LIT_SOURCE_DIR} llvm-lit)

tests/libcxx/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set(LIBCXX_ENVLST "${CMAKE_CURRENT_SOURCE_DIR}/usual_matrix.lst")
+set(LIBCXX_EXPECTED_RESULTS "${CMAKE_CURRENT_SOURCE_DIR}/expected_results.txt")
+set(LIBCXX_TEST_OUTPUT_DIR "${STL_TEST_OUTPUT_DIR}/libcxx")
+
+configure_file(
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg)
+
+get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
+string(APPEND LLVM_LIT_CONFIG_MAP "map_config(\"${LIBCXX_SOURCE_DIR}/test/lit.cfg\", \"${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg\")\n")
+set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP ${LLVM_LIT_CONFIG_MAP})
+
+if(ENABLE_XUNIT_OUTPUT)
+    list(APPEND LIBCXX_ADDITIONAL_LIT_FLAGS "--xunit-xml-output" "${CMAKE_CURRENT_BINARY_DIR}/libcxx.test.xml")
+endif()
+
+list(APPEND LIBCXX_LLVM_LIT_COMMAND "${LLVM_LIT_LOCATION}"
+                                    "${ADDITIONAL_LIT_FLAGS}"
+                                    "${LIBCXX_ADDITIONAL_LIT_FLAGS}"
+                                    "${CMAKE_CURRENT_BINARY_DIR}")
+
+add_test(NAME libcxx COMMAND ${Python3_EXECUTABLE} ${LIBCXX_LLVM_LIT_COMMAND} COMMAND_EXPAND_LISTS)
+set_tests_properties(libcxx PROPERTIES RUN_SERIAL TRUE)