diff --git a/.gitignore b/.gitignore index 757e498526e..c55023a9a53 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +__pycache__/ .vs/ .vscode/ /build/ diff --git a/CMakeLists.txt b/CMakeLists.txt index d2ee9e77b84..5de1c9e7ce2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,3 +87,7 @@ get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\ set(TOOLSET_LIB "${TOOLSET_ROOT_DIR}/lib/${VCLIBS_X86_OR_X64}") add_subdirectory(stl) + +# Since we don't configure any of the tests with CMake, there is no downside to enabling testing unconditionally. +enable_testing() +add_subdirectory(tests) diff --git a/azure-devops/run-build.yml b/azure-devops/run-build.yml index 426b56de3b2..ec45989f85e 100644 --- a/azure-devops/run-build.yml +++ b/azure-devops/run-build.yml @@ -3,6 +3,7 @@ jobs: - job: ${{ parameters.targetPlatform }} + timeoutInMinutes: 360 pool: name: StlBuild-2020-03-24 @@ -24,15 +25,16 @@ jobs: condition: and(ne(variables.CACHE_RESTORED, 'true'), contains('${{ parameters.targetPlatform }}', 'arm')) timeoutInMinutes: 10 inputs: - vcpkgArguments: 'boost-build:x86-windows' + vcpkgArguments: 'boost-build' vcpkgDirectory: '$(vcpkgLocation)' + vcpkgTriplet: 'x86-windows' - task: run-vcpkg@0 displayName: 'Run vcpkg' timeoutInMinutes: 10 inputs: vcpkgArguments: 'boost-math' vcpkgDirectory: '$(vcpkgLocation)' - vcpkgTriplet: ${{ parameters.targetPlatform }}-windows + vcpkgTriplet: '${{ parameters.targetPlatform }}-windows' - task: run-cmake@0 displayName: 'Build Support Tools' timeoutInMinutes: 2 @@ -62,7 +64,40 @@ jobs: displayName: 'Build the STL' timeoutInMinutes: 10 inputs: - cmakeListsTxtPath: 'CMakeSettings.json' - useVcpkgToolchainFile: true - configurationRegexFilter: '.*${{ parameters.targetPlatform }}.*' + cmakeListsOrSettingsJson: 'CMakeListsTxtAdvanced' + cmakeListsTxtPath: '$(Build.SourcesDirectory)/CMakeLists.txt' buildDirectory: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }} + useVcpkgToolchainFile: true + cmakeAppendedArgs: '-G Ninja -DENABLE_XUNIT_OUTPUT=TRUE' + - task: PowerShell@2 + displayName: 'Run Tests' + timeoutInMinutes: 120 + condition: in('${{ parameters.targetPlatform }}', 'x64', 'x86') + inputs: + workingDirectory: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }} + targetType: inline + script: | + Add-Content -Path ./run-tests.cmd -Value "set PATH=$(split-path (get-command ctest).source);%PATH%" + Add-Content -Path ./run-tests.cmd -Value "set PATH=$(split-path (get-command clang-cl).source);%PATH%" + $currentDir = "$((Get-Item -Path "./").FullName)" + Add-Content -Path ./run-tests.cmd -Value "cd $currentDir" + Add-Content -Path ./run-tests.cmd -Value "ctest -V" + $(vcpkgLocation)/vcpkg.exe env --triplet ${{ parameters.targetPlatform }}-windows "$currentDir/run-tests.cmd" + - task: PublishTestResults@2 + displayName: 'Publish libcxx Tests' + timeoutInMinutes: 10 + condition: in('${{ parameters.targetPlatform }}', 'x64', 'x86') + inputs: + searchFolder: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }} + testResultsFormat: JUnit + testResultsFiles: '**/libcxx.test.xml' + testRunTitle: 'libcxx-${{ parameters.targetPlatform }}' + - task: PublishTestResults@2 + displayName: 'Publish std Tests' + timeoutInMinutes: 10 + condition: in('${{ parameters.targetPlatform }}', 'x64', 'x86') + inputs: + searchFolder: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }} + testResultsFormat: JUnit + testResultsFiles: '**/std.test.xml' + testRunTitle: 'std-${{ parameters.targetPlatform }}' diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 00000000000..17ac762f8d6 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set(STL_BUILD_ROOT "${PROJECT_BINARY_DIR}/out") +set(STL_SOURCE_DIR "${PROJECT_SOURCE_DIR}") +set(STL_TEST_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}") +set(STL_TEST_UTILS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/utils") +set(STL_TESTED_HEADERS_DIR "${STL_BUILD_ROOT}/inc") + +set(LLVM_PROJECT_SOURCE_DIR "${STL_SOURCE_DIR}/llvm-project" CACHE PATH + "Location of the llvm-project source tree") +# This being a cache variable serves the dual purpose of letting the user point +# to different out-of-tree LLVM components and also override the set() call +# in llvm-lit's CMakeLists.txt. +set(LLVM_SOURCE_DIR "${LLVM_PROJECT_SOURCE_DIR}/llvm" CACHE PATH + "Location of the llvm source tree") +set(LLVM_LIT_SOURCE_DIR "${LLVM_SOURCE_DIR}/utils/llvm-lit") +set(LLVM_LIT_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/llvm-lit") + +# We need to define this here as opposed to the libcxx CMakeLists.txt as +# the std testsuite also needs to know where libcxx lives +set(LIBCXX_SOURCE_DIR "${LLVM_PROJECT_SOURCE_DIR}/libcxx") + +list(APPEND CMAKE_MODULE_PATH "${LLVM_SOURCE_DIR}/cmake/modules") +include(AddLLVM) # for get_llvm_lit_path needed in llvm-lit's CMakeLists.txt + +# The result of the first call to get_llvm_lit_path is cached +get_llvm_lit_path(LIT_BASE_DIR LIT_FILE_NAME) +set(LLVM_LIT_LOCATION "${LIT_BASE_DIR}/${LIT_FILE_NAME}") + +set(Python_FIND_STRATEGY VERSION) +find_package(Python3) + +add_subdirectory(std) +add_subdirectory(libcxx) + +# Add the llvm-lit subdirectory last so all the test directories have had a +# chance to add to the config map. +add_subdirectory(${LLVM_LIT_SOURCE_DIR} llvm-lit) diff --git a/tests/libcxx/CMakeLists.txt b/tests/libcxx/CMakeLists.txt new file mode 100644 index 00000000000..0dfeaf509eb --- /dev/null +++ b/tests/libcxx/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set(LIBCXX_ENVLST "${CMAKE_CURRENT_SOURCE_DIR}/usual_matrix.lst") +set(LIBCXX_EXPECTED_RESULTS "${CMAKE_CURRENT_SOURCE_DIR}/expected_results.txt") +set(LIBCXX_TEST_OUTPUT_DIR "${STL_TEST_OUTPUT_DIR}/libcxx") + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg) + +get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP) +string(APPEND LLVM_LIT_CONFIG_MAP "map_config(\"${LIBCXX_SOURCE_DIR}/test/lit.cfg\", \"${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg\")\n") +set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP ${LLVM_LIT_CONFIG_MAP}) + +if(ENABLE_XUNIT_OUTPUT) + list(APPEND LIBCXX_ADDITIONAL_LIT_FLAGS "--xunit-xml-output" "${CMAKE_CURRENT_BINARY_DIR}/libcxx.test.xml") +endif() + +list(APPEND LIBCXX_LLVM_LIT_COMMAND "${LLVM_LIT_LOCATION}" + "${ADDITIONAL_LIT_FLAGS}" + "${LIBCXX_ADDITIONAL_LIT_FLAGS}" + "${CMAKE_CURRENT_BINARY_DIR}") + +add_test(NAME libcxx COMMAND ${Python3_EXECUTABLE} ${LIBCXX_LLVM_LIT_COMMAND} COMMAND_EXPAND_LISTS) +set_tests_properties(libcxx PROPERTIES RUN_SERIAL TRUE) diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt new file mode 100644 index 00000000000..18f489c73f1 --- /dev/null +++ b/tests/libcxx/expected_results.txt @@ -0,0 +1,1048 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# *** ISSUES REPORTED/KNOWN UPSTREAM *** +# Non-Standard regex behavior. +# "It seems likely that the test is still non-conforming due to how libc++ handles the 'w' character class." +std/re/re.traits/lookup_classname.pass.cpp SKIP + +# These tests are extremely slow, taking over 23 minutes to execute (in debug mode, non-optimized). +# They contain 10K^2 / 2 == 50M loops. +std/input.output/iostreams.base/ios.base/ios.base.storage/iword.pass.cpp SKIP +std/input.output/iostreams.base/ios.base/ios.base.storage/pword.pass.cpp SKIP + +# "The behavior demonstrated in this test is not meant to be standard" +std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/null.pass.cpp SKIP + +# allocator. +std/utilities/memory/default.allocator/allocator.ctor.pass.cpp SKIP + +# path::value_type is char assumptions +std/input.output/file.streams/fstreams/filebuf.members/open_path.pass.cpp SKIP +std/input.output/file.streams/fstreams/fstream.cons/path.pass.cpp SKIP +std/input.output/file.streams/fstreams/fstream.members/open_path.pass.cpp SKIP +std/input.output/file.streams/fstreams/ofstream.cons/path.pass.cpp SKIP +std/input.output/file.streams/fstreams/ofstream.members/open_path.pass.cpp SKIP + +# This test is passing non-BidirectionalIterators to std::prev. +# LWG-3197 "std::prev should not require BidirectionalIterator" (New) +std/iterators/iterator.primitives/iterator.operations/prev.pass.cpp SKIP + +# Itanium ABI assumptions that current_exception and rethrow_exception don't copy the exception object +std/language.support/support.exception/propagation/current_exception.pass.cpp SKIP +std/language.support/support.exception/propagation/make_exception_ptr.pass.cpp SKIP +std/language.support/support.exception/propagation/rethrow_exception.pass.cpp SKIP +std/language.support/support.exception/except.nested/rethrow_if_nested.pass.cpp SKIP + +# Testing nonstandard behavior +std/utilities/template.bitset/bitset.cons/string_ctor.pass.cpp SKIP + +# This test has undefined behavior under N4842 [basic.start.term]/6 +std/thread/futures/futures.task/futures.task.members/dtor.pass.cpp SKIP + +# libcxx is incorrect on what the type passed to allocator construct should be +# See https://reviews.llvm.org/D61364 +std/containers/associative/map/map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp SKIP +std/containers/associative/set/insert_and_emplace_allocator_requirements.pass.cpp SKIP +std/containers/unord/unord.map/unord.map.modifiers/insert_and_emplace_allocator_requirements.pass.cpp SKIP +std/containers/unord/unord.set/insert_and_emplace_allocator_requirements.pass.cpp SKIP + +# libcxx doesn't yet implement P1423R3, so it expects an "old" value for __cpp_lib_char8_t +std/language.support/support.limits/support.limits.general/filesystem.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/istream.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/limits.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/locale.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/ostream.version.pass.cpp SKIP + + +# *** INTERACTIONS WITH CONTEST / C1XX THAT UPSTREAM LIKELY WON'T FIX *** +# Tracked by VSO-593630 " Enable libcxx filesystem tests" +# rapid-cxx-test.hpp uses pragma system_header +# test header filesystem_test_helper.hpp emits "error: "STATIC TESTS DISABLED"" +# const_cast from const std::wstring& to std::string& is not allowed +std/input.output/filesystems/class.directory_entry/directory_entry.cons/copy.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.cons/copy_assign.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.cons/move.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.cons/move_assign.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.mods/assign.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.mods/refresh.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.obs/status.pass.cpp SKIP +std/input.output/filesystems/class.directory_entry/directory_entry.obs/symlink_status.pass.cpp SKIP +std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy.pass.cpp SKIP +std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy_assign.pass.cpp SKIP +std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp SKIP +std/input.output/filesystems/class.directory_iterator/directory_iterator.members/increment.pass.cpp SKIP +std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move.pass.cpp SKIP +std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move_assign.pass.cpp SKIP +std/input.output/filesystems/class.directory_iterator/directory_iterator.nonmembers/begin_end.pass.cpp SKIP +std/input.output/filesystems/class.path/synop.pass.cpp SKIP +std/input.output/filesystems/class.path/path.itr/iterator.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.append.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.concat.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.assign/braced_init.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.assign/copy.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.assign/move.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.construct/copy.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.construct/move.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.construct/source.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.decompose/path.decompose.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.gen/lexically_normal.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.generic.obs/generic_string_alloc.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.generic.obs/named_overloads.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.modifiers/clear.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.modifiers/make_preferred.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.modifiers/remove_filename.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.modifiers/replace_extension.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.modifiers/replace_filename.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.modifiers/swap.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.native.obs/c_str.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.native.obs/named_overloads.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.native.obs/native.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.native.obs/operator_string.pass.cpp SKIP +std/input.output/filesystems/class.path/path.member/path.native.obs/string_alloc.pass.cpp SKIP +std/input.output/filesystems/class.path/path.nonmember/append_op.pass.cpp SKIP +std/input.output/filesystems/class.path/path.nonmember/path.factory.pass.cpp SKIP +std/input.output/filesystems/class.path/path.nonmember/path.io.pass.cpp SKIP +std/input.output/filesystems/class.path/path.nonmember/path.io.unicode_bug.pass.cpp SKIP +std/input.output/filesystems/class.path/path.nonmember/swap.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy_assign.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/depth.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/disable_recursion_pending.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move_assign.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/pop.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/recursion_pending.pass.cpp SKIP +std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.nonmembers/begin_end.pass.cpp SKIP +std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp SKIP +std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp SKIP +std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp SKIP +std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp SKIP +std/input.output/filesystems/fs.enum/enum.perms.pass.cpp SKIP +std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp SKIP +std/input.output/filesystems/fs.filesystem.synopsis/file_time_type.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.copy/copy.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file_large.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.copy_symlink/copy_symlink.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.create_directories/create_directories.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.create_directory_symlink/create_directory_symlink.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.create_hard_link/create_hard_link.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.create_symlink/create_symlink.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.exists/exists.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.file_size/file_size.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_block_file/is_block_file.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_char_file/is_character_file.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_fifo/is_fifo.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_other/is_other.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_regular_file/is_regular_file.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_socket/is_socket.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.proximate/proximate.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.read_symlink/read_symlink.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.remove/remove.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.remove_all/remove_all.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.rename/rename.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.resize_file/resize_file.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.space/space.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.status_known/status_known.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp SKIP +std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp SKIP + +# We need some way to turn on deprecations for tests asking for _LIBCPP_ENABLE_DEPRECATION_WARNINGS +std/utilities/function.objects/negators/binary_negate.depr_in_cxx17.fail.cpp SKIP +std/utilities/function.objects/negators/unary_negate.depr_in_cxx17.fail.cpp SKIP + +# generate_feature_test_macro_components.py needs to learn about C1XX +std/language.support/support.limits/support.limits.general/new.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/type_traits.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/version.version.pass.cpp SKIP + +# Contest does not understand .sh tests, which must be run specially +std/depr/depr.c.headers/stdint_h.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align_nothrow.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.array/new_size_nothrow.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.array/new_size.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align_nothrow.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.sh.cpp SKIP +std/thread/thread.condition/thread.condition.condvarany/wait_terminates.sh.cpp SKIP + +# These tests set an allocator with a max_size() too small to default construct an unordered container +# (due to our minimum bucket size). +std/containers/unord/unord.map/max_size.pass.cpp SKIP +std/containers/unord/unord.multimap/max_size.pass.cpp SKIP +std/containers/unord/unord.multiset/max_size.pass.cpp SKIP +std/containers/unord/unord.set/max_size.pass.cpp SKIP + +# Requires too great a template instantiation depth for C1XX. +std/utilities/tuple/tuple.tuple/tuple.apply/apply_large_arity.pass.cpp SKIP + + +# *** MISSING STL FEATURES *** +# C++20 P0019R8 "atomic_ref" +std/language.support/support.limits/support.limits.general/atomic.version.pass.cpp SKIP + +# C++20 P0355R7 " Calendars And Time Zones" +std/utilities/time/days.pass.cpp SKIP +std/utilities/time/months.pass.cpp SKIP +std/utilities/time/weeks.pass.cpp SKIP +std/utilities/time/years.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.members/decrement.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.members/increment.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.members/plus_minus_equal.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/literals.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.last/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.md/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.md/time.cal.md.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.md/time.cal.md.members/day.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.md/time.cal.md.members/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.md/time.cal.md.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.md/time.cal.md.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.md/time.cal.md.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mdlast/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mdlast/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mdlast/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mdlast/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mdlast/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mdlast/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.members/decrement.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.members/increment.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.members/plus_minus_equal.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/literals.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwd/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.members/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.members/weekday_indexed.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwdlast/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.members/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.members/weekday_last.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/month_day.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/month_day_last.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/month_weekday.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/month_weekday_last.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/year_month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/year_month_day.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/year_month_day_last.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/year_month_weekday.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.operators/year_month_weekday_last.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdidx/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.members/index.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.members/weekday.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdlast/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.members/weekday.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/c_encoding.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/ctor.local_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/ctor.sys_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/decrement.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/increment.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/iso_encoding.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/operator[].pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.members/plus_minus_equal.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/literals.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.members/decrement.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.members/increment.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.members/is_leap.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.members/plus_minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.members/plus_minus_equal.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/literals.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.members/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.members/plus_minus_equal_month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.members/plus_minus_equal_year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.members/year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/ctor.local_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/ctor.sys_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/ctor.year_month_day_last.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/day.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/op.local_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/op.sys_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/plus_minus_equal_month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/plus_minus_equal_year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.members/year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/day.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/month_day_last.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/op_local_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/op_sys_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/plus_minus_equal_month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/plus_minus_equal_year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.members/year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/ctor.local_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/ctor.sys_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/index.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/op.local_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/op.sys_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/plus_minus_equal_month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/plus_minus_equal_year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/weekday.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/weekday_indexed.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.members/year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/types.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/ctor.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/ok.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/op_local_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/op_sys_days.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/plus_minus_equal_month.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/plus_minus_equal_year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/weekday.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.members/year.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.nonmembers/comparisons.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.nonmembers/minus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.nonmembers/plus.pass.cpp SKIP +std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.nonmembers/streaming.pass.cpp SKIP +std/utilities/time/time.clock/time.clock.file/consistency.pass.cpp SKIP +std/utilities/time/time.clock/time.clock.file/file_time.pass.cpp SKIP +std/utilities/time/time.clock/time.clock.file/now.pass.cpp SKIP +std/utilities/time/time.clock/time.clock.file/rep_signed.pass.cpp SKIP +std/utilities/time/time.clock/time.clock.system/local_time.types.pass.cpp SKIP +std/utilities/time/time.clock/time.clock.system/sys.time.types.pass.cpp SKIP +std/utilities/time/time.duration/time.duration.literals/literals1.pass.cpp SKIP +std/utilities/time/time.hms/time.12/is_am.pass.cpp SKIP +std/utilities/time/time.hms/time.12/is_pm.pass.cpp SKIP +std/utilities/time/time.hms/time.12/make12.pass.cpp SKIP +std/utilities/time/time.hms/time.12/make24.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/hours.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/is_negative.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/minutes.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/precision.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/precision_type.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/seconds.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/subseconds.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/to_duration.pass.cpp SKIP +std/utilities/time/time.hms/time.hms.members/width.pass.cpp SKIP + +# C++20 P0415R1 "constexpr For (Again)" +std/numerics/complex.number/cmplx.over/imag.pass.cpp SKIP +std/numerics/complex.number/cmplx.over/real.pass.cpp SKIP + +# C++20 P0476R2 " bit_cast" +std/language.support/support.limits/support.limits.general/bit.version.pass.cpp SKIP + +# C++20 P0553R4 " Rotating And Counting Functions" +std/numerics/bit/bitops.count/countl_one.pass.cpp SKIP +std/numerics/bit/bitops.count/countl_zero.pass.cpp SKIP +std/numerics/bit/bitops.count/countr_one.pass.cpp SKIP +std/numerics/bit/bitops.count/countr_zero.pass.cpp SKIP +std/numerics/bit/bitops.count/popcount.pass.cpp SKIP +std/numerics/bit/bitops.rot/rotl.pass.cpp SKIP +std/numerics/bit/bitops.rot/rotr.pass.cpp SKIP + +# C++20 P0556R3 " ispow2(), ceil2(), floor2(), log2p1()" +std/numerics/bit/bit.pow.two/ceil2.pass.cpp SKIP +std/numerics/bit/bit.pow.two/floor2.pass.cpp SKIP +std/numerics/bit/bit.pow.two/ispow2.pass.cpp SKIP +std/numerics/bit/bit.pow.two/log2p1.pass.cpp SKIP + +# C++20 P0608R3 "Improving variant's Converting Constructor/Assignment" +std/utilities/variant/variant.variant/variant.assign/conv.pass.cpp SKIP +std/utilities/variant/variant.variant/variant.assign/T.pass.cpp SKIP +std/utilities/variant/variant.variant/variant.ctor/conv.pass.cpp SKIP +std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp SKIP + +# C++20 P0768R1 "Library Support for the Spaceship (Comparison) Operator" +std/language.support/support.limits/support.limits.general/compare.version.pass.cpp SKIP + +# C++20 P0811R2 "midpoint(), lerp()" +std/language.support/support.limits/support.limits.general/numeric.version.pass.cpp SKIP +std/numerics/c.math/c.math.lerp/c.math.lerp.pass.cpp SKIP + +# C++20 P0896R4 "" +std/language.support/support.limits/support.limits.general/algorithm.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/functional.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/iterator.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/memory.version.pass.cpp SKIP + +# C++20 P1023R0 "constexpr For std::array Comparisons" +std/containers/sequences/array/compare.pass.cpp SKIP + +# C++20 P1032R1 "Miscellaneous constexpr" +std/language.support/support.limits/support.limits.general/array.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/functional.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/iterator.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/string_view.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/tuple.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/utility.version.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char/assign3.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char/copy.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char/move.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char16_t/assign3.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char16_t/copy.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char16_t/move.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char32_t/assign3.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char32_t/copy.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char32_t/move.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char8_t/assign3.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char8_t/copy.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.char8_t/move.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.wchar.t/assign3.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.wchar.t/copy.pass.cpp SKIP +std/strings/char.traits/char.traits.specializations/char.traits.specializations.wchar.t/move.pass.cpp SKIP + +# C++20 P1135R6 "The C++20 Synchronization Library" +std/atomics/types.pass.cpp SKIP +std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait.pass.cpp SKIP +std/thread/thread.barrier/arrive.pass.cpp SKIP +std/thread/thread.barrier/arrive_and_drop.pass.cpp SKIP +std/thread/thread.barrier/arrive_and_wait.pass.cpp SKIP +std/thread/thread.barrier/completion.pass.cpp SKIP +std/thread/thread.barrier/max.pass.cpp SKIP +std/thread/thread.barrier/version.pass.cpp SKIP +std/thread/thread.latch/arrive_and_wait.pass.cpp SKIP +std/thread/thread.latch/count_down.pass.cpp SKIP +std/thread/thread.latch/max.pass.cpp SKIP +std/thread/thread.latch/try_wait.pass.cpp SKIP +std/thread/thread.latch/version.pass.cpp SKIP +std/thread/thread.semaphore/acquire.pass.cpp SKIP +std/thread/thread.semaphore/binary.pass.cpp SKIP +std/thread/thread.semaphore/max.pass.cpp SKIP +std/thread/thread.semaphore/release.pass.cpp SKIP +std/thread/thread.semaphore/timed.pass.cpp SKIP +std/thread/thread.semaphore/try_acquire.pass.cpp SKIP +std/thread/thread.semaphore/version.pass.cpp SKIP + + +# *** MISSING COMPILER FEATURES *** +# C++20 P0722R3 "Efficient sized delete for variable sized classes" +std/language.support/support.dynamic/destroying_delete_t_declaration.pass.cpp SKIP +std/language.support/support.dynamic/destroying_delete_t.pass.cpp SKIP + + +# *** MISSING LWG ISSUE RESOLUTIONS *** +# LWG-2532 "Satisfying a promise at thread exit" (Open) +# WCFB02 implements the proposed resolution for this issue +std/thread/futures/futures.promise/set_exception_at_thread_exit.pass.cpp SKIP +std/thread/futures/futures.promise/set_lvalue_at_thread_exit.pass.cpp SKIP +std/thread/futures/futures.promise/set_rvalue_at_thread_exit.pass.cpp SKIP +std/thread/futures/futures.promise/set_value_at_thread_exit_const.pass.cpp SKIP +std/thread/futures/futures.promise/set_value_at_thread_exit_void.pass.cpp SKIP +std/thread/futures/futures.task/futures.task.members/make_ready_at_thread_exit.pass.cpp SKIP + + +# *** C1XX COMPILER BUGS *** +# Compiler bug: VSO-120957 "alignas by-value parameters should be permitted" +std/utilities/meta/meta.trans/meta.trans.other/aligned_storage.pass.cpp SKIP + +# Compiler bug: VSO-106654 "error C2580 "multiple versions of a defaulted special member functions are not allowed" is bogus and ungrammatical" +std/utilities/tuple/tuple.tuple/tuple.cnstr/test_lazy_sfinae.pass.cpp:0 SKIP + +# Compiler bug: VSO-406936 "is_constructible and is_constructible> should be true" +std/utilities/meta/meta.unary/meta.unary.prop/is_constructible.pass.cpp:0 SKIP + +# Compiler bug: DevCom-876860 "conditional operator errors" blocks readable. +std/containers/views/span.cons/ptr_len.pass.cpp:0 SKIP +std/containers/views/span.cons/ptr_ptr.pass.cpp:0 SKIP + + +# *** CLANG COMPILER BUGS *** +# LLVM-33230 "Clang on Windows should define __STDCPP_THREADS__ to be 1" +std/thread/macro.pass.cpp:1 SKIP + +# hasn't been enabled for Clang yet. +std/iterators/iterator.primitives/iterator.traits/pointer.pass.cpp:1 SKIP +std/iterators/iterator.primitives/std.iterator.tags/contiguous_iterator_tag.pass.cpp:1 SKIP + +# Clang 9 doesn't support comparison rewriting; implemented in Clang 10. +std/containers/views/span.iterators/begin.pass.cpp SKIP +std/containers/views/span.iterators/end.pass.cpp SKIP +std/containers/views/span.iterators/rbegin.pass.cpp SKIP +std/containers/views/span.iterators/rend.pass.cpp SKIP + + +# *** CLANG ISSUES, NOT YET ANALYZED *** +# Clang doesn't enable sized deallocation by default. Should we add -fsized-deallocation or do something else? +std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array14.pass.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.sh.cpp SKIP +std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete14.pass.cpp SKIP + +# Not yet analyzed. Clang apparently defines platform macros differently from C1XX. +std/language.support/support.limits/limits/numeric.limits.members/traps.pass.cpp SKIP + + +# *** STL BUGS *** +# STL bug: VSO-121977 ": the enum value of std::money_base is not correct[libcxx]" +std/localization/locale.categories/category.monetary/locale.moneypunct/money_base.pass.cpp FAIL + +# STL Bug: VSO-595631 basic_filebuf doesn't comply with setbuf(0,0) requirement in the standard +std/input.output/file.streams/fstreams/filebuf.virtuals/overflow.pass.cpp FAIL +std/input.output/file.streams/fstreams/filebuf.virtuals/underflow.pass.cpp FAIL + +# STL bug: We don't have tgmath.h. +std/depr/depr.c.headers/tgmath_h.pass.cpp SKIP + +# STL bug: Our inheritance implementation is allowing this to compile when it shouldn't. +std/numerics/complex.number/complex.special/double_long_double_implicit.fail.cpp FAIL +std/numerics/complex.number/complex.special/float_double_implicit.fail.cpp FAIL +std/numerics/complex.number/complex.special/float_long_double_implicit.fail.cpp FAIL + +# STL bug: regex_traits::transform() isn't following the Standard. +std/re/re.traits/transform.pass.cpp FAIL + +# STL bug: Incorrect return types. +std/numerics/complex.number/cmplx.over/conj.pass.cpp FAIL +std/numerics/complex.number/cmplx.over/pow.pass.cpp FAIL +std/numerics/complex.number/cmplx.over/proj.pass.cpp FAIL + +# STL bug: Missing assignment operators. +std/numerics/numarray/template.mask.array/mask.array.assign/mask_array.pass.cpp FAIL +std/numerics/numarray/template.slice.array/slice.arr.assign/slice_array.pass.cpp FAIL + +# STL bug: We allow fill() and swap() for array. +std/containers/sequences/array/array.fill/fill.fail.cpp FAIL +std/containers/sequences/array/array.swap/swap.fail.cpp FAIL + +# STL bug: VSO-207715 We reject array. +std/containers/sequences/array/array.cons/default.pass.cpp FAIL +std/containers/sequences/array/array.cons/implicit_copy.pass.cpp FAIL +std/containers/sequences/array/array.data/data_const.pass.cpp FAIL +std/containers/sequences/array/array.data/data.pass.cpp FAIL +std/containers/sequences/array/begin.pass.cpp FAIL + +# STL bug: string_view doesn't enforce "non-array trivial standard-layout", related to LWG-3034. +std/strings/string.view/char.bad.fail.cpp:0 FAIL + +# Predicate count assertions - IDL2 is slightly bending the Standard's rules here. +std/algorithms/alg.sorting/alg.heap.operations/make.heap/make_heap_comp.pass.cpp FAIL +std/algorithms/alg.sorting/alg.merge/inplace_merge_comp.pass.cpp FAIL +std/algorithms/alg.sorting/alg.min.max/minmax_init_list_comp.pass.cpp FAIL + +# STL bug: We don't match strtod / strtof when doing field extraction for hexfloats, or special cases like inf +std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp FAIL +std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp FAIL +std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp FAIL + +# STL bug: We don't match numpunct groups correctly in do_get +std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long.pass.cpp FAIL + +# STL test bug: We don't have the locale names libcxx wants specialized in platform_support.hpp +# More bugs may be uncovered when the locale names are present. +std/input.output/iostreams.base/ios/basic.ios.members/move.pass.cpp FAIL +std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_1.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_many.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct.byname/curr_symbol.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct.byname/decimal_point.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct.byname/neg_format.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct.byname/negative_sign.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct.byname/pos_format.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct.byname/thousands_sep.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get.byname/get_date_wide.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname_wide.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get.byname/get_one_wide.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday_wide.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.put.byname/put1.pass.cpp FAIL +std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp FAIL +std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp FAIL + +# STL Bug? Our wbuffer_convert does not implement seek. [depr.conversions.buffer] is completely underspecified. +std/localization/locales/locale.convenience/conversions/conversions.buffer/seekoff.pass.cpp FAIL + +# STL Bug: error_category's default ctor isn't constexpr. (Should be fixed in vNext.) +std/diagnostics/syserr/syserr.errcat/syserr.errcat.nonvirtuals/default_ctor.pass.cpp:1 FAIL + +# STL Bug: future incorrectly uses copy assignment instead of copy construction in set_value. (Should be fixed in vNext.) +std/thread/futures/futures.promise/set_value_const.pass.cpp FAIL + + +# *** CRT BUGS *** +# We're permanently missing aligned_alloc(). +std/depr/depr.c.headers/stdlib_h.pass.cpp SKIP +std/language.support/support.runtime/cstdlib.pass.cpp SKIP + +# OS-11107628 "_Exit allows cleanup in other DLLs" +std/thread/thread.threads/thread.thread.class/thread.thread.assign/move2.pass.cpp SKIP +std/thread/thread.threads/thread.thread.class/thread.thread.member/join.pass.cpp SKIP + + +# *** LIKELY BOGUS TESTS *** +# Test bug. See VSO-521345 " We're missing integral overloads for some math.h functions, including isfinite" +std/depr/depr.c.headers/math_h.pass.cpp SKIP +std/numerics/c.math/cmath.pass.cpp SKIP + +# Tests need to be updated for P1115R3 "erase()/erase_if() Return size_type". +std/containers/associative/map/map.erasure/erase_if.pass.cpp SKIP +std/containers/associative/multimap/multimap.erasure/erase_if.pass.cpp SKIP +std/containers/associative/multiset/multiset.erasure/erase_if.pass.cpp SKIP +std/containers/associative/set/set.erasure/erase_if.pass.cpp SKIP +std/containers/sequences/deque/deque.erasure/erase_if.pass.cpp SKIP +std/containers/sequences/deque/deque.erasure/erase.pass.cpp SKIP +std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp SKIP +std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp SKIP +std/containers/sequences/list/list.erasure/erase_if.pass.cpp SKIP +std/containers/sequences/list/list.erasure/erase.pass.cpp SKIP +std/containers/sequences/vector/vector.erasure/erase_if.pass.cpp SKIP +std/containers/sequences/vector/vector.erasure/erase.pass.cpp SKIP +std/containers/unord/unord.map/erase_if.pass.cpp SKIP +std/containers/unord/unord.multimap/erase_if.pass.cpp SKIP +std/containers/unord/unord.multiset/erase_if.pass.cpp SKIP +std/containers/unord/unord.set/erase_if.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/deque.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/forward_list.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/list.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/map.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/set.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/unordered_map.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/unordered_set.version.pass.cpp SKIP +std/language.support/support.limits/support.limits.general/vector.version.pass.cpp SKIP +std/strings/strings.erasure/erase_if.pass.cpp SKIP +std/strings/strings.erasure/erase.pass.cpp SKIP + +# Test needs to be updated for P1976R2 "Explicit Constructors For Fixed-Extent span From Dynamic-Extent Ranges". +std/containers/views/span.cons/assign.pass.cpp SKIP + +# Test bug after LWG-2899 "is_(nothrow_)move_constructible and tuple, optional and unique_ptr" was accepted. +std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.asgn/move_convert.pass.cpp SKIP +std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.asgn/move_convert.runtime.pass.cpp SKIP +std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.asgn/move_convert.single.pass.cpp SKIP +std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.asgn/move.pass.cpp SKIP + +# Test bug after LWG-3257 "Missing feature testing macro update from P0858" was accepted. +std/language.support/support.limits/support.limits.general/string.version.pass.cpp SKIP + +# Test needs to be updated for LWG-3320 removing span::const_iterator. +std/containers/views/types.pass.cpp SKIP +std/containers/views/span.iterators/begin.pass.cpp SKIP +std/containers/views/span.iterators/rbegin.pass.cpp SKIP +std/containers/views/span.iterators/end.pass.cpp SKIP +std/containers/views/span.iterators/rend.pass.cpp SKIP + +# Test needs to be updated for P2116R0 removing the tuple interface of span +std/containers/views/span.tuple/get.fail.cpp SKIP +std/containers/views/span.tuple/get.pass.cpp SKIP +std/containers/views/span.tuple/tuple_element.fail.cpp SKIP +std/containers/views/span.tuple/tuple_element.pass.cpp SKIP +std/containers/views/span.tuple/tuple_size.fail.cpp SKIP +std/containers/views/span.tuple/tuple_size.pass.cpp SKIP + +# Test bug. See LWG-3099 "is_assignable" +std/utilities/utility/pairs/pairs.pair/assign_pair.pass.cpp SKIP + +# Not yet analyzed, likely bogus tests. Appears to be timing assumptions. +std/thread/futures/futures.async/async.pass.cpp SKIP +std/thread/futures/futures.shared_future/get.pass.cpp SKIP +std/thread/futures/futures.shared_future/wait_for.pass.cpp SKIP +std/thread/futures/futures.unique_future/wait_for.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvar/notify_all.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvar/notify_one.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvar/wait_for_pred.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvar/wait_for.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvar/wait_until_pred.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvar/wait_until.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvarany/notify_all.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvarany/notify_one.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvarany/wait_for_pred.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvarany/wait_for.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvarany/wait_until_pred.pass.cpp SKIP +std/thread/thread.condition/thread.condition.condvarany/wait_until.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.guard/adopt_lock.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.guard/mutex.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_try_to_lock.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/lock.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_try_to_lock.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex.pass.cpp SKIP +std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/try_lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.recursive/lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.recursive/try_lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.class/lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.class/try_lock_for.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.class/try_lock_until.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.class/try_lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/lock.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/try_lock_for.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/try_lock_until.pass.cpp SKIP +std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/try_lock.pass.cpp SKIP +std/thread/thread.threads/thread.thread.class/thread.thread.destr/dtor.pass.cpp SKIP +std/thread/thread.threads/thread.thread.class/thread.thread.member/detach.pass.cpp SKIP +std/thread/thread.threads/thread.thread.this/sleep_until.pass.cpp SKIP + +# Not yet analyzed, likely bogus tests. Various assertions, probably POSIX assumptions. +std/diagnostics/syserr/syserr.compare/eq_error_code_error_code.pass.cpp SKIP +std/diagnostics/syserr/syserr.errcat/syserr.errcat.derived/message.pass.cpp SKIP +std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/system_category.pass.cpp SKIP +std/diagnostics/syserr/syserr.syserr/syserr.syserr.members/ctor_error_code_const_char_pointer.pass.cpp SKIP +std/diagnostics/syserr/syserr.syserr/syserr.syserr.members/ctor_error_code_string.pass.cpp SKIP +std/diagnostics/syserr/syserr.syserr/syserr.syserr.members/ctor_error_code.pass.cpp SKIP +std/diagnostics/syserr/syserr.syserr/syserr.syserr.members/ctor_int_error_category_const_char_pointer.pass.cpp SKIP +std/diagnostics/syserr/syserr.syserr/syserr.syserr.members/ctor_int_error_category_string.pass.cpp SKIP +std/diagnostics/syserr/syserr.syserr/syserr.syserr.members/ctor_int_error_category.pass.cpp SKIP + +# libc++ disagrees with libstdc++ and MSVC on whether setstate calls during I/O that throw set failbit; see open issue LWG-2349 +std/input.output/iostream.format/input.streams/istream.unformatted/get_pointer_size_chart.pass.cpp SKIP +std/input.output/iostream.format/input.streams/istream.unformatted/get_pointer_size.pass.cpp SKIP + +# Sensitive to implementation details. Assertion failed: test_alloc_base::count == expected_num_allocs +std/containers/container.requirements/container.requirements.general/allocator_move.pass.cpp SKIP + +# Tests std::weak_equality/strong_equality which were removed by P1959R0 +std/language.support/cmp/cmp.common/common_comparison_category.pass.cpp SKIP +std/language.support/cmp/cmp.partialord/partialord.pass.cpp SKIP +std/language.support/cmp/cmp.strongeq/cmp.strongeq.pass.cpp SKIP +std/language.support/cmp/cmp.strongord/strongord.pass.cpp SKIP +std/language.support/cmp/cmp.weakeq/cmp.weakeq.pass.cpp SKIP +std/language.support/cmp/cmp.weakord/weakord.pass.cpp SKIP + +# Comment: "Test C99 compound literal." +# Code: `(int[]){3, 4}` +# error C4576: a parenthesized type followed by an initializer list is a non-standard explicit type conversion syntax +std/containers/sequences/array/array.creation/to_array.pass.cpp SKIP + +# Tests that need to learn that insert iterators have non-void difference type in C++20 +std/iterators/predef.iterators/insert.iterators/back.insert.iterator/types.pass.cpp SKIP +std/iterators/predef.iterators/insert.iterators/front.insert.iterator/types.pass.cpp SKIP +std/iterators/predef.iterators/insert.iterators/insert.iterator/types.pass.cpp SKIP + + +# *** LIKELY STL BUGS *** +# Not yet analyzed, likely STL bugs. Assertions and other runtime failures. +std/numerics/rand/rand.dis/rand.dist.bern/rand.dist.bern.bin/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.bern/rand.dist.bern.bin/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.bern/rand.dist.bern.geo/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.bern/rand.dist.bern.geo/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.bern/rand.dist.bern.negbin/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.bern/rand.dist.bern.negbin/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.cauchy/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.cauchy/min.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.chisq/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.chisq/min.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.f/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.lognormal/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.lognormal/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.lognormal/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.lognormal/min.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.normal/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.normal/min.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.t/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.t/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.t/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.norm/rand.dist.norm.t/min.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.exp/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.extreme/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.extreme/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.extreme/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.extreme/min.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.gamma/eq.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.gamma/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.gamma/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.gamma/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.gamma/min.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.poisson/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.poisson/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.pois/rand.dist.pois.weibull/max.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.pconst/ctor_init_func.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.pconst/ctor_iterator.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.pconst/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.pconst/param_ctor_init_func.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.pconst/param_ctor_iterator.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/ctor_default.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/ctor_func.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/ctor_init_func.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/ctor_iterator.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/ctor_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/eval_param.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/eval.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/param_ctor_default.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/param_ctor_func.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/param_ctor_init_func.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.samp/rand.dist.samp.plinear/param_ctor_iterator.pass.cpp FAIL +std/numerics/rand/rand.dis/rand.dist.uni/rand.dist.uni.real/param_ctor.pass.cpp FAIL + +# Not yet analyzed, likely STL bugs. Various assertions. +std/re/re.alg/re.alg.match/awk.pass.cpp FAIL +std/re/re.alg/re.alg.match/basic.pass.cpp FAIL +std/re/re.alg/re.alg.match/ecma.pass.cpp FAIL +std/re/re.alg/re.alg.match/extended.pass.cpp FAIL +std/re/re.alg/re.alg.search/awk.pass.cpp FAIL +std/re/re.alg/re.alg.search/basic.pass.cpp FAIL +std/re/re.alg/re.alg.search/ecma.pass.cpp FAIL +std/re/re.alg/re.alg.search/extended.pass.cpp FAIL +std/re/re.alg/re.alg.search/no_update_pos.pass.cpp FAIL +std/re/re.badexp/regex_error.pass.cpp FAIL +std/re/re.const/re.synopt/syntax_option_type.pass.cpp FAIL +std/re/re.grammar/excessive_brace_count.pass.cpp FAIL +std/re/re.regex/re.regex.construct/bad_backref.pass.cpp FAIL +std/re/re.regex/re.regex.construct/bad_escape.pass.cpp FAIL +std/re/re.regex/re.regex.construct/bad_range.pass.cpp FAIL +std/re/re.regex/re.regex.construct/default.pass.cpp FAIL +std/re/re.regex/re.regex.nonmemb/re.regex.nmswap/swap.pass.cpp FAIL +std/re/re.regex/re.regex.swap/swap.pass.cpp FAIL +std/re/re.traits/lookup_collatename.pass.cpp FAIL +std/re/re.traits/transform_primary.pass.cpp FAIL + +# Not yet analyzed, likely STL bugs. Various assertions. +std/numerics/complex.number/complex.member.ops/divide_equal_complex.pass.cpp FAIL +std/numerics/complex.number/complex.ops/complex_divide_complex.pass.cpp FAIL +std/numerics/complex.number/complex.ops/complex_times_complex.pass.cpp FAIL +std/numerics/complex.number/complex.ops/scalar_divide_complex.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/acos.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/acosh.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/asin.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/asinh.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/atanh.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/cos.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/cosh.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/exp.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/log10.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/pow_complex_complex.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/pow_complex_scalar.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/pow_scalar_complex.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/sin.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/sinh.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/sqrt.pass.cpp FAIL +std/numerics/complex.number/complex.transcendentals/tanh.pass.cpp FAIL +std/numerics/complex.number/complex.value.ops/norm.pass.cpp FAIL +std/numerics/complex.number/complex.value.ops/polar.pass.cpp FAIL +std/numerics/complex.number/complex.value.ops/proj.pass.cpp FAIL + +# Not yet analyzed, likely STL bugs. Many various assertions. +std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.statics/classic_table.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_ru_RU.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_zh_CN.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_ru_RU.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_zh_CN.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct/locale.moneypunct.members/decimal_point.pass.cpp FAIL +std/localization/locale.categories/category.monetary/locale.moneypunct/locale.moneypunct.members/thousands_sep.pass.cpp FAIL +std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_double.pass.cpp FAIL +std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname_wide.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time_wide.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday_wide.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp FAIL +std/localization/locale.categories/category.time/locale.time.put/locale.time.put.members/put2.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf16_in.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf16_length.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf16_max_length.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf16_out.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf16.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8_in.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8_length.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8_max_length.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8_out.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8_utf16_in.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8_utf16_length.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8_utf16_max_length.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8_utf16_out.pass.cpp FAIL +std/localization/locale.stdcvt/codecvt_utf8.pass.cpp FAIL +std/localization/locales/locale.convenience/conversions/conversions.buffer/overflow.pass.cpp FAIL +std/localization/locales/locale.convenience/conversions/conversions.buffer/pbackfail.pass.cpp FAIL +std/localization/locales/locale.convenience/conversions/conversions.buffer/underflow.pass.cpp FAIL +std/localization/locales/locale.convenience/conversions/conversions.string/ctor_err_string.pass.cpp FAIL + +# Not yet analyzed, likely STL bugs. Various assertions. +std/input.output/iostream.format/ext.manip/get_money.pass.cpp FAIL +std/input.output/iostream.format/ext.manip/put_money.pass.cpp FAIL +std/input.output/iostreams.base/ios/basic.ios.members/copyfmt.pass.cpp FAIL + +# Not yet analyzed, likely STL bugs. Assertion failed: os.str() == a +std/numerics/rand/rand.adapt/rand.adapt.disc/ctor_result_type.pass.cpp FAIL +std/numerics/rand/rand.adapt/rand.adapt.disc/ctor_sseq.pass.cpp FAIL +std/numerics/rand/rand.adapt/rand.adapt.ibits/ctor_result_type.pass.cpp FAIL +std/numerics/rand/rand.adapt/rand.adapt.ibits/ctor_sseq.pass.cpp FAIL +std/numerics/rand/rand.eng/rand.eng.mers/ctor_result_type.pass.cpp FAIL +std/numerics/rand/rand.eng/rand.eng.mers/ctor_sseq.pass.cpp FAIL +std/numerics/rand/rand.eng/rand.eng.sub/ctor_result_type.pass.cpp FAIL +std/numerics/rand/rand.eng/rand.eng.sub/ctor_sseq.pass.cpp FAIL + +# Not yet analyzed, likely STL bugs. Assertion failed: e1 == e2 +std/numerics/rand/rand.adapt/rand.adapt.disc/io.pass.cpp FAIL +std/numerics/rand/rand.adapt/rand.adapt.ibits/io.pass.cpp FAIL +std/numerics/rand/rand.eng/rand.eng.sub/io.pass.cpp FAIL + +# Likely STL bug: Looks like we shouldn't be using assignment. +std/thread/futures/futures.promise/set_rvalue.pass.cpp FAIL + +# Possible STL bugs in pair and tuple. +std/utilities/tuple/tuple.tuple/tuple.cnstr/PR23256_constrain_UTypes_ctor.pass.cpp:0 FAIL +std/utilities/tuple/tuple.tuple/tuple.cnstr/PR31384.pass.cpp FAIL + +# Likely STL bugs in mersenne_twister; also fails at runtime +# random(1186,26): error: constexpr variable '_WMSK' must be initialized by a constant expression +# static constexpr _Ty _WMSK = ~((~_Ty(0) << (_Wx - 1)) << 1); +# ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +std/numerics/rand/rand.eng/rand.eng.mers/ctor_sseq_all_zero.pass.cpp FAIL + +# Bugs/questionable choices in codecvt, which we probably will not fix since +# (1) they are deprecated, and (2) we don't want to break existing users. +std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_max_length.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_unshift.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_encoding.pass.cpp FAIL +std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_max_length.pass.cpp FAIL + +# Likely STL bug in : "result type of conditional expression is ambiguous" +std/utilities/time/time.duration/time.duration.alg/abs.pass.cpp FAIL + + +# *** NOT YET ANALYZED *** +# Not yet analyzed. Asserting about alloc_count. +std/thread/futures/futures.promise/alloc_ctor.pass.cpp FAIL +std/thread/futures/futures.promise/move_assign.pass.cpp FAIL +std/thread/futures/futures.promise/move_ctor.pass.cpp FAIL +std/thread/futures/futures.promise/swap.pass.cpp FAIL +std/thread/futures/futures.shared_future/dtor.pass.cpp FAIL +std/thread/futures/futures.unique_future/dtor.pass.cpp FAIL + +# Not yet analyzed. libc++ seems to have a different opinion about what tuple_size should do. +std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.pass.cpp FAIL +std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_structured_bindings.pass.cpp FAIL + +# Not yet analyzed. Possibly testing nonstandard deduction guides. +std/containers/associative/map/map.cons/deduct_const.pass.cpp FAIL +std/containers/associative/multimap/multimap.cons/deduct_const.pass.cpp FAIL +std/containers/unord/unord.map/unord.map.cnstr/deduct_const.pass.cpp FAIL +std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.pass.cpp:0 FAIL +std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct_const.pass.cpp FAIL +std/utilities/tuple/tuple.tuple/tuple.cnstr/deduct.pass.cpp:0 FAIL + +# Not yet analyzed. Assertion failed: f16_8.out(mbs, c16, c_c16p, c_c16p, c8, c8+4, c8p) == F32_8::ok +std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/utf_sanity_check.pass.cpp FAIL + + +# *** XFAILs WHICH PASS *** +# Not yet implemented in libcxx and marked as XFAIL +std/strings/c.strings/cuchar.pass.cpp PASS +std/utilities/function.objects/func.search/func.search.bm/default.pass.cpp PASS +std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp PASS +std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp PASS +std/utilities/function.objects/func.search/func.search.bm/pred.pass.cpp PASS +std/utilities/function.objects/func.search/func.search.bmh/default.pass.cpp PASS +std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp PASS +std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp PASS +std/utilities/function.objects/func.search/func.search.bmh/pred.pass.cpp PASS diff --git a/tests/libcxx/lit.site.cfg.in b/tests/libcxx/lit.site.cfg.in new file mode 100644 index 00000000000..41cb71337fd --- /dev/null +++ b/tests/libcxx/lit.site.cfg.in @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import site +site.addsitedir("@STL_TEST_UTILS_DIR@") + +config.configuration_variant = "stl" +config.cxx_archive_root = "@CMAKE_ARCHIVE_OUTPUT_DIRECTORY@" +config.cxx_headers = "@STL_TESTED_HEADERS_DIR@" +config.cxx_library_root = "@CMAKE_LIBRARY_OUTPUT_DIRECTORY@" +config.cxx_runtime_root = "@CMAKE_RUNTIME_OUTPUT_DIRECTORY@" +config.envlst_path = "@LIBCXX_ENVLST@" +config.expected_results_list_path = "@LIBCXX_EXPECTED_RESULTS@" +config.format_name = "LibcxxTestFormat" +config.include_dirs = ["@LIBCXX_SOURCE_DIR@/test/support"] +config.libcxx_obj_root = "@LIBCXX_TEST_OUTPUT_DIR@" +config.msvc_toolset_libs_root = "@TOOLSET_LIB@" +config.stl_build_root = "@STL_BUILD_ROOT@" +config.stl_src_root = "@STL_SOURCE_DIR@" +config.target_arch = "@VCLIBS_TARGET_ARCHITECTURE@" +config.test_subdirs = ["@LIBCXX_SOURCE_DIR@/test/std"] + +config.loaded_site_config = True +lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg") + +config.test_exec_root = "@LIBCXX_TEST_OUTPUT_DIR@" diff --git a/tests/libcxx/skipped_tests.txt b/tests/libcxx/skipped_tests.txt index 44cf507b797..f05ff56d52d 100644 --- a/tests/libcxx/skipped_tests.txt +++ b/tests/libcxx/skipped_tests.txt @@ -730,6 +730,7 @@ utilities\utility\pairs\pairs.pair\assign_pair.pass.cpp # Not yet analyzed, likely bogus tests. Appears to be timing assumptions. thread\futures\futures.async\async.pass.cpp +thread\futures\futures.shared_future\get.pass.cpp thread\futures\futures.shared_future\wait_for.pass.cpp thread\futures\futures.unique_future\wait_for.pass.cpp thread\thread.condition\thread.condition.condvar\notify_all.pass.cpp diff --git a/tests/std/CMakeLists.txt b/tests/std/CMakeLists.txt new file mode 100644 index 00000000000..4a0a3781d71 --- /dev/null +++ b/tests/std/CMakeLists.txt @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set(STD_EXPECTED_RESULTS "${CMAKE_CURRENT_SOURCE_DIR}/expected_results.txt") +set(STD_TEST_OUTPUT_DIR "${STL_TEST_OUTPUT_DIR}/std") +set(STD_TEST_SUBDIRS_FILE "${CMAKE_CURRENT_SOURCE_DIR}/test.lst") +set(STD_TEST_SUBDIRS_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg) + +get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP) +string(APPEND LLVM_LIT_CONFIG_MAP "map_config(\"${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg\", \"${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg\")\n") +set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP ${LLVM_LIT_CONFIG_MAP}) + +if(ENABLE_XUNIT_OUTPUT) + list(APPEND STD_ADDITIONAL_LIT_FLAGS "--xunit-xml-output" "${CMAKE_CURRENT_BINARY_DIR}/std.test.xml") +endif() + +list(APPEND STD_LLVM_LIT_COMMAND "${LLVM_LIT_LOCATION}" + "${ADDITIONAL_LIT_FLAGS}" + "${STD_ADDITIONAL_LIT_FLAGS}" + "${CMAKE_CURRENT_BINARY_DIR}") + +add_test(NAME std COMMAND ${Python3_EXECUTABLE} ${STD_LLVM_LIT_COMMAND} COMMAND_EXPAND_LISTS) +set_tests_properties(std PROPERTIES RUN_SERIAL TRUE) diff --git a/tests/std/expected_results.txt b/tests/std/expected_results.txt new file mode 100644 index 00000000000..c46d6aec0cd --- /dev/null +++ b/tests/std/expected_results.txt @@ -0,0 +1,46 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# *** INSUFFICIENTLY NEW PUBLIC TOOLSET *** +# TRANSITION, VS 2019 16.6p2 +tests/P0768R1_spaceship_operator:00 FAIL +tests/P0768R1_spaceship_operator:01 FAIL +tests/P0768R1_spaceship_operator:02 FAIL +tests/P0768R1_spaceship_operator:03 FAIL +tests/P0768R1_spaceship_operator:04 FAIL +tests/P0768R1_spaceship_operator:05 FAIL +tests/P0768R1_spaceship_operator:06 FAIL +tests/P0768R1_spaceship_operator:07 FAIL +tests/P0768R1_spaceship_operator:08 FAIL +tests/P0768R1_spaceship_operator:09 FAIL +tests/P0768R1_spaceship_operator:10 FAIL +tests/P0768R1_spaceship_operator:11 FAIL +tests/P0768R1_spaceship_operator:12 FAIL +tests/P0768R1_spaceship_operator:13 FAIL +tests/P0768R1_spaceship_operator:14 FAIL + +# TRANSITION, VS 2019 16.6p2 +tests/VSO_0157762_feature_test_macros:03 FAIL +tests/VSO_0157762_feature_test_macros:04 FAIL +tests/VSO_0157762_feature_test_macros:05 FAIL +tests/VSO_0157762_feature_test_macros:06 FAIL +tests/VSO_0157762_feature_test_macros:10 FAIL +tests/VSO_0157762_feature_test_macros:11 FAIL +tests/VSO_0157762_feature_test_macros:12 FAIL +tests/VSO_0157762_feature_test_macros:13 FAIL +tests/VSO_0157762_feature_test_macros:14 FAIL +tests/VSO_0157762_feature_test_macros:15 FAIL +tests/VSO_0157762_feature_test_macros:16 FAIL +tests/VSO_0157762_feature_test_macros:17 FAIL +tests/VSO_0157762_feature_test_macros:18 FAIL +tests/VSO_0157762_feature_test_macros:23 FAIL +tests/VSO_0157762_feature_test_macros:24 FAIL +tests/VSO_0157762_feature_test_macros:25 FAIL +tests/VSO_0157762_feature_test_macros:29 FAIL +tests/VSO_0157762_feature_test_macros:30 FAIL +tests/VSO_0157762_feature_test_macros:31 FAIL +tests/VSO_0157762_feature_test_macros:32 FAIL +tests/VSO_0157762_feature_test_macros:33 FAIL +tests/VSO_0157762_feature_test_macros:34 FAIL +tests/VSO_0157762_feature_test_macros:35 FAIL +tests/VSO_0157762_feature_test_macros:36 FAIL diff --git a/tests/std/lit.cfg b/tests/std/lit.cfg new file mode 100644 index 00000000000..fc5389e0361 --- /dev/null +++ b/tests/std/lit.cfg @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from pathlib import Path +import importlib +import site + +site.addsitedir(str(Path(__file__).parents[1] / "utils")) +site.addsitedir(str(Path(__file__).parent / "tests")) + +config.name = 'std' +config.suffixes.add('test.cpp') + +if config.test_source_root is None: + config.test_source_root = str(Path(__file__).parent) + +config.test_exec_root = getattr(config, 'test_exec_root', None) + +if not config.test_exec_root: + import tempfile + config.test_exec_root = tempfile.mkdtemp(prefix=config.name + '-testsuite-') + lit_config.note('Creating temporary directory for tests: %s' % config.test_exec_root) + +config_module_name = getattr(config, 'config_module_name', 'stl.test.config') +config_module = importlib.import_module(config_module_name) + +configuration = config_module.Configuration(lit_config, config) +configuration.configure() +config.test_format = configuration.get_test_format() diff --git a/tests/std/lit.site.cfg.in b/tests/std/lit.site.cfg.in new file mode 100644 index 00000000000..e3998e65f93 --- /dev/null +++ b/tests/std/lit.site.cfg.in @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +config.cxx_archive_root = "@CMAKE_ARCHIVE_OUTPUT_DIRECTORY@" +config.cxx_headers = "@STL_TESTED_HEADERS_DIR@" +config.cxx_library_root = "@CMAKE_LIBRARY_OUTPUT_DIRECTORY@" +config.cxx_runtime_root = "@CMAKE_RUNTIME_OUTPUT_DIRECTORY@" +config.expected_results_list_path = "@STD_EXPECTED_RESULTS@" +config.include_dirs = ["@LIBCXX_SOURCE_DIR@/test/support", "@STL_SOURCE_DIR@/tests/std/include"] +config.msvc_toolset_libs_root = "@TOOLSET_LIB@" +config.stl_build_root = "@STL_BUILD_ROOT@" +config.stl_src_root = "@STL_SOURCE_DIR@" +config.target_arch = "@VCLIBS_TARGET_ARCHITECTURE@" +config.test_exec_root = "@STD_TEST_OUTPUT_DIR@" +config.test_subdirs_file = "@STD_TEST_SUBDIRS_FILE@" +config.test_subdirs_root = "@STD_TEST_SUBDIRS_ROOT@" + +lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg") diff --git a/tests/std/tests/Dev09_056375_locale_cleanup/__init__.py b/tests/std/tests/Dev09_056375_locale_cleanup/__init__.py new file mode 100644 index 00000000000..2ac2a854cb0 --- /dev/null +++ b/tests/std/tests/Dev09_056375_locale_cleanup/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/tests/std/tests/Dev09_056375_locale_cleanup/custom_format.py b/tests/std/tests/Dev09_056375_locale_cleanup/custom_format.py new file mode 100644 index 00000000000..9d7c64eeaec --- /dev/null +++ b/tests/std/tests/Dev09_056375_locale_cleanup/custom_format.py @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from pathlib import Path + +from stl.test.format import STLTestFormat, TestStep + + +class CustomTestFormat(STLTestFormat): + def getBuildSteps(self, test, lit_config, shared): + exe_source = Path(test.getSourcePath()) + dll_source = exe_source.parent / 'TestDll.cpp' + shared.exec_dir = test.getExecDir() + output_base = test.getOutputBaseName() + output_dir = test.getOutputDir() + pass_var, fail_var = test.getPassFailResultCodes() + dll_output = output_dir / 'TestDll.DLL' + + dll_compile_cmd, out_files, exec_file = \ + test.cxx.executeBasedOnFlagsCmd([dll_source], output_dir, + shared.exec_dir, 'TestDll', + ['/Fe' + str(dll_output)], + [], ['/DLL']) + + shared.dll_file = dll_output + + yield TestStep(dll_compile_cmd, shared.exec_dir, [dll_source], + test.cxx.compile_env) + + exe_compile_cmd, out_files, shared.exec_file = \ + test.cxx.executeBasedOnFlagsCmd([exe_source], output_dir, + shared.exec_dir, output_base, + [], [], []) + + yield TestStep(exe_compile_cmd, shared.exec_dir, [exe_source], + test.cxx.compile_env) + + def getTestSteps(self, test, lit_config, shared): + if shared.exec_file is not None: + yield TestStep([str(shared.exec_file)], shared.exec_dir, + [shared.exec_file, shared.dll_file], + test.cxx.compile_env) diff --git a/tests/std/tests/Dev09_056375_locale_cleanup/lit.local.cfg b/tests/std/tests/Dev09_056375_locale_cleanup/lit.local.cfg new file mode 100644 index 00000000000..5a7c740f058 --- /dev/null +++ b/tests/std/tests/Dev09_056375_locale_cleanup/lit.local.cfg @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import Dev09_056375_locale_cleanup.custom_format + +config.test_format = \ + Dev09_056375_locale_cleanup.custom_format.CustomTestFormat(config.test_format.cxx, + config.test_format.execute_external, + config.test_format.build_executor, + config.test_format.test_executor) diff --git a/tests/std/tests/GH_000545_include_compare/__init__.py b/tests/std/tests/GH_000545_include_compare/__init__.py new file mode 100644 index 00000000000..2ac2a854cb0 --- /dev/null +++ b/tests/std/tests/GH_000545_include_compare/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/tests/std/tests/GH_000545_include_compare/custom_format.py b/tests/std/tests/GH_000545_include_compare/custom_format.py new file mode 100644 index 00000000000..05c32016fde --- /dev/null +++ b/tests/std/tests/GH_000545_include_compare/custom_format.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from pathlib import Path +import os + +from stl.test.format import STLTestFormat, TestStep + + +class CustomTestFormat(STLTestFormat): + def getBuildSteps(self, test, lit_config, shared): + shared.exec_dir = test.getExecDir() + output_base = test.getOutputBaseName() + output_dir = test.getOutputDir() + exe_source_dir = Path(test.getSourcePath()).parent + + source_files = [] + for filename in os.listdir(exe_source_dir): + if filename.endswith('.cpp'): + source_files.append(exe_source_dir / filename) + + cmd, out_files, shared.exec_file = \ + test.cxx.executeBasedOnFlagsCmd(source_files, output_dir, + shared.exec_dir, output_base, + [], [], []) + + yield TestStep(cmd, shared.exec_dir, source_files, + test.cxx.compile_env) diff --git a/tests/std/tests/GH_000545_include_compare/lit.local.cfg b/tests/std/tests/GH_000545_include_compare/lit.local.cfg new file mode 100644 index 00000000000..cbdcadad3d6 --- /dev/null +++ b/tests/std/tests/GH_000545_include_compare/lit.local.cfg @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import GH_000545_include_compare.custom_format + +config.test_format = \ + GH_000545_include_compare.custom_format.CustomTestFormat(config.test_format.cxx, + config.test_format.execute_external, + config.test_format.build_executor, + config.test_format.test_executor) diff --git a/tests/std/tests/P0607R0_inline_variables/__init__.py b/tests/std/tests/P0607R0_inline_variables/__init__.py new file mode 100644 index 00000000000..2ac2a854cb0 --- /dev/null +++ b/tests/std/tests/P0607R0_inline_variables/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/tests/std/tests/P0607R0_inline_variables/custom_format.py b/tests/std/tests/P0607R0_inline_variables/custom_format.py new file mode 100644 index 00000000000..323790ec7cb --- /dev/null +++ b/tests/std/tests/P0607R0_inline_variables/custom_format.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from pathlib import Path + +from stl.test.format import STLTestFormat, TestStep + + +class CustomTestFormat(STLTestFormat): + def getBuildSteps(self, test, lit_config, shared): + shared.exec_dir = test.getExecDir() + exe_source = Path(test.getSourcePath()) + test2_source = exe_source.parent / 'test2.cpp' + output_base = test.getOutputBaseName() + output_dir = test.getOutputDir() + + cmd, out_files, shared.exec_file = \ + test.cxx.executeBasedOnFlagsCmd([exe_source, test2_source], + output_dir, shared.exec_dir, + output_base, [], [], []) + + yield TestStep(cmd, shared.exec_dir, [exe_source, test2_source], + test.cxx.compile_env) diff --git a/tests/std/tests/P0607R0_inline_variables/lit.local.cfg b/tests/std/tests/P0607R0_inline_variables/lit.local.cfg new file mode 100644 index 00000000000..55628eacde9 --- /dev/null +++ b/tests/std/tests/P0607R0_inline_variables/lit.local.cfg @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import P0607R0_inline_variables.custom_format + +config.test_format = \ + P0607R0_inline_variables.custom_format.CustomTestFormat(config.test_format.cxx, + config.test_format.execute_external, + config.test_format.build_executor, + config.test_format.test_executor) diff --git a/tests/std/tests/VSO_0000000_any_calling_conventions/__init__.py b/tests/std/tests/VSO_0000000_any_calling_conventions/__init__.py new file mode 100644 index 00000000000..2ac2a854cb0 --- /dev/null +++ b/tests/std/tests/VSO_0000000_any_calling_conventions/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/tests/std/tests/VSO_0000000_any_calling_conventions/custom_format.py b/tests/std/tests/VSO_0000000_any_calling_conventions/custom_format.py new file mode 100644 index 00000000000..07b7dff9cfe --- /dev/null +++ b/tests/std/tests/VSO_0000000_any_calling_conventions/custom_format.py @@ -0,0 +1,52 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from pathlib import Path + +from stl.test.format import STLTestFormat, TestStep +from stl.test.tests import STLTest + + +class CustomTest(STLTest): + def __init__(self, suite, path_in_suite, lit_config, test_config, + envlst_entry, env_num, default_cxx, file_path=None): + STLTest.__init__(self, suite, path_in_suite, lit_config, test_config, + envlst_entry, env_num, default_cxx, file_path) + self.calling_convention_a = \ + envlst_entry.getEnvVal('CALLING_CONVENTION_A') + self.calling_convention_b = \ + envlst_entry.getEnvVal('CALLING_CONVENTION_B') + + +class CustomTestFormat(STLTestFormat): + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig, test_class=CustomTest): + return super().getTestsInDirectory(testSuite, path_in_suite, litConfig, + localConfig, test_class) + + def getBuildSteps(self, test, lit_config, shared): + shared.exec_dir = test.getExecDir() + exe_source = Path(test.getSourcePath()) + a_source = exe_source.parent / 'a.cpp' + output_base = test.getOutputBaseName() + output_dir = test.getOutputDir() + + a_compile_cmd, out_files, exec_file = \ + test.cxx.executeBasedOnFlagsCmd([a_source], output_dir, + shared.exec_dir, 'a', + [test.calling_convention_a, '/c'], + [], []) + + yield TestStep(a_compile_cmd, shared.exec_dir, [a_source], + test.cxx.compile_env) + + a_output = output_dir / 'a.obj' + + exe_compile_cmd, out_files, shared.exec_file = \ + test.cxx.executeBasedOnFlagsCmd([exe_source], output_dir, + shared.exec_dir, output_base, + [test.calling_convention_b, + str(a_output)], [], []) + + yield TestStep(exe_compile_cmd, shared.exec_dir, + [a_output, exe_source], test.cxx.compile_env) diff --git a/tests/std/tests/VSO_0000000_any_calling_conventions/lit.local.cfg b/tests/std/tests/VSO_0000000_any_calling_conventions/lit.local.cfg new file mode 100644 index 00000000000..9f4ac1b16d8 --- /dev/null +++ b/tests/std/tests/VSO_0000000_any_calling_conventions/lit.local.cfg @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +config.suffixes = {'b.cpp'} + +import VSO_0000000_any_calling_conventions.custom_format + +config.test_format = \ + VSO_0000000_any_calling_conventions.custom_format.CustomTestFormat(config.test_format.cxx, + config.test_format.execute_external, + config.test_format.build_executor, + config.test_format.test_executor) diff --git a/tests/utils/stl/__init__.py b/tests/utils/stl/__init__.py new file mode 100644 index 00000000000..2ac2a854cb0 --- /dev/null +++ b/tests/utils/stl/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/tests/utils/stl/compiler.py b/tests/utils/stl/compiler.py new file mode 100644 index 00000000000..6a638046938 --- /dev/null +++ b/tests/utils/stl/compiler.py @@ -0,0 +1,210 @@ +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +from itertools import chain +from pathlib import Path +from typing import List +import os + +import stl.util + + +class CXXCompiler: + CM_Default = 0 + CM_PreProcess = 1 + CM_Compile = 2 + CM_Link = 3 + CM_Analyze = 4 + + def __init__(self, path, flags=None, compile_flags=None, + link_flags=None, compile_env=None): + self.path = path + if path is not None: + self.name = os.path.basename(path).split('.')[0] + else: + self.name = None + + self.compile_flags = compile_flags or [] + self.flags = flags or [] + self.link_flags = link_flags or [] + + self.compile_env = compile_env + + def _basicCmd(self, source_files: List[Path], out: Path, + mode=CM_Default, flags=[], compile_flags=[], link_flags=[], + skip_mode_flags=False): + out_files = [] + cmd = [] + + if out is not None: + out_files.append(out) + + cmd.append(self.path) + + if mode == self.CM_PreProcess: + if out is not None: + cmd.extend(('/P', '/Fi' + str(out))) + else: + cmd.append('/EP') + elif mode == self.CM_Compile: + if not skip_mode_flags: + cmd.append('/c') + if out is not None and len(source_files) <= 1: + cmd.append('/Fo' + str(out)) + else: + for source_file in source_files: + out_files.append( + Path(source_file.name.rsplit('.', 1)[0] + '.obj')) + elif mode == self.CM_Analyze: + if not skip_mode_flags: + cmd.append('/analyze:only') + if out is not None: + cmd.append('/analyze:log' + str(out)) + else: + for source_file in source_files: + out_files.append( + Path(source_file.name.rsplit('.', 1)[0] + + '.nativecodeanalysis.xml')) + elif out is not None: + cmd.append('/Fe' + str(out)) + + if len(source_files) <= 1: + out_obj = str(out).rsplit('.', 1)[0] + '.obj' + cmd.append('/Fo' + out_obj) + out_files.append(Path(out_obj)) + else: + for source_file in source_files: + out_files.append( + Path(source_file.name.rsplit('.', 1)[0] + '.obj')) + + if mode in (self.CM_Analyze, self.CM_Compile, self.CM_Default): + cmd.extend(self.compile_flags) + cmd.extend(compile_flags) + + cmd.extend(self.flags) + cmd.extend(flags) + cmd.extend([str(file) for file in source_files]) + + if mode in (self.CM_Default, self.CM_Link): + cmd.append('/link') + cmd.extend(self.link_flags) + cmd.extend(link_flags) + + return cmd, out_files + + def executeBasedOnFlagsCmd(self, source_files, out_dir, cwd=None, + out_base=None, flags=[], compile_flags=[], + link_flags=[]): + mode = self.CM_Default + exec_file = None + output_file = None + + if out_base is None: + out_base = source_files[0].name.rsplit('.', 1)[0] + + add_analyze_output = False + for flag in chain(flags, self.flags): + flag = flag[1:] + + if flag == 'c': + mode = self.CM_Compile + exec_file = None + output_file = out_dir / (out_base + '.obj') + elif flag.startswith('Fe'): + output_file = Path(flag[2:]) + exec_file = output_file + elif flag == 'analyze:only': + mode = self.CM_Analyze + output_file = out_dir / (out_base + '.nativecodeanalysis.xml') + exec_file = None + elif flag.startswith('analyze') and flag[-1] != '-': + add_analyze_output = True + + if mode is self.CM_Default and output_file is None: + output_file = out_dir / (out_base + '.exe') + exec_file = output_file + + if add_analyze_output and mode != self.CM_Analyze: + flags.append('/analyze:log' + + str(out_dir / (out_base + '.nativecodeanalysis.xml'))) + + cmd, out_files = self._basicCmd(source_files, output_file, mode, flags, + compile_flags, link_flags, True) + return cmd, out_files, exec_file + + def executeBasedOnFlags(self, source_files, out_dir, cwd=None, + out_base=None, flags=[], compile_flags=[], + link_flags=[]): + cmd, out_files, exec_file = \ + self.executeBasedOnFlagsCmd(source_files, out_dir, cwd, out_base, + flags, compile_flags, link_flags) + out, err, rc = stl.util.executeCommand(cmd, env=self.compile_env, + cwd=cwd) + return cmd, out, err, rc, out_files, exec_file + + def preprocessCmd(self, source_files, out=None, flags=[]): + return self._basicCmd(source_files, out, flags=flags, compile_flags=[], + link_flags=[], mode=self.CM_PreProcess) + + def compileCmd(self, source_files, out=None, flags=[]): + return self._basicCmd(source_files, out, flags=flags, compile_flags=[], + link_flags=[], mode=self.CM_Compile) + + def linkCmd(self, source_files, out=None, flags=[]): + return self._basicCmd(source_files, out, flags=flags, compile_flags=[], + link_flags=[], mode=self.CM_Link) + + def compileLinkCmd(self, source_files, out=None, flags=[]): + return self._basicCmd(source_files, out, flags=flags, compile_flags=[], + link_flags=[]) + + def preprocess(self, source_files, out=None, flags=[], cwd=None): + cmd, _ = self.preprocessCmd(source_files, out, flags) + out, err, rc = stl.util.executeCommand(cmd, env=self.compile_env, + cwd=cwd) + return cmd, out, err, rc + + def compile(self, source_files, out=None, flags=[], cwd=None): + cmd, _ = self.compileCmd(source_files, out, flags) + out, err, rc = stl.util.executeCommand(cmd, env=self.compile_env, + cwd=cwd) + return cmd, out, err, rc + + def link(self, source_files, exec_path=None, flags=[], cwd=None): + cmd, _ = self.linkCmd(source_files, exec_path, flags) + out, err, rc = stl.util.executeCommand(cmd, env=self.compile_env, + cwd=cwd) + return cmd, out, err, rc + + def compileLink(self, source_files, exec_path=None, flags=[], + cwd=None): + cmd, _ = self.compileLinkCmd(source_files, exec_path, flags) + out, err, rc = stl.util.executeCommand(cmd, env=self.compile_env, + cwd=cwd) + return cmd, out, err, rc + + def compileLinkTwoSteps(self, source_file, out=None, object_file=None, + flags=[], cwd=None): + if not isinstance(source_file, str): + raise TypeError('This function only accepts a single input file') + if object_file is None: + # Create, use, and delete a temporary object file if none is given. + def with_fn(): return stl.util.guardedTempFilename(suffix='.obj') + else: + # Otherwise wrap the filename in a context manager function. + def with_fn(): return stl.util.nullContext(object_file) + with with_fn() as object_file: + cc_cmd, cc_stdout, cc_stderr, rc = self.compile( + source_file, object_file, flags=flags, cwd=cwd) + if rc != 0: + return cc_cmd, cc_stdout, cc_stderr, rc + + link_cmd, link_stdout, link_stderr, rc = self.link( + object_file, exec_path=out, flags=flags, cwd=cwd) + return (cc_cmd + ['&'] + link_cmd, cc_stdout + link_stdout, + cc_stderr + link_stderr, rc) diff --git a/tests/utils/stl/test/__init__.py b/tests/utils/stl/test/__init__.py new file mode 100644 index 00000000000..2ac2a854cb0 --- /dev/null +++ b/tests/utils/stl/test/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/tests/utils/stl/test/config.py b/tests/utils/stl/test/config.py new file mode 100644 index 00000000000..87f20a393a5 --- /dev/null +++ b/tests/utils/stl/test/config.py @@ -0,0 +1,425 @@ +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +from pathlib import Path +import os +import platform +import shlex + +from stl.test.executor import LocalExecutor +from stl.compiler import CXXCompiler +import stl.util +import stl.test.file_parsing +import stl.test.target_info + + +# Extract the value of a numeric macro such as __cplusplus or a feature-test +# macro. +def intMacroValue(token): + return int(token.rstrip('LlUu')) + + +class Configuration: + # pylint: disable=redefined-outer-name + def __init__(self, lit_config, config): + self.build_executor = None + self.config = config + self.cxx_headers = None + self.cxx_library_root = None + self.cxx_runtime_root = None + self.default_compiler = None + self.execute_external = False + self.expected_results_list_path = None + self.expected_results_list_root = None + self.format_name = None + self.lit_config = lit_config + self.link_shared = True + self.long_tests = None + self.msvc_toolset_libs_root = None + self.stl_build_root = None + self.stl_inc_env_var = None + self.stl_lib_env_var = None + self.stl_path_env_var = None + self.stl_src_root = None + self.stl_test_env = None + self.target_arch = None + self.target_info = stl.test.target_info.WindowsLocalTI(lit_config) + self.test_executor = None + self.test_source_root = None + + def get_lit_conf(self, name, default=None): + val = self.lit_config.params.get(name, None) + if val is None: + val = getattr(self.config, name, None) + if val is None: + val = default + return val + + def get_lit_bool(self, name, default=None, env_var=None): + def check_value(value, var_name): + if value is None: + return default + if isinstance(value, bool): + return value + if not isinstance(value, str): + raise TypeError('expected bool or string') + if value.lower() in ('1', 'true'): + return True + if value.lower() in ('', '0', 'false'): + return False + self.lit_config.fatal( + "parameter '{}' should be true or false".format(var_name)) + + conf_val = self.get_lit_conf(name) + if env_var is not None and env_var in os.environ and \ + os.environ[env_var] is not None: + val = os.environ[env_var] + if conf_val is not None: + self.lit_config.warning( + 'Environment variable %s=%s is overriding explicit ' + '--param=%s=%s' % (env_var, val, name, conf_val)) + return check_value(val, env_var) + return check_value(conf_val, name) + + def configure(self): + self.configure_features() + self.configure_default_compiler() + self.configure_executors() + self.configure_expected_results() + self.configure_test_dirs() + self.configure_test_format() + + def configure_test_format(self): + format_name = self.get_lit_conf('format_name', None) + + if format_name is None: + format_name = 'STLTestFormat' + + self.format_name = format_name + + def configure_test_dirs(self): + test_subdirs = list() + + test_subdirs_file = self.get_lit_conf('test_subdirs_file', None) + + if test_subdirs_file is not None: + test_subdirs_file = Path(test_subdirs_file) + test_subdirs += \ + stl.test.file_parsing.parse_commented_file(test_subdirs_file) + + test_subdirs += self.get_lit_conf('test_subdirs', []) + + test_subdirs_root = Path(self.get_lit_conf('test_subdirs_root', '')) + self.lit_config.test_subdirs = \ + list(map(lambda x: str((test_subdirs_root / x).resolve()), + test_subdirs)) + + # TRANSITION: Don't hard-code features. + def configure_features(self): + self.config.available_features.add('long_tests') + self.config.available_features.add('c++2a') + self.config.available_features.add('msvc') + self.config.available_features.update(self.target_info.features) + + def configure_test_source_root(self): + test_source_root = self.get_lit_conf('test_source_root', None) + + if test_source_root is not None: + self.test_source_root = Path(test_source_root) + + def configure_src_root(self): + stl_src_root = self.get_lit_conf('stl_src_root', None) + + if stl_src_root is None: + if self.test_source_root is None: + self.configure_test_source_root() + + if self.test_source_root is None: + self.lit_config.fatal( + "Could not infer stl_src_root from test_source_root. " + "test_source_root is None") + + if self.config.name == 'libc++': + stl_src_root = self.test_source_root.parents[2] + else: + stl_src_root = self.test_source_root.parents[1] + + self.lit_config.note( + "stl_src_root was not specified. Defaulting to: %s." % + str(stl_src_root)) + + self.stl_src_root = Path(stl_src_root) + + def configure_target_architecture(self): + target_arch = self.get_lit_conf('target_arch', None) + + if target_arch is None: + target_arch = platform.machine() + + self.lit_config.note( + "target_arch was not specified. Defaulting to %s." % + target_arch) + + self.target_arch = target_arch + self.config.target_arch = target_arch + + def configure_build_root(self): + stl_build_root = self.get_lit_conf('stl_build_root', None) + + if stl_build_root is None: + if self.stl_src_root is None: + self.configure_src_root() + + if self.target_arch is None: + self.configure_target_architecture() + + stl_build_root = self.stl_src_root / 'out' / 'build' + + if self.target_arch.casefold() == 'AMD64'.casefold(): + stl_build_root = stl_build_root / 'x64' / 'out' + elif self.target_arch.casefold() == 'X86'.casefold(): + stl_build_root = stl_build_root / 'i386' / 'out' + else: + stl_build_root = \ + stl_build_root / self.target_arch.upper() / 'out' + + self.lit_config.note( + "stl_build_root was not specified. Defaulting to: %s." % + str(stl_build_root)) + + self.stl_build_root = Path(stl_build_root) + + def configure_library_roots(self): + cxx_runtime_root = self.get_lit_conf('cxx_runtime_root', None) + cxx_library_root = self.get_lit_conf('cxx_library_root', None) + + if cxx_runtime_root is None or cxx_library_root is None: + if self.stl_build_root is None: + self.configure_build_root() + + if self.target_arch is None: + self.configure_target_architecture() + + subfolder_name = None + if self.target_arch.casefold() == 'AMD64'.casefold(): + subfolder_name = 'amd64' + elif self.target_arch.casefold() == 'X86'.casefold(): + subfolder_name = 'i386' + else: + subfolder_name = self.target_arch.lower() + + if cxx_runtime_root is None: + cxx_runtime_root = self.stl_build_root / 'bin' / subfolder_name + + self.lit_config.note( + "cxx_runtime_root was not specified. Defaulting to: %s." % + str(cxx_runtime_root)) + + if cxx_library_root is None: + cxx_library_root = \ + self.stl_build_root / 'lib' / subfolder_name + + self.lit_config.note( + "cxx_library_root was not specified. Defaulting to: %s." + % str(cxx_library_root)) + + self.cxx_runtime_root = Path(cxx_runtime_root) + self.cxx_library_root = Path(cxx_library_root) + + def configure_inc_env_var(self): + stl_inc_env_var = self.get_lit_conf('stl_inc_env_var', None) + + if stl_inc_env_var is None: + stl_inc_env_var = self.config.environment.get('INCLUDE', '') + + if self.cxx_headers is None: + self.configure_cxx_headers() + + include_dirs = self.get_lit_conf('include_dirs', []) + + if stl_inc_env_var != '': + stl_inc_env_var = ';'.join((str(self.cxx_headers), + *include_dirs, + stl_inc_env_var)) + else: + stl_inc_env_var = ';'.join((str(self.cxx_headers), + *include_dirs)) + + self.stl_inc_env_var = stl_inc_env_var + + # Note: This relies on kernel32.lib and ucrt.lib being in the LIB env var + def configure_lib_env_var(self): + stl_lib_env_var = self.get_lit_conf('stl_lib_env_var', None) + + if stl_lib_env_var is None: + stl_lib_env_var = self.config.environment.get('LIB', '') + + if stl_lib_env_var != '': + stl_lib_env_var = ';'.join((str(self.cxx_library_root), + stl_lib_env_var)) + else: + stl_lib_env_var = str(self.cxx_library_root) + + self.stl_lib_env_var = stl_lib_env_var + + def configure_path_env_var(self): + stl_path_env_var = self.get_lit_conf('stl_path_env_var', None) + + if stl_path_env_var is None: + path_list = list() + + msvc_bin_dir = self.get_lit_conf('msvc_bin_dir', None) + if msvc_bin_dir is not None: + path_list.append(msvc_bin_dir) + + if self.cxx_runtime_root is None: + self.configure_library_roots() + path_list.append(str(self.cxx_runtime_root)) + + config_env = self.config.environment.get('PATH', None) + if config_env is not None: + path_list.append(config_env) + + stl_path_env_var = ';'.join(path_list) + + self.stl_path_env_var = stl_path_env_var + + def configure_test_env(self): + stl_test_env = self.get_lit_conf('stl_test_env', None) + + if stl_test_env is None: + stl_test_env = self.config.environment + + if self.stl_lib_env_var is None: + self.configure_inc_env_var() + + if self.stl_lib_env_var is None: + self.configure_lib_env_var() + + if self.stl_path_env_var is None: + self.configure_path_env_var() + + stl_test_env['INCLUDE'] = self.stl_inc_env_var + stl_test_env['LIB'] = self.stl_lib_env_var + stl_test_env['PATH'] = self.stl_path_env_var + + self.config.environment = stl_test_env + + def configure_expected_results_list_location(self): + expected_results_list_path = self.get_lit_conf( + 'expected_results_list_path', None) + + if expected_results_list_path is not None: + self.expected_results_list_path = Path( + expected_results_list_path) + else: + self.expected_results_list_path = Path(os.devnull) + + def configure_expected_results(self): + expected_results = getattr(self.lit_config, 'expected_results', dict()) + + if self.expected_results_list_path is None: + self.configure_expected_results_list_location() + + expected_results[self.config.name] = \ + stl.test.file_parsing.parse_result_file( + self.expected_results_list_path) + + self.lit_config.expected_results = expected_results + self.config.expected_results = \ + getattr(self.config, 'expected_results', dict()) + + def configure_default_compiler(self): + self.default_compiler = CXXCompiler(None) + self.configure_compile_flags() + self.configure_link_flags() + self.configure_test_env() + + self.default_compiler.compile_env = self.config.environment + + # TRANSITION: Investigate using SSHExecutor for ARM + def configure_executors(self): + self.build_executor = LocalExecutor() + self.test_executor = LocalExecutor() + + def configure_compile_flags(self): + self.configure_compile_flags_header_includes() + # Configure extra flags + self.default_compiler.compile_flags += \ + self.get_lit_conf('compile_flags', []) + additional_flags_str = self.get_lit_conf('addtional_compiler_flags') + if additional_flags_str: + self.default_compiler.compile_flags += \ + shlex.split(additional_flags_str) + + def configure_compile_flags_header_includes(self): + if self.cxx_headers is None: + self.configure_cxx_headers() + + self.default_compiler.compile_flags += ['/I' + str(self.cxx_headers)] + + include_dirs = self.get_lit_conf('include_dirs', []) + + for directory in include_dirs: + self.default_compiler.compile_flags.append('/I' + directory) + + def configure_cxx_headers(self): + cxx_headers = self.get_lit_conf('cxx_headers') + + if cxx_headers is None: + if self.stl_build_root is None: + self.configure_build_root() + + cxx_headers = self.stl_build_root / 'inc' + + if not os.path.isdir(cxx_headers): + self.lit_config.fatal("cxx_headers='%s' is not a directory." + % str(cxx_headers)) + else: + cxx_headers = Path(cxx_headers) + + self.cxx_headers = cxx_headers + + def configure_link_flags(self): + if self.cxx_library_root is None: + self.configure_library_roots() + + if self.msvc_toolset_libs_root is None: + self.configure_msvc_toolset_libs_root() + + self.default_compiler.link_flags.append( + '/LIBPATH:' + str(self.cxx_library_root)) + + self.default_compiler.link_flags.append( + '/LIBPATH:' + str(self.msvc_toolset_libs_root)) + + additional_flags_str = self.get_lit_conf('additional_link_flags') + if additional_flags_str: + self.default_compiler.link_flags += \ + shlex.split(additional_flags_str) + + def configure_msvc_toolset_libs_root(self): + msvc_toolset_libs_root = self.get_lit_conf('msvc_toolset_libs_root') + + if msvc_toolset_libs_root is None: + self.lit_config.fatal('msvc_toolset_libs_root must be specified') + + self.msvc_toolset_libs_root = Path(msvc_toolset_libs_root) + + def get_test_format(self): + import stl.test.format + + return getattr(stl.test.format, self.format_name)( + self.default_compiler, + self.execute_external, + self.build_executor, + self.test_executor) + + # TRANSITION: Might be nice to actually print something + def print_config_info(self): + pass diff --git a/tests/utils/stl/test/executor.py b/tests/utils/stl/test/executor.py new file mode 100644 index 00000000000..cee2b71bf7f --- /dev/null +++ b/tests/utils/stl/test/executor.py @@ -0,0 +1,235 @@ +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +import platform +import os +import posixpath +import ntpath + +from stl.test import tracing +from stl.util import executeCommand + + +class Executor: + def __init__(self): + self.target_info = None + + def run(self, cmd, local_cwd, file_deps=None, env=None): + """Execute a command. + Be very careful not to change shared state in this function. + Executor objects are shared between python processes in `lit -jN`. + Args: + cmd: [str]: subprocess.call style command + local_cwd: str: Local path to the working directory + file_deps: [str]: Files required by the cmd + env: {str: str}: Environment variables to execute under + Returns: + cmd, out, err, exitCode + """ + raise NotImplementedError + + def merge_environments(self, current_env, updated_env): + """Merges two execution environments. + + If both environments contain the PATH variables, they are also merged + using the proper separator. + """ + result_env = dict(current_env) + for k, v in updated_env.items(): + if k == 'PATH' and self.target_info: + self.target_info.add_path(result_env, v) + else: + result_env[k] = v + return result_env + + +class LocalExecutor(Executor): + def __init__(self): + super(LocalExecutor, self).__init__() + self.is_windows = platform.system() == 'Windows' + + def run(self, cmd, work_dir='.', file_deps=None, env=None): + if str(work_dir) == '.': + work_dir = os.getcwd() + + if env: + env = self.merge_environments(os.environ, env) + + out, err, rc = executeCommand(cmd, cwd=work_dir, env=env) + return (cmd, out, err, rc) + + +class PrefixExecutor(Executor): + """Prefix an executor with some other command wrapper. + + Most useful for setting ulimits on commands, or running an emulator like + qemu and valgrind. + """ + def __init__(self, commandPrefix, chain): + super(PrefixExecutor, self).__init__() + + self.commandPrefix = commandPrefix + self.chain = chain + + def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None): + cmd = cmd or [exe_path] + return self.chain.run(exe_path, self.commandPrefix + cmd, work_dir, + file_deps, env=env) + + +class PostfixExecutor(Executor): + """Postfix an executor with some args.""" + def __init__(self, commandPostfix, chain): + super(PostfixExecutor, self).__init__() + + self.commandPostfix = commandPostfix + self.chain = chain + + def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None): + cmd = cmd or [exe_path] + return self.chain.run(cmd + self.commandPostfix, work_dir, file_deps, + env=env) + + +class RemoteExecutor(Executor): + def __init__(self): + super(RemoteExecutor, self).__init__() + self.local_run = executeCommand + + def remote_temp_dir(self): + return self._remote_temp(True) + + def remote_temp_file(self): + return self._remote_temp(False) + + def _remote_temp(self, is_dir): + raise NotImplementedError() + + def copy_in(self, local_srcs, remote_dsts): + # This could be wrapped up in a tar->scp->untar for performance + # if there are lots of files to be copied/moved + for src, dst in zip(local_srcs, remote_dsts): + self._copy_in_file(src, dst) + + def _copy_in_file(self, src, dst): + raise NotImplementedError() + + def delete_remote(self, remote): + try: + self._execute_command_remote(['rm', '-rf', remote]) + except OSError: + # TRANSITION: Log failure to delete? + pass + + def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None): + target_exe_path = None + target_cwd = None + try: + target_cwd = self.remote_temp_dir() + executable_name = 'libcxx_test.exe' + if self.target_info.is_windows(): + target_exe_path = ntpath.join(target_cwd, executable_name) + else: + target_exe_path = posixpath.join(target_cwd, executable_name) + + if cmd: + # Replace exe_path with target_exe_path. + cmd = [c if c != exe_path else target_exe_path for c in cmd] + else: + cmd = [target_exe_path] + + srcs = [exe_path] + dsts = [target_exe_path] + if file_deps is not None: + dev_paths = [os.path.join(target_cwd, os.path.basename(f)) + for f in file_deps] + srcs.extend(file_deps) + dsts.extend(dev_paths) + self.copy_in(srcs, dsts) + + # When testing executables that were cross-compiled on Windows for + # Linux, we may need to explicitly set the execution permission to + # avoid the 'Permission denied' error: + chmod_cmd = ['chmod', '+x', target_exe_path] + + return self._execute_command_remote(chmod_cmd + ['&&'] + cmd, + target_cwd, + env) + finally: + if target_cwd: + self.delete_remote(target_cwd) + + def _execute_command_remote(self, cmd, remote_work_dir='.', env=None): + raise NotImplementedError() + + +class SSHExecutor(RemoteExecutor): + def __init__(self, host, username=None): + super(SSHExecutor, self).__init__() + + self.user_prefix = username + '@' if username else '' + self.host = host + self.scp_command = 'scp' + self.ssh_command = 'ssh' + + if False: + self.local_run = tracing.trace_function( + self.local_run, log_calls=True, log_results=True, + label='ssh_local') + + def _remote_temp(self, is_dir): + # TRANSITION: detect what the target system is, and use the correct + # mktemp command for it. (linux and darwin differ here, and I'm + # sure windows has another way to do it) + + # Not sure how to do suffix on osx yet + dir_arg = '-d' if is_dir else '' + cmd = 'mktemp -q {} /tmp/stl.XXXXXXXXXX'.format(dir_arg) + _, temp_path, err, exitCode = self._execute_command_remote([cmd]) + temp_path = temp_path.strip() + if exitCode != 0: + raise RuntimeError(err) + return temp_path + + def _copy_in_file(self, src, dst): + scp = self.scp_command + remote = self.host + remote = self.user_prefix + remote + cmd = [scp, '-p', src, remote + ':' + dst] + self.local_run(cmd) + + def _export_command(self, env): + if not env: + return [] + + export_cmd = ['export'] + + for k, v in env.items(): + v = v.replace('\\', '\\\\') + if k == 'PATH': + # Pick up the existing paths, so we don't lose any commands + if self.target_info and self.target_info.is_windows(): + export_cmd.append('PATH="%s;%PATH%"' % v) + else: + export_cmd.append('PATH="%s:$PATH"' % v) + else: + export_cmd.append('"%s"="%s"' % (k, v)) + + return export_cmd + + def _execute_command_remote(self, cmd, remote_work_dir='.', env=None): + remote = self.user_prefix + self.host + ssh_cmd = [self.ssh_command, '-oBatchMode=yes', remote] + export_cmd = self._export_command(env) + remote_cmd = ' '.join(cmd) + if export_cmd: + remote_cmd = ' '.join(export_cmd) + ' && ' + remote_cmd + if remote_work_dir != '.': + remote_cmd = 'cd ' + remote_work_dir + ' && ' + remote_cmd + out, err, rc = self.local_run(ssh_cmd + [remote_cmd]) + return (remote_cmd, out, err, rc) diff --git a/tests/utils/stl/test/file_parsing.py b/tests/utils/stl/test/file_parsing.py new file mode 100644 index 00000000000..9a1308b9d5a --- /dev/null +++ b/tests/utils/stl/test/file_parsing.py @@ -0,0 +1,143 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from collections import namedtuple +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple, Union +import itertools +import os +import re + +import lit.Test + +import stl.test.tests + +_envlst_cache = dict() +_preprocessed_file_cache = dict() +_expected_result_entry_cache = dict() + + +@dataclass +class _TmpEnvEntry: + env: Dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class EnvEntry: + def __init__(self, tmp_env: _TmpEnvEntry): + object.__setattr__(self, "_env_keys", tuple(tmp_env.env.keys())) + object.__setattr__(self, "_env_vals", tuple(tmp_env.env.values())) + + def getEnvVal(self, key: str, default: Optional[str] = None) \ + -> Optional[str]: + if key not in self._env_keys: + return default + + # TRANSITION: All of this is to avoid having to install frozendict. + # Reconsider this at a future date. + return self._env_vals[self._env_keys.index(key)] + + _env_keys: Tuple[str] + _env_vals: Tuple[str] + + +@dataclass +class _ParseCtx: + current: List[_TmpEnvEntry] = field(default_factory=list) + result: List[List[_TmpEnvEntry]] = field(default_factory=list) + + +_COMMENT_REGEX = re.compile(r"\s*#.*", re.DOTALL) +_INCLUDE_REGEX = re.compile(r'^RUNALL_INCLUDE (?P.+$)') +_ENV_VAR_MULTI_ITEM_REGEX = re.compile(r'(?P\w+)="(?P.*?)"') +_CROSSLIST_REGEX = re.compile(r'^RUNALL_CROSSLIST$') +_EXPECTED_RESULT_REGEX = re.compile(r'^(?P.*) (?P.*?)$') + + +def _parse_env_line(line: str) -> Optional[_TmpEnvEntry]: + result = _TmpEnvEntry() + for env_match in _ENV_VAR_MULTI_ITEM_REGEX.finditer(line): + name = env_match.group("name") + value = env_match.group("value") + result.env[env_match.group("name")] = env_match.group("value") + return result + + +def _append_env_entries(*args) -> _TmpEnvEntry: + result = _TmpEnvEntry() + for entry in args: + for k, v in entry.env.items(): + if k not in result.env: + result.env[k] = v + else: + result.env[k] = ' '.join((result.env[k], v)) + return result + + +def _do_crosslist(ctx: _ParseCtx): + return itertools.starmap(_append_env_entries, + itertools.product(*ctx.result)) + + +def _parse_env_lst(env_lst: Path, ctx: _ParseCtx): + for line in parse_commented_file(env_lst): + if (m:=_INCLUDE_REGEX.match(line)) is not None: + p = env_lst.parent / Path(m.group("filename")) + _parse_env_lst(p, ctx) + elif _CROSSLIST_REGEX.match(line) is not None: + ctx.result.append(ctx.current) + ctx.current = [] + else: + ctx.current.append(_parse_env_line(line)) + + +def parse_commented_file(filename: Union[str, bytes, os.PathLike]) \ + -> List[str]: + if str(filename) in _preprocessed_file_cache: + return _preprocessed_file_cache[str(filename)] + + filename_path = Path(filename) + result = list() + with filename_path.open() as f: + for line in f.readlines(): + if (line:=_COMMENT_REGEX.sub("", line)): + line = line.strip() + if line: + result.append(line) + + _preprocessed_file_cache[str(filename)] = result + return result + + +def parse_result_file(filename: Union[str, bytes, os.PathLike]) \ + -> Dict[str, lit.Test.ResultCode]: + if str(filename) in _expected_result_entry_cache: + return _expected_result_entry_cache[str(filename)] + + res = dict() + for line in parse_commented_file(filename): + m = _EXPECTED_RESULT_REGEX.match(line) + prefix = m.group("prefix") + result = m.group("result") + result_code = getattr(lit.Test, result, None) + if result_code is None: + result_code = getattr(stl.test.tests, result) + res[prefix] = result_code + + _expected_result_entry_cache[str(filename)] = res + return res + + +def parse_env_lst_file(env_list: Union[str, bytes, os.PathLike]) \ + -> Tuple[EnvEntry, ...]: + if str(env_list) in _envlst_cache: + return _envlst_cache[str(env_list)] + + env_list_path = Path(env_list) + ctx = _ParseCtx() + _parse_env_lst(env_list_path, ctx) + ctx.result.append(ctx.current) + res = tuple(map(EnvEntry, _do_crosslist(ctx))) + _envlst_cache[env_list] = res + return res diff --git a/tests/utils/stl/test/format.py b/tests/utils/stl/test/format.py new file mode 100644 index 00000000000..717fb926c9c --- /dev/null +++ b/tests/utils/stl/test/format.py @@ -0,0 +1,285 @@ +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional +import copy +import itertools +import errno +import os +import shutil +import time + +import lit.Test # pylint: disable=import-error +import lit.TestRunner # pylint: disable=import-error + +from stl.test.tests import STLTest, LibcxxTest +import stl.test.file_parsing +import stl.util + + +@dataclass +class TestStep: + cmd: List[str] = field(default_factory=list) + work_dir: os.PathLike = field(default=Path('.')) + file_deps: List[os.PathLike] = field(default_factory=list) + env: Dict[str, str] = field(default_factory=dict) + should_fail: bool = field(default=False) + + +class STLTestFormat: + """ + Custom test format handler to run MSVC tests. + """ + + def __init__(self, default_cxx, execute_external, + build_executor, test_executor): + self.cxx = default_cxx + self.execute_external = execute_external + self.build_executor = build_executor + self.test_executor = test_executor + + def isLegalDirectory(self, source_path, litConfig): + found = False + for prefix in getattr(litConfig, 'test_subdirs', []): + if os.path.commonpath((source_path, prefix)) == prefix or \ + os.path.commonpath((prefix, source_path)) == source_path: + found = True + break + + return found + + def getEnvLst(self, source_path, localConfig): + envlst_path = getattr(localConfig, 'envlst_path', None) + if envlst_path is None: + cwd = Path(source_path) + if (cwd / 'env.lst').is_file(): + envlst_path = cwd / 'env.lst' + else: + for parent in cwd.parents: + if (parent / 'env.lst').is_file(): + envlst_path = parent / 'env.lst' + break + + return envlst_path + + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig, test_class=STLTest): + source_path = testSuite.getSourcePath(path_in_suite) + + if not self.isLegalDirectory(source_path, litConfig): + return + + envlst_path = self.getEnvLst(source_path, localConfig) + + for filename in os.listdir(source_path): + # Ignore dot files and excluded tests. + filepath = os.path.join(source_path, filename) + if filename.startswith('.'): + continue + + if not os.path.isdir(filepath): + if any([filename.endswith(ext) + for ext in localConfig.suffixes]): + + if envlst_path is None: + litConfig.fatal("Could not find an env.lst file.") + + env_entries = \ + stl.test.file_parsing.parse_env_lst_file(envlst_path) + format_string = "{:0" + str(len(str(len(env_entries)))) + \ + "d}" + for env_entry, env_num \ + in zip(env_entries, itertools.count()): + test_config = copy.deepcopy(localConfig) + test_path_in_suite = path_in_suite + (filename,) + + yield test_class(testSuite, + test_path_in_suite, + litConfig, test_config, + env_entry, + format_string.format(env_num), + self.cxx) + + def setup(self, test): + exec_dir = test.getExecDir() + output_dir = test.getOutputDir() + source_dir = Path(test.getSourcePath()).parent + + shutil.rmtree(exec_dir, ignore_errors=True) + shutil.rmtree(output_dir, ignore_errors=True) + exec_dir.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + + # TRANSITION: This should be handled by a TestStep with a dependency + # on the .dat files the test requires. + for path in source_dir.iterdir(): + if path.is_file() and path.name.endswith('.dat'): + os.link(path, exec_dir / path.name) + + def cleanup(self, test): + shutil.rmtree(test.getExecDir(), ignore_errors=True) + shutil.rmtree(test.getOutputDir(), ignore_errors=True) + + def getIntegratedScriptResult(self, test, lit_config): + if test.skipped: + return (stl.test.tests.SKIP, "Test was marked as skipped") + + name = test.path_in_suite[-1] + name_root, name_ext = os.path.splitext(name) + is_sh_test = name_root.endswith('.sh') + is_objcxx_test = name.endswith('.mm') + + if is_sh_test: + return (lit.Test.UNSUPPORTED, + "Sh tests are currently unsupported") + + if is_objcxx_test: + return (lit.Test.UNSUPPORTED, + "Objective-C tests are unsupported") + + if test.config.unsupported: + return (lit.Test.UNSUPPORTED, + "A lit.local.cfg marked this unsupported") + + if lit_config.noExecute: + return lit.Test.Result(lit.Test.PASS) + + if test.expected_result is None: + script = lit.TestRunner.parseIntegratedTestScript( + test, require_script=False) + + if isinstance(script, lit.Test.Result): + return script + + return None + + def execute(self, test, lit_config): + result = None + while True: + try: + result = self._execute(test, lit_config) + break + except OSError as oe: + if oe.errno != errno.ETXTBSY: + raise + time.sleep(0.1) + + return result + + def _execute(self, test, lit_config): + # TRANSITION: It is potentially wasteful that all the skipping and + # unsupported logic lives here when it is known at time of test + # discovery. Investigate + script_result = self.getIntegratedScriptResult(test, lit_config) + if script_result is not None: + return script_result + + try: + self.setup(test) + pass_var, fail_var = test.getPassFailResultCodes() + buildSteps, testSteps = self.getSteps(test, lit_config) + + report = "" + for step in buildSteps: + cmd, out, err, rc = \ + self.build_executor.run(step.cmd, step.work_dir, + step.file_deps, step.env) + + if step.should_fail and rc == 0: + report += "Build step succeeded unexpectedly.\n" + elif rc != 0: + report += "Build step failed unexpectedly.\n" + + report += stl.util.makeReport(cmd, out, err, rc) + if (step.should_fail and rc == 0) or \ + (not step.should_fail and rc != 0): + lit_config.note(report) + return lit.Test.Result(fail_var, report) + + for step in testSteps: + cmd, out, err, rc = \ + self.test_executor.run(step.cmd, step.work_dir, + step.file_deps, step.env) + + if step.should_fail and rc == 0: + report += "Test step succeeded unexpectedly.\n" + elif rc != 0: + report += "Test step failed unexpectedly.\n" + + report += stl.util.makeReport(cmd, out, err, rc) + if (step.should_fail and rc == 0) or \ + (not step.should_fail and rc != 0): + lit_config.note(report) + return lit.Test.Result(fail_var, report) + + return lit.Test.Result(pass_var, report) + + except Exception as e: + lit_config.warning(str(e)) + raise e + finally: + self.cleanup(test) + + def getSteps(self, test, lit_config): + @dataclass + class SharedState: + exec_file: Optional[os.PathLike] = field(default=None) + exec_dir: os.PathLike = field(default_factory=Path) + + shared = SharedState() + return self.getBuildSteps(test, lit_config, shared), \ + self.getTestSteps(test, lit_config, shared) + + def getBuildSteps(self, test, lit_config, shared): + if not test.path_in_suite[-1].endswith('.fail.cpp'): + shared.exec_dir = test.getExecDir() + output_base = test.getOutputBaseName() + output_dir = test.getOutputDir() + source_path = Path(test.getSourcePath()) + + cmd, out_files, shared.exec_file = \ + test.cxx.executeBasedOnFlagsCmd([source_path], output_dir, + shared.exec_dir, output_base, + [], [], []) + + yield TestStep(cmd, shared.exec_dir, [source_path], + test.cxx.compile_env) + + def getTestSteps(self, test, lit_config, shared): + if shared.exec_file is not None: + yield TestStep([str(shared.exec_file)], shared.exec_dir, + [shared.exec_file], test.cxx.compile_env) + elif test.path_in_suite[-1].endswith('.fail.cpp'): + exec_dir = test.getExecDir() + source_path = Path(test.getSourcePath()) + + flags = [] + if test.cxx.name == 'cl' and \ + ('/analyze' in test.cxx.flags or + '/analyze' in test.cxx.compile_flags): + output_base = test.getOutputBaseName() + output_dir = test.getOutputDir() + analyze_path = output_dir / (output_base + + '.nativecodeanalysis.xml') + flags.append('/analyze:log' + str(analyze_path)) + + cmd, _ = test.cxx.compileCmd([source_path], os.devnull, flags) + yield TestStep(cmd, exec_dir, [source_path], + test.cxx.compile_env, True) + + +class LibcxxTestFormat(STLTestFormat): + """ + Custom test format handler to run the libcxx tests for the MSVC STL. + """ + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig, test_class=LibcxxTest): + return super().getTestsInDirectory(testSuite, path_in_suite, litConfig, + localConfig, test_class) diff --git a/tests/utils/stl/test/target_info.py b/tests/utils/stl/test/target_info.py new file mode 100644 index 00000000000..137e2a78cae --- /dev/null +++ b/tests/utils/stl/test/target_info.py @@ -0,0 +1,45 @@ +#===----------------------------------------------------------------------===// +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===// + +import locale + + +def _test_locale(loc): + assert loc is not None + default_locale = locale.setlocale(locale.LC_ALL) + try: + locale.setlocale(locale.LC_ALL, loc) + return True + except locale.Error: + return False + finally: + locale.setlocale(locale.LC_ALL, default_locale) + + +class WindowsLocalTI: + def __init__(self, lit_config): + self.features = set() + self._add_common_locales(lit_config) + + def _add_common_locales(self, lit_config): + locales = [ + ('en_US.UTF-8', 'English_United States.1252'), + ('fr_FR.UTF-8', 'French_France.1252'), + ('ru_RU.UTF-8', 'Russian_Russia.1251'), + ('zh_CN.UTF-8', 'Chinese_China.936'), + ('fr_CA.ISO8859-1', 'French_Canada.1252'), + ('cs_CZ.ISO8859-2', 'Czech_Czech Republic.1250') + ] + + for loc_id, loc_name in locales: + if _test_locale(loc_name): + self.features.add('locale.{0}'.format(loc_id)) + else: + lit_config.warning('The locale {0} is not supported by ' + 'your platform. Some tests will be ' + 'unsupported.'.format(loc_name)) diff --git a/tests/utils/stl/test/tests.py b/tests/utils/stl/test/tests.py new file mode 100644 index 00000000000..053479e9cea --- /dev/null +++ b/tests/utils/stl/test/tests.py @@ -0,0 +1,217 @@ +# Copyright (c) Microsoft Corporation. +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +from itertools import chain +from pathlib import Path +from xml.sax.saxutils import quoteattr +import os +import shutil + +from lit.Test import FAIL, PASS, ResultCode, Test, UNSUPPORTED, XPASS, XFAIL + +from stl.compiler import CXXCompiler + +_compiler_path_cache = dict() + +SKIP = ResultCode('SKIP', False) + + +class STLTest(Test): + def __init__(self, suite, path_in_suite, lit_config, test_config, + envlst_entry, env_num, default_cxx, file_path=None): + self.env_num = env_num + self.skipped = False + Test.__init__(self, suite, path_in_suite, test_config, file_path) + + self._configure_expected_result(suite, path_in_suite, lit_config, + test_config, env_num) + if self.skipped: + return + + self._configure_cxx(lit_config, envlst_entry, default_cxx) + + # TRANSITION: These configurations should be enabled in the future. + for flag in chain(self.cxx.flags, self.cxx.compile_flags): + if flag.startswith('clr:pure', 1): + self.requires.append('clr_pure') + elif flag.startswith('BE', 1): + self.requires.append('edg') + + def getOutputDir(self): + return Path(os.path.join( + self.suite.getExecPath(self.path_in_suite[:-1]))) / self.env_num + + def getOutputBaseName(self): + return self.path_in_suite[-2] + + def getExecDir(self): + return self.getOutputDir() + + def getExecPath(self): + return self.getExecDir() / (self.getOutputBaseName() + '.exe') + + def getTestName(self): + return '/'.join(self.path_in_suite[:-1]) + ":" + self.env_num + + def getFullName(self): + return self.suite.config.name + ' :: ' + self.getTestName() + + def getPassFailResultCodes(self): + should_fail = self.isExpectedToFail() + pass_var = XPASS if should_fail else PASS + fail_var = XFAIL if should_fail else FAIL + + return pass_var, fail_var + + def getXMLOutputTestName(self): + return ':'.join((self.path_in_suite[-2], self.env_num)) + + def getXMLOutputClassName(self): + safe_test_path = [x.replace(".", "_") for x in self.path_in_suite[:-1]] + safe_suite_name = self.suite.name.replace(".", "-") + + if safe_test_path: + return safe_suite_name + "." + "/".join(safe_test_path) + else: + return safe_suite_name + "." + safe_suite_name + + def _configure_expected_result(self, suite, path_in_suite, lit_config, + test_config, env_num): + test_name = self.getTestName() + self.expected_result = None + + current_prefix = "" + for prefix, result in \ + chain(test_config.expected_results.items(), + lit_config.expected_results.get(test_config.name, + dict()).items()): + if test_name == prefix: + self.expected_result = result + break + elif test_name.startswith(prefix) and \ + len(prefix) > len(current_prefix): + current_prefix = prefix + self.expected_result = result + + if test_name in test_config.expected_results: + self.expected_result = test_config.expected_results[test_name] + elif test_name in lit_config.expected_results: + self.expected_result = lit_config.expected_results[test_name] + + if self.expected_result is not None: + if self.expected_result == SKIP: + self.skipped = True + elif self.expected_result.isFailure: + self.xfails = ['*'] + + def _configure_cxx(self, lit_config, envlst_entry, default_cxx): + env_compiler = envlst_entry.getEnvVal('PM_COMPILER', 'cl') + + if not os.path.isfile(env_compiler): + cxx = _compiler_path_cache.get(env_compiler, None) + + if cxx is None: + search_paths = self.config.environment['PATH'] + cxx = shutil.which(env_compiler, path=search_paths) + _compiler_path_cache[env_compiler] = cxx + + if not cxx: + lit_config.fatal('Could not find: %r' % env_compiler) + + flags = list() + compile_flags = list() + link_flags = list() + + flags.extend(default_cxx.flags or []) + compile_flags.extend(default_cxx.compile_flags or []) + link_flags.extend(default_cxx.link_flags or []) + + flags.extend(envlst_entry.getEnvVal('PM_CL', '').split()) + link_flags.extend(envlst_entry.getEnvVal('PM_LINK', '').split()) + + if ('clang'.casefold() in os.path.basename(cxx).casefold()): + target_arch = self.config.target_arch.casefold() + if (target_arch == 'x64'.casefold()): + compile_flags.append('-m64') + elif (target_arch == 'x86'.casefold()): + compile_flags.append('-m32') + + self.cxx = CXXCompiler(cxx, flags, compile_flags, link_flags, + default_cxx.compile_env) + + # This is partially lifted from lit's Test class. The changes here are to + # handle skipped tests, our env.lst format, and different naming schemes. + def writeJUnitXML(self, fil): + """Write the test's report xml representation to a file handle.""" + test_name = quoteattr(self.getXMLOutputTestName()) + class_name = quoteattr(self.getXMLOutputClassName()) + + testcase_template = \ + '\n\t", "]]]]>")) + fil.write("]]>\n") + elif self.result.code == UNSUPPORTED: + unsupported_features = self.getMissingRequiredFeatures() + if unsupported_features: + skip_message = \ + "Skipping because of: " + ", ".join(unsupported_features) + else: + skip_message = "Skipping because of configuration." + skip_message = quoteattr(skip_message) + + fil.write( + ">\n\t\n".format( + skip_message)) + elif self.result.code == SKIP: + message = quoteattr('Test is explicitly marked as skipped') + fil.write(">\n\t\n".format( + message)) + else: + fil.write("/>") + + +class LibcxxTest(STLTest): + def getOutputBaseName(self): + output_base = self.path_in_suite[-1] + + if output_base.endswith('.cpp'): + return output_base[:-4] + else: + return output_base + + def getOutputDir(self): + dir_name = self.path_in_suite[-1] + if dir_name.endswith('.cpp'): + dir_name = dir_name[:-4] + + return Path(os.path.join( + self.suite.getExecPath(self.path_in_suite[:-1]))) / dir_name / \ + self.env_num + + def getXMLOutputTestName(self): + return ':'.join((self.path_in_suite[-1], self.env_num)) + + def getTestName(self): + return '/'.join(self.path_in_suite) + ':' + self.env_num diff --git a/tests/utils/stl/test/tracing.py b/tests/utils/stl/test/tracing.py new file mode 100644 index 00000000000..8871c7ef638 --- /dev/null +++ b/tests/utils/stl/test/tracing.py @@ -0,0 +1,41 @@ +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +import inspect + + +def trace_function(function, log_calls, log_results, label=''): + def wrapper(*args, **kwargs): + kwarg_strs = ['{}={}'.format(k, v) for (k, v) in kwargs] + arg_str = ', '.join([str(a) for a in args] + kwarg_strs) + call_str = '{}({})'.format(function.__name__, arg_str) + + # Perform the call itself, logging before, after, and anything thrown. + try: + if log_calls: + print('{}: Calling {}'.format(label, call_str)) + res = function(*args, **kwargs) + if log_results: + print('{}: {} -> {}'.format(label, call_str, res)) + return res + except Exception as ex: + if log_results: + print('{}: {} raised {}'.format(label, call_str, type(ex))) + raise ex + + return wrapper + + +def trace_object(obj, log_calls, log_results, label=''): + for name, member in inspect.getmembers(obj): + if inspect.ismethod(member): + # Skip meta-functions, decorate everything else + if not member.__name__.startswith('__'): + setattr(obj, name, trace_function(member, log_calls, + log_results, label)) + return obj diff --git a/tests/utils/stl/util.py b/tests/utils/stl/util.py new file mode 100644 index 00000000000..98e52193a96 --- /dev/null +++ b/tests/utils/stl/util.py @@ -0,0 +1,181 @@ +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +from contextlib import contextmanager +from pathlib import Path +import os +import platform +import signal +import subprocess +import sys +import tempfile +import threading + + +@contextmanager +def guardedTempFilename(suffix='', prefix='', dir=None): + # Creates and yields a temporary filename within a with statement. The file + # is removed upon scope exit. + handle, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir) + os.close(handle) + yield name + Path(name).unlink(True) + + +@contextmanager +def guardedFilename(name): + # Yields a filename within a with statement. The file is removed upon scope + # exit. + yield name + Path(name).unlink(True) + + +@contextmanager +def nullContext(value): + # Yields a variable within a with statement. No action is taken upon scope + # exit. + yield value + + +def makeReport(cmd, out, err, rc): + report = "Command: %s\n" % ' '.join(cmd) + report += "Exit Code: %d\n" % rc + if out: + report += "Standard Output:\n--\n%s--\n" % out + if err: + report += "Standard Error:\n--\n%s--\n" % err + report += '\n' + return report + + +class ExecuteCommandTimeoutException(Exception): + def __init__(self, msg, out, err, exitCode): + assert isinstance(msg, str) + assert isinstance(out, str) + assert isinstance(err, str) + assert isinstance(exitCode, int) + self.msg = msg + self.out = out + self.err = err + self.exitCode = exitCode + + +# Close extra file handles on UNIX (on Windows this cannot be done while +# also redirecting input). +kUseCloseFDs = not (platform.system() == 'Windows') + + +def executeCommand(command, cwd=None, env=None, input=None, timeout=0): + """ + Execute command ``command`` (list of arguments or string) + with + * working directory ``cwd`` (str), use None to use the current + working directory + * environment ``env`` (dict), use None for none + * Input to the command ``input`` (str), use string to pass + no input. + * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. + + Returns a tuple (out, err, exitCode) where + * ``out`` (str) is the standard output of running the command + * ``err`` (str) is the standard error of running the command + * ``exitCode`` (int) is the exitCode of running the command + + If the timeout is hit an ``ExecuteCommandTimeoutException`` + is raised. + """ + if input is not None: + input = input.encode() + + p = subprocess.Popen(command, cwd=cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, close_fds=kUseCloseFDs) + timerObject = None + hitTimeOut = False + try: + if timeout > 0: + def killProcess(): + # We may be invoking a shell so we need to kill the + # process and all its children. + nonlocal hitTimeOut + hitTimeOut = True + killProcessAndChildren(p.pid) + + timerObject = threading.Timer(timeout, killProcess) + timerObject.start() + + out, err = p.communicate(input=input) + exitCode = p.wait() + finally: + if timerObject is not None: + timerObject.cancel() + + # Ensure the resulting output is always of string type. + out = out.decode() + err = err.decode() + + if hitTimeOut: + raise ExecuteCommandTimeoutException( + msg='Reached timeout of {} seconds'.format(timeout), + out=out, + err=err, + exitCode=exitCode + ) + + # Detect Ctrl-C in subprocess. + if exitCode == -signal.SIGINT: + raise KeyboardInterrupt + + return out, err, exitCode + + +def killProcessAndChildren(pid): + """ + This function kills a process with ``pid`` and all its + running children (recursively). It is currently implemented + using the psutil module which provides a simple platform + neutral implementation. + + TRANSITION: Jobify this + """ + if platform.system() == 'AIX': + subprocess.call('kill -kill $(ps -o pid= -L{})'.format(pid), + shell=True) + else: + import psutil + try: + psutilProc = psutil.Process(pid) + # Handle the different psutil API versions + try: + # psutil >= 2.x + children_iterator = psutilProc.children(recursive=True) + except AttributeError: + # psutil 1.x + children_iterator = psutilProc.get_children(recursive=True) + for child in children_iterator: + try: + child.kill() + except psutil.NoSuchProcess: + pass + psutilProc.kill() + except psutil.NoSuchProcess: + pass + + +def executeCommandVerbose(cmd, *args, **kwargs): + """ + Execute a command and print its output on failure. + """ + out, err, exitCode = executeCommand(cmd, *args, **kwargs) + if exitCode != 0: + report = makeReport(cmd, out, err, exitCode) + report += "\n\nFailed!" + sys.stderr.write('%s\n' % report) + return out, err, exitCode diff --git a/tools/validate/validate.cpp b/tools/validate/validate.cpp index 97fe232f434..c6f6aacbb8f 100644 --- a/tools/validate/validate.cpp +++ b/tools/validate/validate.cpp @@ -156,6 +156,7 @@ int main() { ".git"sv, ".vs"sv, ".vscode"sv, + "__pycache__"sv, "llvm-project"sv, "out"sv, "vcpkg"sv,