diff --git a/CMakeLists.txt b/CMakeLists.txt index 97adae96212..7c7dee67a91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,6 +88,7 @@ set(TOOLSET_LIB "${TOOLSET_ROOT_DIR}/lib/${VCLIBS_X86_OR_X64}") add_subdirectory(stl) -# Since we don't configure any of the tests with CMake, there is no downside to enabling testing unconditionally. -enable_testing() -add_subdirectory(tests) +if(ENABLE_TESTS) + enable_testing() + add_subdirectory(tests) +endif() diff --git a/README.md b/README.md index baaca174e16..48a53a4feaa 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ flavor of the STL (native desktop). We need to extend this to build all of the f because they need to be updated whenever source files are added/renamed/deleted. We'll delete the legacy machinery as soon as possible.) -* Tests: **Coming soon.** We rely on three test suites: std, tr1, and [libcxx][]. We need to replace our current test -harness, which extensively uses Microsoft-internal machinery. +* Tests: **In progress.** We rely on three test suites: std, tr1, and [libcxx][]. We've partially ported std and tr1, +and fully ported libcxx to run under [lit][] using the various configurations/compilers we test internally. * Continuous Integration: **In progress.** We've set up Azure Pipelines to validate changes to the repository. Currently, it builds the STL (native desktop for x86, x64, ARM, and ARM64). Also, it strictly verifies that all of our @@ -220,6 +220,159 @@ C:\Users\bion\Desktop>dumpbin /IMPORTS .\example.exe | findstr msvcp msvcp140d_oss.dll ``` +# How To Run The Tests From The Developer Command Prompt For VS + +1. Follow steps 1-9 of [How To Build With A Native Tools Command Prompt][]. +2. Invoke `git submodule update --init llvm-project` +3. Invoke `cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE={where your vcpkg clone is located}\scripts\buildsystems\vcpkg.cmake +-DENABLE_TESTS=TRUE -S . -B {wherever you want binaries}`. This differs from above only in `-DENABLE_TESTS=TRUE`. +4. If you have already followed the steps from [How To Build With A Native Tools Command Prompt][], and have not +changed the value of `{wherever you want binaries}` in step 4, then there is no need to rebuild to run the tests. +Otherwise, invoke `ninja -C {wherever you want binaries}` to build the project. + +In addition to following the above steps you must also have [Python][] 3.8 or newer, and have LLVM's `bin` directory on +the PATH. + +## Running All The Tests + +After configuring and building the project, running `ctest` from the build output directory will run all the tests. +CTest will only display the standard error output of tests that failed. In order to get more details from CTest's +`lit` invocations, run the tests with `ctest -V`. + +## Running A Subset Of The Tests + +`${PROJECT_BINARY_DIR}\tests\llvm-lit\llvm-lit.py` can be invoked on a subdirectory of a testsuite and will execute all +the tests under that subdirectory. This can mean executing the entirety of a single testsuite, running all tests under +a category in libcxx, or running a single test in `std` and `tr1`. + +## Examples + +``` +:: This command configures the project with tests enabled. It assumes you are using the vcpkg submodule and have +:: already installed boost. It also assumes you have inited and updated the llvm-project submodule. + +C:\STL\build>cmake -GNinja -DCMAKE_CXX_COMPILER=cl -DCMAKE_TOOLCHAIN_FILE=..\vcpkg\scripts\buildsystems\vcpkg.cmake ^ +-DENABLE_TESTS=TRUE .. + +:: As stated above, this step is only strictly necessary if you have yet to build the STL or if you have changed the +:: output directory of the binaries. Any changes or additions in any of the existing testsuites do not require +:: recompilation or reconfiguration to take effect when running the tests. + +C:\STL\build>ninja + +:: This command will run all of the testsuites with verbose output. + +C:\STL\build>ctest -V + +:: This command will run all of the std testsuite. + +C:\STL\build>ctest -R std + +:: This command will also run all of the std testsuite. + +C:\STL\build>python tests\llvm-lit\llvm-lit.py ..\tests\std + +:: If you want to run a subset of a testsuite you need to point it to the right place in the sources. The following +:: will run the single test found under VSO_0000000_any_calling_conventions. + +C:\STL\build>python tests\llvm-lit\llvm-lit.py ..\tests\std\tests\VSO_0000000_any_calling_conventions + +:: You can invoke llvm-lit with any arbitrary subdirectory of a testsuite. In libcxx this allows you to have finer +:: control over what category of tests you would like to run. The following will run all the libcxx map tests. + +C:\STL\build>python tests\llvm-lit\llvm-lit.py ..\llvm-project\libcxx\test\std\containers\associative\map +``` + +## Interpreting The Results Of Tests + +### CTest + +When running the tests via CTest each of the testsuites is considered to be a single test. If any single test in a +testsuite fails, CTest will report the test which represents that testsuite as failed. + +Example: +``` +67% tests passed, 1 tests failed out of 3 + +Total Test time (real) = 2441.55 sec + +The following tests FAILED: + 1 - libcxx (Failed) +``` + +CTest will output everything that was sent to stderr for each of the failed testsuites, which can be used to identify +which individual test within the testsuite failed. It can sometimes be helpful to run CTest with the `-V` option in +order to see the stdout of the tests. + +### llvm-lit + +When running the tests directly via the generated `llvm-lit.py` script the result of each test will be printed. The +format of each result is `{Result Code}: {Testsuite Name} :: {Test Name}:{Configuration Number}`. + +Example: +``` +-- Testing: 28 tests, 12 workers -- +PASS: tr1 :: tests/cwchar1:01 (1 of 28) +PASS: tr1 :: tests/cwchar1:11 (2 of 28) +PASS: tr1 :: tests/cwchar1:02 (3 of 28) +PASS: tr1 :: tests/cwchar1:03 (4 of 28) +PASS: tr1 :: tests/cwchar1:00 (5 of 28) +PASS: tr1 :: tests/cwchar1:04 (6 of 28) +PASS: tr1 :: tests/cwchar1:05 (7 of 28) +PASS: tr1 :: tests/cwchar1:09 (8 of 28) +PASS: tr1 :: tests/cwchar1:06 (9 of 28) +UNSUPPORTED: tr1 :: tests/cwchar1:20 (10 of 28) +UNSUPPORTED: tr1 :: tests/cwchar1:21 (11 of 28) +UNSUPPORTED: tr1 :: tests/cwchar1:22 (12 of 28) +UNSUPPORTED: tr1 :: tests/cwchar1:23 (13 of 28) +UNSUPPORTED: tr1 :: tests/cwchar1:24 (14 of 28) +PASS: tr1 :: tests/cwchar1:07 (15 of 28) +PASS: tr1 :: tests/cwchar1:08 (16 of 28) +PASS: tr1 :: tests/cwchar1:10 (17 of 28) +PASS: tr1 :: tests/cwchar1:16 (18 of 28) +PASS: tr1 :: tests/cwchar1:17 (19 of 28) +PASS: tr1 :: tests/cwchar1:14 (20 of 28) +PASS: tr1 :: tests/cwchar1:12 (21 of 28) +PASS: tr1 :: tests/cwchar1:13 (22 of 28) +PASS: tr1 :: tests/cwchar1:19 (23 of 28) +PASS: tr1 :: tests/cwchar1:18 (24 of 28) +PASS: tr1 :: tests/cwchar1:15 (25 of 28) +PASS: tr1 :: tests/cwchar1:25 (26 of 28) +PASS: tr1 :: tests/cwchar1:26 (27 of 28) +PASS: tr1 :: tests/cwchar1:27 (28 of 28) + +Testing Time: 3.96s + Expected Passes : 23 + Unsupported Tests : 5 +``` + +In the above example we see that 23 tests succeeded and 5 were unsupported. + +### Result Code Values + +Our tests use the standard [lit result codes][], and a non-standard result code: `SKIP`. For our tests, only the +`PASS`, `XFAIL`, `XPASS`, `FAIL`, and `UNSUPPORTED` standard result codes are relevant. + +The `PASS` and `FAIL` result codes are self-explanatory. We want our tests to `PASS` and not `FAIL`. + +The `XPASS` and `XFAIL` result codes are less obvious. `XPASS` is actually a failure result and indicates that we +expected a test to fail but it passed. `XFAIL` is a successful result and indicates that we expected the test to fail +and it did. Typically an `XPASS` result means that the `expected_results.txt` file for the testsuite needs to be +modified. If the `XPASS` result is a test legitimately passing, the usual course of action would be to remove a `FAIL` +entry from the `expected_results.txt`. However, some tests from `libcxx` mark themselves as `XFAIL` (meaning they +expect to fail) for features they have added tests for but have yet to implement in `libcxx`. If the STL implements +those features first the tests will begin passing unexpectedly for us and return `XPASS` results. In order to resolve +this it is necessary to add a `PASS` entry to the `expected_results.txt` of the testsuite in question. + +The `UNSUPPORTED` result code means that the requirements for a test are not met and so it will not be run. Currently +all tests which use the `/BE` or `/clr:pure` options are unsupported. + +The `SKIP` result code indicates that a given test was explicitly skipped by adding a `SKIP` entry to the +`expected_results.txt`. A test may be skipped for a number of reasons, which include, but are not limited to: +* being an incorrect test +* taking a very long time to run +* failing or passing for the incorrect reason + # Block Diagram The STL is built atop other compiler support libraries that ship with Windows and Visual Studio, like the UCRT, @@ -255,6 +408,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception [Code of Conduct FAQ]: https://opensource.microsoft.com/codeofconduct/faq/ [Compiler Explorer]: https://godbolt.org [Developer Community]: https://developercommunity.visualstudio.com/spaces/62/index.html +[How To Build With A Native Tools Command Prompt]: #how-to-build-with-a-native-tools-command-prompt [LICENSE.txt]: LICENSE.txt [LWG issues]: https://cplusplus.github.io/LWG/lwg-toc.html [LWG tag]: https://github.com/microsoft/STL/issues?q=is%3Aopen+is%3Aissue+label%3ALWG @@ -263,6 +417,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception [NOTICE.txt]: NOTICE.txt [Ninja]: https://ninja-build.org [Pipelines]: https://dev.azure.com/vclibs/STL/_build/latest?definitionId=2&branchName=master +[Python]: https://www.python.org/downloads/windows/ [Roadmap]: https://github.com/microsoft/STL/wiki/Roadmap [Wandbox]: https://wandbox.org [bug tag]: https://github.com/microsoft/STL/issues?q=is%3Aopen+is%3Aissue+label%3Abug @@ -270,6 +425,8 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception [enhancement tag]: https://github.com/microsoft/STL/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement [hub]: https://support.microsoft.com/en-us/help/4021566/windows-10-send-feedback-to-microsoft-with-feedback-hub-app [libcxx]: https://libcxx.llvm.org +[lit]: https://llvm.org/docs/CommandGuide/lit.html +[lit result codes]: https://llvm.org/docs/CommandGuide/lit.html#test-status-results [opencode@microsoft.com]: mailto:opencode@microsoft.com [redistributables]: https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads [vcpkg]: https://github.com/microsoft/vcpkg diff --git a/azure-devops/run-build.yml b/azure-devops/run-build.yml index afe6304e470..e19fc9027e8 100644 --- a/azure-devops/run-build.yml +++ b/azure-devops/run-build.yml @@ -76,7 +76,8 @@ jobs: cmakeListsTxtPath: '$(Build.SourcesDirectory)/CMakeLists.txt' buildDirectory: $(Build.ArtifactStagingDirectory)/${{ parameters.targetPlatform }} useVcpkgToolchainFile: true - cmakeAppendedArgs: '-G Ninja -DENABLE_XUNIT_OUTPUT=TRUE -DADDITIONAL_LIT_FLAGS=-j$(testParallelism)' + cmakeAppendedArgs: | + -G Ninja -DENABLE_TESTS=TRUE -DENABLE_XUNIT_OUTPUT=TRUE -DADDITIONAL_LIT_FLAGS=-j$(testParallelism) - task: PowerShell@2 displayName: 'Run Tests' timeoutInMinutes: 120 diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 39a88e49139..3d49a8274da 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -733,6 +733,7 @@ std/thread/futures/futures.async/async.pass.cpp SKIP std/thread/futures/futures.shared_future/get.pass.cpp SKIP std/thread/futures/futures.shared_future/wait_for.pass.cpp SKIP std/thread/futures/futures.shared_future/wait.pass.cpp SKIP +std/thread/futures/futures.unique_future/get.pass.cpp SKIP std/thread/futures/futures.unique_future/wait_for.pass.cpp SKIP std/thread/futures/futures.unique_future/wait.pass.cpp SKIP std/thread/thread.condition/thread.condition.condvar/notify_all.pass.cpp SKIP diff --git a/tests/libcxx/skipped_tests.txt b/tests/libcxx/skipped_tests.txt index 27551a81790..efb8e47769e 100644 --- a/tests/libcxx/skipped_tests.txt +++ b/tests/libcxx/skipped_tests.txt @@ -733,6 +733,7 @@ thread\futures\futures.async\async.pass.cpp thread\futures\futures.shared_future\get.pass.cpp thread\futures\futures.shared_future\wait_for.pass.cpp thread\futures\futures.shared_future\wait.pass.cpp +thread\futures\futures.unique_future\get.pass.cpp thread\futures\futures.unique_future\wait_for.pass.cpp thread\futures\futures.unique_future\wait.pass.cpp thread\thread.condition\thread.condition.condvar\notify_all.pass.cpp diff --git a/tests/tr1/tests/cwchar1/__init__.py b/tests/tr1/tests/cwchar1/__init__.py deleted file mode 100644 index 2ac2a854cb0..00000000000 --- a/tests/tr1/tests/cwchar1/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/tests/tr1/tests/cwchar1/custom_format.py b/tests/tr1/tests/cwchar1/custom_format.py deleted file mode 100644 index 456c91bb9bf..00000000000 --- a/tests/tr1/tests/cwchar1/custom_format.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -from stl.test.format import STLTestFormat, TestStep - - -class CustomTestFormat(STLTestFormat): - def getTestSteps(self, test, lit_config, shared): - if shared.exec_file is None: - for step in super().getTestSteps(test, lit_config, shared): - yield step - else: - exec_env = test.cxx.compile_env - exec_env['TMP'] = str(shared.exec_dir) - - yield TestStep([str(shared.exec_file)], shared.exec_dir, - [shared.exec_file], exec_env) diff --git a/tests/tr1/tests/cwchar1/lit.local.cfg b/tests/tr1/tests/cwchar1/lit.local.cfg deleted file mode 100644 index c1dd34543ab..00000000000 --- a/tests/tr1/tests/cwchar1/lit.local.cfg +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import cwchar1.custom_format - -config.test_format = \ - cwchar1.custom_format.CustomTestFormat(config.test_format.cxx, - config.test_format.execute_external, - config.test_format.build_executor, - config.test_format.test_executor) diff --git a/tests/utils/stl/test/format.py b/tests/utils/stl/test/format.py index 717fb926c9c..db3b707d63e 100644 --- a/tests/utils/stl/test/format.py +++ b/tests/utils/stl/test/format.py @@ -254,8 +254,11 @@ def getBuildSteps(self, test, lit_config, shared): def getTestSteps(self, test, lit_config, shared): if shared.exec_file is not None: + exec_env = test.cxx.compile_env + exec_env['TMP'] = str(shared.exec_dir) + yield TestStep([str(shared.exec_file)], shared.exec_dir, - [shared.exec_file], test.cxx.compile_env) + [shared.exec_file], exec_env) elif test.path_in_suite[-1].endswith('.fail.cpp'): exec_dir = test.getExecDir() source_path = Path(test.getSourcePath())