From 5353ac4e0c3718507f20c6a0e8eaed235fd55c91 Mon Sep 17 00:00:00 2001 From: Isaac Garzon Date: Tue, 2 Aug 2022 19:27:03 +0300 Subject: [PATCH] cmake: allow running the tests in parallel like in the Makefile (#102) Allow running tests in parallel by default by setting the CTEST_PARALLEL_LEVEL environment variable. The functionality is added using a CMake script in order to determine options when running the tests (rather than when configuring CMake) in compatibility with the Makefile (in particular -- the `J` and `TEST_TMPDIR` environment variables). While at it, add a test timeout parameter in order to kill stuck tests, and set the default timeout to 10 minutes (we don't currently have tests that are running longer than that). Test Plan: run the check target using different generators (ninja, make) to see the improvement in test runtime. --- CMakeLists.txt | 25 +++++++-- Makefile | 6 ++- cmake/CTestRunner.cmake | 113 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 cmake/CTestRunner.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c228d29ab1..454a4b70a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1306,6 +1306,20 @@ if(WITH_TESTS OR WITH_BENCHMARK_TOOLS) endif() if(WITH_TESTS) + # c_test - doesn't use gtest + # env_test - suspicious use of test::TmpDir + # deletefile_test - serial because it generates giant temporary files in + # its various tests. Running its tests in parallel can fill up your /dev/shm + # db_bloom_filter_test - serial because excessive space usage by instances + # of DBFilterConstructionReserveMemoryTestWithParam can fill up /dev/shm + # timer_queue_test - doesn't use gtest + set(NON_PARALLEL_TESTS + c_test + env_test + deletefile_test + db_bloom_filter_test + timer_queue_test + ) set(TESTS db/db_basic_test.cc env/env_basic_test.cc @@ -1520,7 +1534,10 @@ if(WITH_TESTS) utilities/cassandra/test_utils.cc ) enable_testing() - add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND}) + add_custom_target(check + COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/CTestRunner.cmake + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + VERBATIM USES_TERMINAL) set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX}) add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE}) target_link_libraries(${TESTUTILLIB} ${ROCKSDB_LIB} ${FOLLY_LIBS}) @@ -1543,10 +1560,12 @@ if(WITH_TESTS) OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX} ) target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${TESTUTILLIB} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB}) - if(NOT "${exename}" MATCHES "db_sanity_test") + if(NOT "${exename}" IN_LIST NON_PARALLEL_TESTS) gtest_discover_tests(${exename} DISCOVERY_TIMEOUT 120) - add_dependencies(check ${exename}${ARTIFACT_SUFFIX}) + else() + add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX}) endif() + add_dependencies(check ${exename}${ARTIFACT_SUFFIX}) endforeach(sourcefile ${TESTS}) if(WIN32) diff --git a/Makefile b/Makefile index 6f6ff2dd4d..2c52ed1165 100644 --- a/Makefile +++ b/Makefile @@ -985,6 +985,8 @@ PARALLEL ?= parallel PARALLEL_OK := $(shell command -v "$(PARALLEL)" 2>&1 >/dev/null && \ ("$(PARALLEL)" --gnu --version 2>/dev/null | grep -q 'Ole Tange') && \ echo 1) +# Use a timeout of 10 minutes per test by default +TEST_TIMEOUT?=600 # Use this regexp to select the subset of tests whose names match. tests-regexp = . @@ -1014,7 +1016,7 @@ check_0: gen_parallel_tests | grep -E '$(tests-regexp)' \ | grep -E -v '$(EXCLUDE_TESTS_REGEX)' \ | "$(PARALLEL)" -j$(J) --plain --joblog=LOG --eta --gnu \ - --tmpdir=$(TEST_TMPDIR) '{} $(parallel_redir)' ; \ + --tmpdir=$(TEST_TMPDIR) --timeout=$(TEST_TIMEOUT) '{} $(parallel_redir)' ; \ parallel_retcode=$$? ; \ awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \ awk_retcode=$$?; \ @@ -1036,7 +1038,7 @@ valgrind_check_0: gen_parallel_tests | grep -E '$(tests-regexp)' \ | grep -E -v '$(valgrind-exclude-regexp)' \ | "$(PARALLEL)" -j$(J) --plain --joblog=LOG --eta --gnu \ - --tmpdir=$(TEST_TMPDIR) \ + --tmpdir=$(TEST_TMPDIR) --timeout=$(TEST_TIMEOUT) \ '(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) \ $(parallel_redir)' \ diff --git a/cmake/CTestRunner.cmake b/cmake/CTestRunner.cmake new file mode 100644 index 0000000000..3d90d6c58a --- /dev/null +++ b/cmake/CTestRunner.cmake @@ -0,0 +1,113 @@ +# Copyright (C) 2022 Speedb Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 3.12 is needed for FindPython +cmake_minimum_required(VERSION 3.12) + +# Choose the amount of tests to run in parallel if CTEST_PARALLEL_LEVEL wasn't set +if(NOT DEFINED ENV{CTEST_PARALLEL_LEVEL}) + # Compatibility with the Makefile: support the `J` environment variable + if(DEFINED ENV{J} AND "$ENV{J}" GREATER 0) + set(ENV{CTEST_PARALLEL_LEVEL} "$ENV{J}") + else() + include(ProcessorCount) + ProcessorCount(NCPU) + if(NOT NCPU EQUAL 0) + set(ENV{CTEST_PARALLEL_LEVEL} ${NCPU}) + endif() + endif() +endif() + +# For Makefile compatibility try the following sequence if TEST_TMPDIR isn't set: +# * Use TMPD if set +# * Find a suitable base directory and create a temporary directory under it: +# * /dev/shm on Linux if exists and has the sticky bit set +# * TMPDIR if set and exists +# * On Windows use TMP is set and exists +# * On Windows use TEMP is set and exists +# * /tmp if exists +if(NOT DEFINED ENV{TEST_TMPDIR}) + # Use TMPD if set + if(DEFINED ENV{TMPD}) + set(test_dir "$ENV{TMPD}") + else() + # On Linux, use /dev/shm if the sticky bit is set + if("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Linux" AND IS_DIRECTORY "/dev/shm") + execute_process(COMMAND test -k /dev/shm RESULT_VARIABLE status OUTPUT_QUIET ERROR_QUIET) + if(status EQUAL 0) + set(test_dir "/dev/shm") + endif() + endif() + # Use TMPDIR as base if set + if(NOT DEFINED test_dir AND IS_DIRECTORY "$ENV{TMPDIR}") + set(test_dir "$ENV{TMPDIR}") + elseif("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Windows") + # Use TMP or TEMP as base if set + # See https://devblogs.microsoft.com/oldnewthing/20150417-00/?p=44213 + if(IS_DIRECTORY "$ENV{TMP}") + set(test_dir "$ENV{TMP}") + elseif(IS_DIRECTORY "$ENV{TEMP}") + set(test_dir "$ENV{TEMP}") + endif() + endif() + # Fall back to /tmp if exists + if(NOT DEFINED test_dir AND IS_DIRECTORY "/tmp") + set(test_dir "/tmp") + endif() + # Create a temporary directory under the base path that we determined + if(DEFINED test_dir) + include(FindPython) + find_package(Python COMPONENTS Interpreter) + # Try using Python for more portability when creating the temporary + # sub-directory, but don't depend on it + if(Python_Interpreter_FOUND) + execute_process( + COMMAND "${CMAKE_COMMAND}" -E env "test_dir=${test_dir}" + "${Python_EXECUTABLE}" -c "import os, tempfile; print(tempfile.mkdtemp(prefix='rocksdb.', dir=os.environ['test_dir']))" + RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir + OUTPUT_STRIP_TRAILING_WHITESPACE) + if (NOT status EQUAL 0) + message(FATAL_ERROR "Python mkdtemp failed") + endif() + set(test_dir "${tmpdir}") + elseif(NOT "${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Windows") + execute_process( + COMMAND mktemp -d "${test_dir}/rocksdb.XXXXXX" + RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir + OUTPUT_STRIP_TRAILING_WHITESPACE) + if (NOT status EQUAL 0) + message(FATAL_ERROR "mkdtemp failed") + endif() + set(test_dir "${tmpdir}") + endif() + endif() + endif() + if(DEFINED test_dir) + set(ENV{TEST_TMPDIR} "${test_dir}") + endif() +endif() + +if(DEFINED ENV{TEST_TMPDIR}) + message(STATUS "Running $ENV{CTEST_PARALLEL_LEVEL} tests in parallel in $ENV{TEST_TMPDIR}") +endif() + +# Use a timeout of 10 minutes per test by default +if(DEFINED ENV{TEST_TIMEOUT}) + set(test_timeout "$ENV{TEST_TIMEOUT}") +else() + set(test_timeout 600) +endif() + +# Run all tests, and show test output on failure +execute_process(COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --timeout ${test_timeout})