From 8a81432f58596efbb1a3f185b57c9c95d70040d5 Mon Sep 17 00:00:00 2001 From: Amnon Hanuhov Date: Tue, 15 Nov 2022 06:44:43 +0200 Subject: [PATCH] build: remove the dependency on GNU Parallel for running unit tests (#122) Instead, use gtest-parallel as a bundled script for running both gtest tests and non-gtest tests (such as c_test, for example). --- Makefile | 173 +++++----------------------------- build_tools/gtest_parallel.py | 124 ++++++++++++++---------- 2 files changed, 99 insertions(+), 198 deletions(-) diff --git a/Makefile b/Makefile index 212435063f..48634d73df 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,6 @@ MACHINE ?= $(shell uname -m) ARFLAGS = ${EXTRA_ARFLAGS} rs STRIPFLAGS = -S -x -# Transform parallel LOG output into something more readable. -parallel_log_extract = awk \ - 'BEGIN{FS="\t"} { \ - t=$$9; sub(/if *\[\[ *"/,"",t); sub(/" =.*/,"",t); sub(/ >.*/,"",t); sub(/.*--gtest_filter=/,"",t); \ - printf("%7.3f %s %s\n",4,($$7 == 0 ? "PASS" : "FAIL"),t) \ - }' - # DEBUG_LEVEL can have three values: # * DEBUG_LEVEL=2; this is the ultimate debug mode. It will compile Speedb # without any optimizations. To compile with level 2, issue `make dbg` @@ -952,122 +945,29 @@ coverage: clean # Delete intermediate files $(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \; -# Run all tests in parallel, accumulating per-test logs in t/log-*. -# -# Each t/run-* file is a tiny generated bourne shell script that invokes one of -# sub-tests. Why use a file for this? Because that makes the invocation of -# parallel below simpler, which in turn makes the parsing of parallel's -# LOG simpler (the latter is for live monitoring as parallel -# tests run). -# -# Test names are extracted by running tests with --gtest_list_tests. -# This filter removes the "#"-introduced comments, and expands to -# fully-qualified names by changing input like this: -# -# DBTest. -# Empty -# WriteEmptyBatch -# MultiThreaded/MultiThreadedDBTest. -# MultiThreaded/0 # GetParam() = 0 -# MultiThreaded/1 # GetParam() = 1 -# -# into this: -# -# DBTest.Empty -# DBTest.WriteEmptyBatch -# MultiThreaded/MultiThreadedDBTest.MultiThreaded/0 -# MultiThreaded/MultiThreadedDBTest.MultiThreaded/1 -# - -parallel_tests = $(patsubst %,parallel_%,$(PARALLEL_TEST)) -.PHONY: $(parallel_tests) -$(parallel_tests): $(parallel_tests:parallel_%=%) - $(AM_V_at)mkdir -p t; \ - TEST_BINARY=$(patsubst parallel_%,%,$@); \ - TEST_NAMES=` \ - (./$$TEST_BINARY --gtest_list_tests || echo " $${TEST_BINARY}__list_tests_failure") \ - | awk '/^[^ ]/ { prefix = $$1 } /^[ ]/ { print prefix $$1 }'`; \ - echo " Generating parallel test scripts for $$TEST_BINARY"; \ - rm -f t/run-$${TEST_BINARY}-*; \ - for TEST_NAME in $$TEST_NAMES; do \ - TEST_SCRIPT=run-$${TEST_BINARY}-$${TEST_NAME//\//-}; \ - printf '%s\n' \ - '#!/bin/sh' \ - "d=\"$(TEST_TMPDIR)/$$TEST_SCRIPT\"" \ - 'mkdir -p "$$d"' \ - "TEST_TMPDIR=\"\$$d\" $(DRIVER) ./$$TEST_BINARY --gtest_filter=$$TEST_NAME && rm -rf \"\$$d\"" \ - > t/$$TEST_SCRIPT; \ - chmod a=rx t/$$TEST_SCRIPT; \ - done - -# Reorder input lines (which are one per test) so that the -# longest-running tests appear first in the output. -# Do this by prefixing each selected name with its duration, -# sort the resulting names, and remove the leading numbers. -# FIXME: the "100" we prepend is a fake time, for now. -# FIXME: squirrel away timings from each run and use them -# (when present) on subsequent runs to order these tests. -# -# Without this reordering, these two tests would happen to start only -# after almost all other tests had completed, thus adding 100 seconds -# to the duration of parallel "make check". That's the difference -# between 4 minutes (old) and 2m20s (new). -# -# 152.120 PASS t/DBTest.FileCreationRandomFailure -# 107.816 PASS t/DBTest.EncodeDecompressedBlockSizeTest -# -slow_test_regexp = \ - ^.*MySQLStyleTransactionTest.*$$\|^.*SnapshotConcurrentAccessTest.*$$\|^.*SeqAdvanceConcurrentTest.*$$\|^t/run-table_test-HarnessTest.Randomized$$\|^t/run-db_test-.*FileCreationRandomFailure$$\|^t/run-db_test-.*EncodeDecompressedBlockSizeTest$$\|^.*RecoverFromCorruptedWALWithoutFlush$$ -prioritize_long_running_tests = \ - sed 's,\($(slow_test_regexp)\),100 \1,' \ - | sort -k1,1gr \ - | sed 's/^[.0-9]* //' - # "make check" uses # Run with "make J=1 check" to disable parallelism in "make check". -# Run with "make J=200% check" to run two parallel jobs per core. -# The default is to run one job per core (J=100%). -# See "man parallel" for its "-j ..." option. -J ?= 100% - -PARALLEL ?= parallel -PARALLEL_OK := $(shell command -v "$(PARALLEL)" 2>&1 >/dev/null && \ - ("$(PARALLEL)" --gnu --version 2>/dev/null | grep -q 'Ole Tange') && \ - echo 1) -# Use a timeout of 10 minutes per test by default -TEST_TIMEOUT?=600 - -# Use this regexp to select the subset of tests whose names match. -tests-regexp = . -EXCLUDE_TESTS_REGEX ?= "^$$" - -ifeq ($(PRINT_PARALLEL_OUTPUTS), 1) - parallel_redir = -else ifeq ($(QUIET_PARALLEL_TESTS), 1) - parallel_redir = >& t/$(test_log_prefix)log-{/} -else -# Default: print failure output only, as it happens -# Note: parallel --eta is now always used because CircleCI will -# kill a job if no output for 10min. - parallel_redir = >& t/$(test_log_prefix)log-{/} || bash -c "cat t/$(test_log_prefix)log-{/}; exit $$?" -endif - +# Run with "make J= check" to run N jobs at once, for example "make J=16 check". +# The default is to run one job per core (J=number of physical cores). +ifeq ($(PLATFORM), OS_MACOSX) +J ?= $(shell sysctl -n hw.physicalcpu) +else # Unix +J ?= $(shell nproc) +endif +CURRENT_DIR = $(shell pwd) +NON_PARALLEL_TESTS_LIST := $(foreach test,$(NON_PARALLEL_TEST),$(CURRENT_DIR)/$(test)) +space := $(subst ,, ) +comma := , +NON_PARALLEL_TESTS_LIST := $(subst $(space),$(comma),$(NON_PARALLEL_TESTS_LIST)) +PARALLEL_TESTS_LIST := $(foreach test,$(PARALLEL_TEST),$(CURRENT_DIR)/$(test)) +# All logs are available under gtest-parallel-logs/. +# If OUTPUT_DIR is not set, by default the logs will be +# under /tmp/gtest-parallel-logs/. +# Run with OUTPUT_DIR= to replace the default directory. +OUTPUT_DIR ?= /tmp .PHONY: check_0 check_1 -check_0: $(TESTS) $(parallel_tests) - $(AM_V_GEN)printf '%s\n' '' \ - 'Running tests in $(TEST_TMPDIR)' \ - 'To monitor subtest ,' \ - ' run "make watch-log" in a separate window' ''; \ - printf './%s\n' $(filter-out $(PARALLEL_TEST),$(TESTS)) $(PARALLEL_TEST:%=t/run-%-*) \ - | $(prioritize_long_running_tests) \ - | grep -E '$(tests-regexp)' \ - | grep -E -v '$(EXCLUDE_TESTS_REGEX)' \ - | "$(PARALLEL)" -j$(J) --plain --joblog=LOG --eta --gnu \ - --tmpdir=$(TEST_TMPDIR) --timeout=$(TEST_TIMEOUT) '{} $(parallel_redir)' ; \ - parallel_retcode=$$? ; \ - awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \ - awk_retcode=$$?; \ - if [ $$parallel_retcode -ne 0 ] || [ $$awk_retcode -ne 0 ] ; then exit 1 ; fi; +check_0: $(TESTS) + $(AM_V_GEN)./build_tools/gtest-parallel --output_dir=$(OUTPUT_DIR) --workers=$(J) --non_gtest_tests $(NON_PARALLEL_TESTS_LIST) $(PARALLEL_TESTS_LIST) check_1: $(TESTS) $(AM_V_GEN)for t in $(TESTS); do \ @@ -1077,20 +977,8 @@ check_1: $(TESTS) valgrind-exclude-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest .PHONY: valgrind_check_0 valgrind_check_1 -valgrind_check_0: test_log_prefix := valgrind_ -valgrind_check_0: $(TESTS) $(parallel_tests) - $(AM_V_GEN)printf '%s\n' '' \ - 'Running tests in $(TEST_TMPDIR)' \ - 'To monitor subtest ,' \ - ' run "make watch-log" in a separate window' ''; \ - printf './%s\n' $(filter-out $(PARALLEL_TEST) %skiplist_test options_settable_test, $(TESTS)) $(PARALLEL_TEST:%=t/run-%-*) \ - | $(prioritize_long_running_tests) \ - | grep -E '$(tests-regexp)' \ - | grep -E -v '$(valgrind-exclude-regexp)' \ - | "$(PARALLEL)" -j$(J) --plain --joblog=LOG --eta --gnu \ - --tmpdir=$(TEST_TMPDIR) --timeout=$(TEST_TIMEOUT) \ - '(if [[ "{}" == "./"* ]] ; then $(VALGRIND_VER) $(VALGRIND_OPTS) {}; else {}; fi) \ - $(parallel_redir)' \ +valgrind_check_0: $(TESTS) + $(AM_V_GEN) $(VALGRIND_VER) $(VALGRIND_OPTS) ./build_tools/gtest-parallel --output_dir=$(OUTPUT_DIR) --workers=$(J) --non_gtest_tests $(NON_PARALLEL_TESTS_LIST) $(PARALLEL_TESTS_LIST) valgrind_check_1: $(TESTS) $(AM_V_GEN)for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \ @@ -1103,22 +991,9 @@ valgrind_check_1: $(TESTS) CLEAN_FILES += t LOG -# When running parallel "make check", you can monitor its progress -# from another window. -# Run "make watch_LOG" to show the duration,PASS/FAIL,name of parallel -# tests as they are being run. We sort them so that longer-running ones -# appear at the top of the list and any failing tests remain at the top -# regardless of their duration. As with any use of "watch", hit ^C to -# interrupt. -watch-log: - $(WATCH) --interval=0 'tail -n+2 LOG|sort -k7,7nr -k4,4gr|$(subst ','\'',$(parallel_log_extract))' - -dump-log: - tail -n+2 LOG|$(parallel_log_extract) - -# If J != 1 and GNU parallel is installed, run the tests in parallel, +# If J != 1, run the tests in parallel using gtest-parallel, # via the check_0 rule above. Otherwise, run them sequentially via check_1. -check: all $(if $(shell [ "$(J)" != "1" ] && [ "$(PARALLEL_OK)" = "1" ] && echo 1),check_0,check_1) +check: all $(if $(shell [ "$(J)" != "1" ] && echo 1),check_0,check_1) ifneq ($(PLATFORM), OS_AIX) $(PYTHON) tools/check_all_python.py ifeq ($(filter -DROCKSDB_LITE,$(OPT)),) diff --git a/build_tools/gtest_parallel.py b/build_tools/gtest_parallel.py index ac16db6278..5ab3fd18e3 100755 --- a/build_tools/gtest_parallel.py +++ b/build_tools/gtest_parallel.py @@ -604,61 +604,73 @@ def find_tests(binaries, additional_args, options, times): tasks = [] for test_binary in binaries: command = [test_binary] + additional_args - if options.gtest_also_run_disabled_tests: - command += ['--gtest_also_run_disabled_tests'] - - list_command = command + ['--gtest_list_tests'] - if options.gtest_filter != '': - list_command += ['--gtest_filter=' + options.gtest_filter] - - try: - test_list = subprocess.check_output(list_command, - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - sys.exit("%s: %s\n%s" % (test_binary, str(e), e.output)) - - try: - test_list = test_list.split('\n') - except TypeError: - # subprocess.check_output() returns bytes in python3 - test_list = test_list.decode(sys.stdout.encoding).split('\n') - - command += ['--gtest_color=' + options.gtest_color] - - test_group = '' - for line in test_list: - if not line.strip(): - continue - if line[0] != " ": - # Remove comments for typed tests and strip whitespace. - test_group = line.split('#')[0].strip() - continue - # Remove comments for parameterized tests and strip whitespace. - line = line.split('#')[0].strip() - if not line: - continue - - test_name = test_group + line - if not options.gtest_also_run_disabled_tests and 'DISABLED_' in test_name: - continue - - # Skip PRE_ tests which are used by Chromium. - if '.PRE_' in test_name: - continue - + if options.non_gtest_tests and test_binary in options.non_gtest_tests: + test_name = os.path.basename(test_binary) last_execution_time = times.get_test_time(test_binary, test_name) if options.failed and last_execution_time is not None: continue - - test_command = command + ['--gtest_filter=' + test_name] if (test_count - options.shard_index) % options.shard_count == 0: - for execution_number in range(options.repeat): - tasks.append( - Task(test_binary, test_name, test_command, execution_number + 1, - last_execution_time, options.output_dir)) - + for execution_number in range(options.repeat): + tasks.append( + Task(test_binary, test_name, command, execution_number + 1, + last_execution_time, options.output_dir)) test_count += 1 + else: + if options.gtest_also_run_disabled_tests: + command += ['--gtest_also_run_disabled_tests'] + list_command = command + ['--gtest_list_tests'] + if options.gtest_filter != '': + list_command += ['--gtest_filter=' + options.gtest_filter] + + try: + test_list = subprocess.check_output(list_command, + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + sys.exit("%s: %s\n%s" % (test_binary, str(e), e.output)) + + try: + test_list = test_list.split('\n') + except TypeError: + # subprocess.check_output() returns bytes in python3 + test_list = test_list.decode(sys.stdout.encoding).split('\n') + + command += ['--gtest_color=' + options.gtest_color] + + test_group = '' + for line in test_list: + if not line.strip(): + continue + if line[0] != " ": + # Remove comments for typed tests and strip whitespace. + test_group = line.split('#')[0].strip() + continue + # Remove comments for parameterized tests and strip whitespace. + line = line.split('#')[0].strip() + if not line: + continue + + test_name = test_group + line + if not options.gtest_also_run_disabled_tests and 'DISABLED_' in test_name: + continue + + # Skip PRE_ tests which are used by Chromium. + if '.PRE_' in test_name: + continue + + last_execution_time = times.get_test_time(test_binary, test_name) + if options.failed and last_execution_time is not None: + continue + + test_command = command + ['--gtest_filter=' + test_name] + if (test_count - options.shard_index) % options.shard_count == 0: + for execution_number in range(options.repeat): + tasks.append( + Task(test_binary, test_name, test_command, execution_number + 1, + last_execution_time, options.output_dir)) + + test_count += 1 + # Sort the tasks to run the slowest tests first, so that faster ones can be # finished in parallel. return sorted(tasks, reverse=True) @@ -723,6 +735,10 @@ def start_daemon(func): task_manager.register_exit(task) +def list_non_gtest_tests(option, opt, value, parser): + setattr(parser.values, option.dest, value.split(',')) + + def default_options_parser(): parser = optparse.OptionParser( usage='usage: %prog [options] binary [binary ...] -- [additional args]') @@ -797,6 +813,13 @@ def default_options_parser(): default=False, help='Do not run tests from the same test ' 'case in parallel.') + parser.add_option('--non_gtest_tests', + type='string', + action='callback', + callback=list_non_gtest_tests, + dest='non_gtest_tests', + help='A list of comma separated tests that do not use ' + 'gtest, that should also be run') return parser @@ -824,6 +847,9 @@ def main(): if options.output_dir: options.output_dir = os.path.join(options.output_dir, 'gtest-parallel-logs') + if options.non_gtest_tests: + binaries += options.non_gtest_tests + if binaries == []: parser.print_usage() sys.exit(1)