Skip to content

Commit

Permalink
build: remove the dependency on GNU Parallel for running unit tests (#…
Browse files Browse the repository at this point in the history 
…122)

Instead, use gtest-parallel as a bundled script for running both
gtest tests and non-gtest tests (such as c_test, for example).
  • Loading branch information
@AmnonHanuhov
AmnonHanuhov authored and Amnon Hanuhov committed Jan 6, 2023
1 parent 1e6fd16 commit 0ee37ec
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 198 deletions.
173 changes: 24 additions & 149 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,6 @@ MACHINE ?= $(shell uname -m)
ARFLAGS = ${EXTRA_ARFLAGS} rs
STRIPFLAGS = -S -x

# Transform parallel LOG output into something more readable.
parallel_log_extract = awk \
'BEGIN{FS="\t"} { \
t=$$9; sub(/if *\[\[ *"/,"",t); sub(/" =.*/,"",t); sub(/ >.*/,"",t); sub(/.*--gtest_filter=/,"",t); \
printf("%7.3f %s %s\n",4,($$7 == 0 ? "PASS" : "FAIL"),t) \
}'

# DEBUG_LEVEL can have three values:
# * DEBUG_LEVEL=2; this is the ultimate debug mode. It will compile Speedb
# without any optimizations. To compile with level 2, issue `make dbg`
Expand Down Expand Up @@ -952,122 +945,29 @@ coverage: clean
# Delete intermediate files
$(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \;

# Run all tests in parallel, accumulating per-test logs in t/log-*.
#
# Each t/run-* file is a tiny generated bourne shell script that invokes one of
# sub-tests. Why use a file for this? Because that makes the invocation of
# parallel below simpler, which in turn makes the parsing of parallel's
# LOG simpler (the latter is for live monitoring as parallel
# tests run).
#
# Test names are extracted by running tests with --gtest_list_tests.
# This filter removes the "#"-introduced comments, and expands to
# fully-qualified names by changing input like this:
#
# DBTest.
# Empty
# WriteEmptyBatch
# MultiThreaded/MultiThreadedDBTest.
# MultiThreaded/0 # GetParam() = 0
# MultiThreaded/1 # GetParam() = 1
#
# into this:
#
# DBTest.Empty
# DBTest.WriteEmptyBatch
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
#

parallel_tests = $(patsubst %,parallel_%,$(PARALLEL_TEST))
.PHONY: $(parallel_tests)
$(parallel_tests): $(parallel_tests:parallel_%=%)
$(AM_V_at)mkdir -p t; \
TEST_BINARY=$(patsubst parallel_%,%,$@); \
TEST_NAMES=` \
(./$$TEST_BINARY --gtest_list_tests || echo " $${TEST_BINARY}__list_tests_failure") \
| awk '/^[^ ]/ { prefix = $$1 } /^[ ]/ { print prefix $$1 }'`; \
echo " Generating parallel test scripts for $$TEST_BINARY"; \
rm -f t/run-$${TEST_BINARY}-*; \
for TEST_NAME in $$TEST_NAMES; do \
TEST_SCRIPT=run-$${TEST_BINARY}-$${TEST_NAME//\//-}; \
printf '%s\n' \
'#!/bin/sh' \
"d=\"$(TEST_TMPDIR)/$$TEST_SCRIPT\"" \
'mkdir -p "$$d"' \
"TEST_TMPDIR=\"\$$d\" $(DRIVER) ./$$TEST_BINARY --gtest_filter=$$TEST_NAME && rm -rf \"\$$d\"" \
> t/$$TEST_SCRIPT; \
chmod a=rx t/$$TEST_SCRIPT; \
done

# Reorder input lines (which are one per test) so that the
# longest-running tests appear first in the output.
# Do this by prefixing each selected name with its duration,
# sort the resulting names, and remove the leading numbers.
# FIXME: the "100" we prepend is a fake time, for now.
# FIXME: squirrel away timings from each run and use them
# (when present) on subsequent runs to order these tests.
#
# Without this reordering, these two tests would happen to start only
# after almost all other tests had completed, thus adding 100 seconds
# to the duration of parallel "make check". That's the difference
# between 4 minutes (old) and 2m20s (new).
#
# 152.120 PASS t/DBTest.FileCreationRandomFailure
# 107.816 PASS t/DBTest.EncodeDecompressedBlockSizeTest
#
slow_test_regexp = \
^.*MySQLStyleTransactionTest.*$$\|^.*SnapshotConcurrentAccessTest.*$$\|^.*SeqAdvanceConcurrentTest.*$$\|^t/run-table_test-HarnessTest.Randomized$$\|^t/run-db_test-.*FileCreationRandomFailure$$\|^t/run-db_test-.*EncodeDecompressedBlockSizeTest$$\|^.*RecoverFromCorruptedWALWithoutFlush$$
prioritize_long_running_tests = \
sed 's,\($(slow_test_regexp)\),100 \1,' \
| sort -k1,1gr \
| sed 's/^[.0-9]* //'

# "make check" uses
# Run with "make J=1 check" to disable parallelism in "make check".
# Run with "make J=200% check" to run two parallel jobs per core.
# The default is to run one job per core (J=100%).
# See "man parallel" for its "-j ..." option.
J ?= 100%

PARALLEL ?= parallel
PARALLEL_OK := $(shell command -v "$(PARALLEL)" 2>&1 >/dev/null && \
("$(PARALLEL)" --gnu --version 2>/dev/null | grep -q 'Ole Tange') && \
echo 1)
# Use a timeout of 10 minutes per test by default
TEST_TIMEOUT?=600

# Use this regexp to select the subset of tests whose names match.
tests-regexp = .
EXCLUDE_TESTS_REGEX ?= "^$$"

ifeq ($(PRINT_PARALLEL_OUTPUTS), 1)
parallel_redir =
else ifeq ($(QUIET_PARALLEL_TESTS), 1)
parallel_redir = >& t/$(test_log_prefix)log-{/}
else
# Default: print failure output only, as it happens
# Note: parallel --eta is now always used because CircleCI will
# kill a job if no output for 10min.
parallel_redir = >& t/$(test_log_prefix)log-{/} || bash -c "cat t/$(test_log_prefix)log-{/}; exit $$?"
endif

# Run with "make J=<N> check" to run N jobs at once, for example "make J=16 check".
# The default is to run one job per core (J=number of physical cores).
ifeq ($(PLATFORM), OS_MACOSX)
J ?= $(shell sysctl -n hw.physicalcpu)
else # Unix
J ?= $(shell nproc)
endif
CURRENT_DIR = $(shell pwd)
NON_PARALLEL_TESTS_LIST := $(foreach test,$(NON_PARALLEL_TEST),$(CURRENT_DIR)/$(test))
space := $(subst ,, )
comma := ,
NON_PARALLEL_TESTS_LIST := $(subst $(space),$(comma),$(NON_PARALLEL_TESTS_LIST))
PARALLEL_TESTS_LIST := $(foreach test,$(PARALLEL_TEST),$(CURRENT_DIR)/$(test))
# All logs are available under gtest-parallel-logs/.
# If OUTPUT_DIR is not set, by default the logs will be
# under /tmp/gtest-parallel-logs/.
# Run with OUTPUT_DIR=<dir> to replace the default directory.
OUTPUT_DIR ?= /tmp
.PHONY: check_0 check_1
check_0: $(TESTS) $(parallel_tests)
$(AM_V_GEN)printf '%s\n' '' \
'Running tests in $(TEST_TMPDIR)' \
'To monitor subtest <duration,pass/fail,name>,' \
' run "make watch-log" in a separate window' ''; \
printf './%s\n' $(filter-out $(PARALLEL_TEST),$(TESTS)) $(PARALLEL_TEST:%=t/run-%-*) \
| $(prioritize_long_running_tests) \
| grep -E '$(tests-regexp)' \
| grep -E -v '$(EXCLUDE_TESTS_REGEX)' \
| "$(PARALLEL)" -j$(J) --plain --joblog=LOG --eta --gnu \
--tmpdir=$(TEST_TMPDIR) --timeout=$(TEST_TIMEOUT) '{} $(parallel_redir)' ; \
parallel_retcode=$$? ; \
awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \
awk_retcode=$$?; \
if [ $$parallel_retcode -ne 0 ] || [ $$awk_retcode -ne 0 ] ; then exit 1 ; fi;
check_0: $(TESTS)
$(AM_V_GEN)./build_tools/gtest-parallel --output_dir=$(OUTPUT_DIR) --workers=$(J) --non_gtest_tests $(NON_PARALLEL_TESTS_LIST) $(PARALLEL_TESTS_LIST)

check_1: $(TESTS)
$(AM_V_GEN)for t in $(TESTS); do \
Expand All @@ -1077,20 +977,8 @@ check_1: $(TESTS)
valgrind-exclude-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest

.PHONY: valgrind_check_0 valgrind_check_1
valgrind_check_0: test_log_prefix := valgrind_
valgrind_check_0: $(TESTS) $(parallel_tests)
$(AM_V_GEN)printf '%s\n' '' \
'Running tests in $(TEST_TMPDIR)' \
'To monitor subtest <duration,pass/fail,name>,' \
' run "make watch-log" in a separate window' ''; \
printf './%s\n' $(filter-out $(PARALLEL_TEST) %skiplist_test options_settable_test, $(TESTS)) $(PARALLEL_TEST:%=t/run-%-*) \
| $(prioritize_long_running_tests) \
| grep -E '$(tests-regexp)' \
| grep -E -v '$(valgrind-exclude-regexp)' \
| "$(PARALLEL)" -j$(J) --plain --joblog=LOG --eta --gnu \
--tmpdir=$(TEST_TMPDIR) --timeout=$(TEST_TIMEOUT) \
'(if [[ "{}" == "./"* ]] ; then $(VALGRIND_VER) $(VALGRIND_OPTS) {}; else {}; fi) \
$(parallel_redir)' \
valgrind_check_0: $(TESTS)
$(AM_V_GEN) $(VALGRIND_VER) $(VALGRIND_OPTS) ./build_tools/gtest-parallel --output_dir=$(OUTPUT_DIR) --workers=$(J) --non_gtest_tests $(NON_PARALLEL_TESTS_LIST) $(PARALLEL_TESTS_LIST)

valgrind_check_1: $(TESTS)
$(AM_V_GEN)for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
Expand All @@ -1103,22 +991,9 @@ valgrind_check_1: $(TESTS)

CLEAN_FILES += t LOG

# When running parallel "make check", you can monitor its progress
# from another window.
# Run "make watch_LOG" to show the duration,PASS/FAIL,name of parallel
# tests as they are being run. We sort them so that longer-running ones
# appear at the top of the list and any failing tests remain at the top
# regardless of their duration. As with any use of "watch", hit ^C to
# interrupt.
watch-log:
$(WATCH) --interval=0 'tail -n+2 LOG|sort -k7,7nr -k4,4gr|$(subst ','\'',$(parallel_log_extract))'

dump-log:
tail -n+2 LOG|$(parallel_log_extract)

# If J != 1 and GNU parallel is installed, run the tests in parallel,
# If J != 1, run the tests in parallel using gtest-parallel,
# via the check_0 rule above. Otherwise, run them sequentially via check_1.
check: all $(if $(shell [ "$(J)" != "1" ] && [ "$(PARALLEL_OK)" = "1" ] && echo 1),check_0,check_1)
check: all $(if $(shell [ "$(J)" != "1" ] && echo 1),check_0,check_1)
ifneq ($(PLATFORM), OS_AIX)
$(PYTHON) tools/check_all_python.py
ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
Expand Down
124 changes: 75 additions & 49 deletions build_tools/gtest_parallel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -604,61 +604,73 @@ def find_tests(binaries, additional_args, options, times):
tasks = []
for test_binary in binaries:
command = [test_binary] + additional_args
if options.gtest_also_run_disabled_tests:
command += ['--gtest_also_run_disabled_tests']

list_command = command + ['--gtest_list_tests']
if options.gtest_filter != '':
list_command += ['--gtest_filter=' + options.gtest_filter]

try:
test_list = subprocess.check_output(list_command,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
sys.exit("%s: %s\n%s" % (test_binary, str(e), e.output))

try:
test_list = test_list.split('\n')
except TypeError:
# subprocess.check_output() returns bytes in python3
test_list = test_list.decode(sys.stdout.encoding).split('\n')

command += ['--gtest_color=' + options.gtest_color]

test_group = ''
for line in test_list:
if not line.strip():
continue
if line[0] != " ":
# Remove comments for typed tests and strip whitespace.
test_group = line.split('#')[0].strip()
continue
# Remove comments for parameterized tests and strip whitespace.
line = line.split('#')[0].strip()
if not line:
continue

test_name = test_group + line
if not options.gtest_also_run_disabled_tests and 'DISABLED_' in test_name:
continue

# Skip PRE_ tests which are used by Chromium.
if '.PRE_' in test_name:
continue

if options.non_gtest_tests and test_binary in options.non_gtest_tests:
test_name = os.path.basename(test_binary)
last_execution_time = times.get_test_time(test_binary, test_name)
if options.failed and last_execution_time is not None:
continue

test_command = command + ['--gtest_filter=' + test_name]
if (test_count - options.shard_index) % options.shard_count == 0:
for execution_number in range(options.repeat):
tasks.append(
Task(test_binary, test_name, test_command, execution_number + 1,
last_execution_time, options.output_dir))

for execution_number in range(options.repeat):
tasks.append(
Task(test_binary, test_name, command, execution_number + 1,
last_execution_time, options.output_dir))
test_count += 1

else:
if options.gtest_also_run_disabled_tests:
command += ['--gtest_also_run_disabled_tests']
list_command = command + ['--gtest_list_tests']
if options.gtest_filter != '':
list_command += ['--gtest_filter=' + options.gtest_filter]

try:
test_list = subprocess.check_output(list_command,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
sys.exit("%s: %s\n%s" % (test_binary, str(e), e.output))

try:
test_list = test_list.split('\n')
except TypeError:
# subprocess.check_output() returns bytes in python3
test_list = test_list.decode(sys.stdout.encoding).split('\n')

command += ['--gtest_color=' + options.gtest_color]

test_group = ''
for line in test_list:
if not line.strip():
continue
if line[0] != " ":
# Remove comments for typed tests and strip whitespace.
test_group = line.split('#')[0].strip()
continue
# Remove comments for parameterized tests and strip whitespace.
line = line.split('#')[0].strip()
if not line:
continue

test_name = test_group + line
if not options.gtest_also_run_disabled_tests and 'DISABLED_' in test_name:
continue

# Skip PRE_ tests which are used by Chromium.
if '.PRE_' in test_name:
continue

last_execution_time = times.get_test_time(test_binary, test_name)
if options.failed and last_execution_time is not None:
continue

test_command = command + ['--gtest_filter=' + test_name]
if (test_count - options.shard_index) % options.shard_count == 0:
for execution_number in range(options.repeat):
tasks.append(
Task(test_binary, test_name, test_command, execution_number + 1,
last_execution_time, options.output_dir))

test_count += 1

# Sort the tasks to run the slowest tests first, so that faster ones can be
# finished in parallel.
return sorted(tasks, reverse=True)
Expand Down Expand Up @@ -723,6 +735,10 @@ def start_daemon(func):
task_manager.register_exit(task)


def list_non_gtest_tests(option, opt, value, parser):
setattr(parser.values, option.dest, value.split(','))


def default_options_parser():
parser = optparse.OptionParser(
usage='usage: %prog [options] binary [binary ...] -- [additional args]')
Expand Down Expand Up @@ -797,6 +813,13 @@ def default_options_parser():
default=False,
help='Do not run tests from the same test '
'case in parallel.')
parser.add_option('--non_gtest_tests',
type='string',
action='callback',
callback=list_non_gtest_tests,
dest='non_gtest_tests',
help='A list of comma separated tests that do not use '
'gtest, that should also be run')
return parser


Expand Down Expand Up @@ -824,6 +847,9 @@ def main():
if options.output_dir:
options.output_dir = os.path.join(options.output_dir, 'gtest-parallel-logs')

if options.non_gtest_tests:
binaries += options.non_gtest_tests

if binaries == []:
parser.print_usage()
sys.exit(1)
Expand Down

0 comments on commit 0ee37ec

Please sign in to comment.