diff --git a/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py b/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py index e520668609..c285bf44e2 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py @@ -188,6 +188,19 @@ def _get_arguments(self, fuzzer_path): return arguments + def fuzz_additional_processing_timeout(self, options): + """Return the maximum additional timeout in seconds for additional + operations in fuzz() (e.g. merging back new items). + + Args: + options: A FuzzOptions object. + + Returns: + An int representing the number of seconds required. + """ + del options + return engine_common.get_merge_timeout(engine_common.DEFAULT_MERGE_TIMEOUT) + # pylint: disable=unused-argument def prepare(self, corpus_dir, target_path, build_dir): """Prepares for a fuzzing session, by generating options. @@ -302,39 +315,14 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): # pylint: disa old_corpus_len = shell.get_directory_file_count(options.corpus_dir) logs.info(f'Corpus length before fuzzing: {old_corpus_len}') - fuzz_result = runner.run_and_wait( additional_args=options.arguments, timeout=timeout) log_lines = fuzz_result.output.splitlines() fuzz_result.output = Engine.trim_logs(fuzz_result.output) - workdir = options.workdir - - try: - time_for_minimize = timeout - fuzz_result.time_executed - - self.minimize_corpus( - target_path=target_path, - arguments=[], - # New units, in addition to the main corpus units, - # are placed in new_corpus_dir. Minimize and merge back - # to the main corpus_dir. - input_dirs=[options.new_corpus_dir], - output_dir=options.corpus_dir, - reproducers_dir=reproducers_dir, - max_time=time_for_minimize, - # Use the same workdir that was used for fuzzing. - # This allows us to skip rerunning the fuzzing inputs. - workdir=workdir) - except: - # TODO(alhijazi): Convert to a warning if this becomes a problem - # caused by user code rather than by ClusterFuzz or Centipede. - logs.error('Corpus minimization failed.') - # If we fail to minimize, fall back to moving the new units - # from the new corpus_dir to the main corpus_dir. - engine_common.move_mergeable_units(options.new_corpus_dir, - options.corpus_dir) + logs.info('Fuzzing run completed.', fuzzing_logs=log_lines) + workdir = options.workdir reproducer_path = _get_reproducer_path(fuzz_result.output, reproducers_dir) crashes = [] if reproducer_path: @@ -364,6 +352,30 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): # pylint: disa stats['average_exec_per_sec'] = num_execs_avg / fuzz_time_secs_avg stats.update(_parse_centipede_logs(log_lines)) + try: + self.minimize_corpus( + target_path=target_path, + arguments=[], + # New units, in addition to the main corpus units, + # are placed in new_corpus_dir. Minimize and merge back + # to the main corpus_dir. + input_dirs=[options.new_corpus_dir], + output_dir=options.corpus_dir, + reproducers_dir=reproducers_dir, + max_time=engine_common.get_merge_timeout( + engine_common.DEFAULT_MERGE_TIMEOUT), + # Use the same workdir that was used for fuzzing. + # This allows us to skip rerunning the fuzzing inputs. + workdir=workdir) + except: + # TODO(alhijazi): Convert to a warning if this becomes a problem + # caused by user code rather than by ClusterFuzz or Centipede. + logs.error('Corpus minimization failed.') + # If we fail to minimize, fall back to moving the new units + # from the new corpus_dir to the main corpus_dir. + engine_common.move_mergeable_units(options.new_corpus_dir, + options.corpus_dir) + new_corpus_len = shell.get_directory_file_count(options.corpus_dir) logs.info(f'Corpus length after fuzzing: {new_corpus_len}') new_units_added = new_corpus_len - old_corpus_len diff --git a/src/clusterfuzz/_internal/bot/fuzzers/engine_common.py b/src/clusterfuzz/_internal/bot/fuzzers/engine_common.py index ebd889950a..b6881d86e5 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/engine_common.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/engine_common.py @@ -74,6 +74,9 @@ HEXDIGITS_SET = set(string.hexdigits) +# Allow 30 minutes to merge the testcases back into the corpus. +DEFAULT_MERGE_TIMEOUT = 30 * 60 + class Generator: """Generators we can use.""" diff --git a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py index bb0b5711f7..3ceb78f353 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py @@ -235,7 +235,7 @@ def _merge_new_units(self, target_path, corpus_dir, new_corpus_dir, output_corpus_dir=merge_corpus, reproducers_dir=None, max_time=engine_common.get_merge_timeout( - libfuzzer.DEFAULT_MERGE_TIMEOUT)) + engine_common.DEFAULT_MERGE_TIMEOUT)) engine_common.move_mergeable_units(merge_corpus, corpus_dir) new_corpus_len = shell.get_directory_file_count(corpus_dir) diff --git a/src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py b/src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py index 48e491fa2e..c8d188c755 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py @@ -39,9 +39,6 @@ # Maximum length of a random chosen length for `-max_len`. MAX_VALUE_FOR_MAX_LENGTH = 10000 -# Allow 30 minutes to merge the testcases back into the corpus. -DEFAULT_MERGE_TIMEOUT = 30 * 60 - StrategyInfo = collections.namedtuple('StrategiesInfo', [ 'fuzzing_strategies', 'arguments', @@ -1296,7 +1293,7 @@ def get_fuzz_timeout(is_mutations_run, total_timeout=None): """Get the fuzz timeout.""" fuzz_timeout = ( engine_common.get_hard_timeout(total_timeout=total_timeout) - - engine_common.get_merge_timeout(DEFAULT_MERGE_TIMEOUT)) + engine_common.get_merge_timeout(engine_common.DEFAULT_MERGE_TIMEOUT)) if is_mutations_run: fuzz_timeout -= engine_common.get_new_testcase_mutations_timeout() diff --git a/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py b/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py index e021dafcf6..c2a9df8d07 100644 --- a/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py +++ b/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py @@ -855,6 +855,10 @@ 'mode': 'NULLABLE', 'name': 'average_exec_per_sec', 'type': 'FLOAT' +}, { + 'mode': 'NULLABLE', + 'name': 'new_units_added', + 'type': 'INTEGER' }] + _COMMON_COLUMNS _SCHEMA = {