diff --git a/src/clusterfuzz/_internal/bot/tasks/setup.py b/src/clusterfuzz/_internal/bot/tasks/setup.py index 76cc206ce7..146fa5dd7f 100644 --- a/src/clusterfuzz/_internal/bot/tasks/setup.py +++ b/src/clusterfuzz/_internal/bot/tasks/setup.py @@ -286,11 +286,6 @@ def setup_testcase(testcase: data_types.Testcase, job_type: str, if environment.is_android(): _copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path) - # Push testcases to worker. - if take_trusted_host_path(): - from clusterfuzz._internal.bot.untrusted_runner import file_host - file_host.push_testcases_to_worker() - # Copy global blacklist into local blacklist. if setup_input.global_blacklisted_functions: # Get local blacklist without this testcase's entry. @@ -467,12 +462,6 @@ def _prepare_update_data_bundle(fuzzer, data_bundle): return data_bundle_directory -def take_trusted_host_path(): - if environment.is_uworker(): - return False - return environment.is_trusted_host() - - def update_data_bundle( fuzzer: data_types.Fuzzer, data_bundle_corpus: uworker_msg_pb2.DataBundleCorpus) -> bool: # pylint: disable=no-member @@ -490,27 +479,8 @@ def update_data_bundle( # No need to sync anything if this is a search index data bundle. In that # case, the fuzzer will generate testcases from a gcs bucket periodically. if not _is_search_index_data_bundle(data_bundle.name): - - if not (take_trusted_host_path() and data_bundle.sync_to_worker): - logs.info('Data bundles: normal path.') - result = corpus_manager.sync_data_bundle_corpus_to_disk( - data_bundle_corpus, data_bundle_directory) - else: - logs.info('Data bundles: untrusted runner path.') - from clusterfuzz._internal.bot.untrusted_runner import \ - corpus_manager as untrusted_corpus_manager - from clusterfuzz._internal.bot.untrusted_runner import file_host - worker_data_bundle_directory = file_host.rebase_to_worker_root( - data_bundle_directory) - - file_host.create_directory( - worker_data_bundle_directory, create_intermediates=True) - result = untrusted_corpus_manager.RemoteGSUtilRunner().rsync( - data_bundle_corpus.gcs_url, - worker_data_bundle_directory, - delete=False) - result = result.return_code == 0 - + result = corpus_manager.sync_data_bundle_corpus_to_disk( + data_bundle_corpus, data_bundle_directory) if not result: logs.error(f'Failed to sync data bundle {data_bundle.name}.') return False @@ -522,11 +492,6 @@ def update_data_bundle( # Write last synced time in the sync file. sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) utils.write_data_to_file(time_before_sync_start, sync_file_path) - if take_trusted_host_path() and data_bundle.sync_to_worker: - from clusterfuzz._internal.bot.untrusted_runner import file_host - worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) - file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path) - return True @@ -697,13 +662,6 @@ def update_fuzzer_and_data_bundles( # For launcher script usecase, we need the entire fuzzer directory on the # worker. - if take_trusted_host_path(): - from clusterfuzz._internal.bot.untrusted_runner import file_host - worker_fuzzer_directory = file_host.rebase_to_worker_root( - fuzzer_directory) - file_host.copy_directory_to_worker( - fuzzer_directory, worker_fuzzer_directory, replace=True) - return fuzzer @@ -717,12 +675,6 @@ def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): """Return true if the data bundle is up to date, false otherwise.""" sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) - if take_trusted_host_path() and data_bundle.sync_to_worker: - from clusterfuzz._internal.bot.untrusted_runner import file_host - worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) - shell.remove_file(sync_file_path) - file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path) - if not os.path.exists(sync_file_path): return False diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py index 4b0862405d..abeb50f461 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py @@ -513,16 +513,10 @@ def _last_sync_time(sync_file_path): class GcsCorpus: """Sync state for a corpus.""" - def __init__(self, engine_name, project_qualified_target_name, - corpus_directory, data_directory, proto_corpus): - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import \ - corpus_manager as remote_corpus_manager - self.gcs_corpus = remote_corpus_manager.RemoteFuzzTargetCorpus( - engine_name, project_qualified_target_name) - else: - self.gcs_corpus = corpus_manager.ProtoFuzzTargetCorpus.deserialize( - proto_corpus) + def __init__(self, project_qualified_target_name, corpus_directory, + data_directory, proto_corpus): + self.gcs_corpus = corpus_manager.ProtoFuzzTargetCorpus.deserialize( + proto_corpus) self._corpus_directory = corpus_directory self._data_directory = data_directory @@ -530,13 +524,9 @@ def __init__(self, engine_name, project_qualified_target_name, self._synced_files = set() def _walk(self): - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import file_host - yield from file_host.list_files(self._corpus_directory, recursive=True) - else: - for root, _, files in shell.walk(self._corpus_directory): - for filename in files: - yield os.path.join(root, filename) + for root, _, files in shell.walk(self._corpus_directory): + for filename in files: + yield os.path.join(root, filename) def _get_gcs_url(self): # TODO(https://github.com/google/clusterfuzz/issues/3726): Get rid of this @@ -554,11 +544,6 @@ def sync_from_gcs(self): self._data_directory, '.%s_sync' % self._project_qualified_target_name) # Get last time we synced corpus. - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import file_host - worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) - shell.remove_file(sync_file_path) - file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path) last_sync_time = _last_sync_time(sync_file_path) # Check if the corpus was recently synced. If yes, set a flag so that we @@ -585,11 +570,6 @@ def sync_from_gcs(self): if result and self._synced_files and not already_synced: utils.write_data_to_file(time_before_sync_start, sync_file_path) - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import file_host - worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) - file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path) - return result def upload_files(self, new_files): @@ -1213,14 +1193,6 @@ def _append(old, new_values): def run_engine_fuzzer(engine_impl, target_name, sync_corpus_directory, testcase_directory): """Run engine for fuzzing.""" - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import tasks_host - logs.info('Running remote engine fuzz.') - result = tasks_host.engine_fuzz(engine_impl, target_name, - sync_corpus_directory, testcase_directory) - logs.info('Done remote engine fuzz.') - return result - logs.info('Worker engine fuzz.') build_dir = environment.get_value('BUILD_DIR') target_path = engine_common.find_fuzzer_path(build_dir, target_name) @@ -1308,8 +1280,7 @@ def fully_qualified_fuzzer_name(self): def sync_corpus(self, sync_corpus_directory): """Sync corpus from GCS.""" # Corpus should always be set at this point. - self.gcs_corpus = GcsCorpus(self.fuzzer_name, - self.fuzz_target.project_qualified_name(), + self.gcs_corpus = GcsCorpus(self.fuzz_target.project_qualified_name(), sync_corpus_directory, self.data_directory, self.uworker_input.fuzz_task_input.corpus) if not self.gcs_corpus.sync_from_gcs(): @@ -1318,13 +1289,7 @@ def sync_corpus(self, sync_corpus_directory): (self.fuzz_target.project_qualified_name(), self.job_type)) def _file_size(self, file_path): - """Return file size depending on whether file is local or remote (untrusted - worker).""" - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import file_host - stat_result = file_host.stat(file_path) - return stat_result.st_size if stat_result else None - + """Return file size.""" return os.path.getsize(file_path) def sync_new_corpus_files(self): @@ -1435,10 +1400,6 @@ def generate_blackbox_testcases( if environment.is_android(): android.device.push_testcases_to_device() - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import file_host - file_host.push_testcases_to_worker() - fuzzer_metadata = get_fuzzer_metadata_from_output(fuzzer_output) _add_issue_metadata_from_environment(fuzzer_metadata) @@ -1724,12 +1685,6 @@ def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type): if thread_error_occurred: break - # Pull testcase directory to host. The testcase file contents could have - # been changed (by e.g. libFuzzer) and stats files could have been written. - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import file_host - file_host.pull_testcases_from_worker() - # Currently, the decision to do fuzzing or running the testcase is based on # the value of |FUZZ_CORPUS_DIR|. Reset it to None, so that later runs of # testForReproducibility run the testcase. diff --git a/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/fuzz_task_test.py b/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/fuzz_task_test.py index c0eb545096..c6646c7298 100644 --- a/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/fuzz_task_test.py +++ b/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/fuzz_task_test.py @@ -1097,8 +1097,7 @@ def _write_corpus_files(self, *args, **kwargs): # pylint: disable=unused-argume def test_sync(self): """Test corpus sync.""" - corpus = fuzz_task.GcsCorpus('parent', 'child', '/dir', '/dir1', - self.corpus) + corpus = fuzz_task.GcsCorpus('child', '/dir', '/dir1', self.corpus) self.mock.rsync_to_disk.side_effect = self._write_corpus_files corpus.upload_files(corpus.get_new_files()) @@ -1116,8 +1115,7 @@ def test_sync(self): def test_no_sync(self): """Test no corpus sync when bundle is not updated since last sync.""" - corpus = fuzz_task.GcsCorpus('parent', 'child', '/dir', '/dir1', - self.corpus) + corpus = fuzz_task.GcsCorpus('child', '/dir', '/dir1', self.corpus) utils.write_data_to_file(time.time(), '/dir1/.child_sync') self.mock.last_updated.return_value = ( @@ -1127,8 +1125,7 @@ def test_no_sync(self): def test_sync_with_failed_last_update(self): """Test corpus sync when failed to get last update info from gcs.""" - corpus = fuzz_task.GcsCorpus('parent', 'child', '/dir', '/dir1', - self.corpus) + corpus = fuzz_task.GcsCorpus('child', '/dir', '/dir1', self.corpus) utils.write_data_to_file(time.time(), '/dir1/.child_sync') self.mock.last_updated.return_value = None