Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove untrusted_worker fuzzing. #4549

Open
wants to merge 3 commits into
base: oss-fuzz
Choose a base branch
from
Open

Remove untrusted_worker fuzzing. #4549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
937a970
Remove untrusted_worker fuzzing.
jonathanmetzman Dec 23, 2024
c72e584
fix tests
jonathanmetzman Dec 23, 2024
4dd2336
Merge branch 'oss-fuzz' into trusted
jonathanmetzman Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 9 additions & 54 deletions src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -513,30 +513,20 @@ def _last_sync_time(sync_file_path):
class GcsCorpus:
"""Sync state for a corpus."""

def __init__(self, engine_name, project_qualified_target_name,
corpus_directory, data_directory, proto_corpus):
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import \
corpus_manager as remote_corpus_manager
self.gcs_corpus = remote_corpus_manager.RemoteFuzzTargetCorpus(
engine_name, project_qualified_target_name)
else:
self.gcs_corpus = corpus_manager.ProtoFuzzTargetCorpus.deserialize(
proto_corpus)
def __init__(self, project_qualified_target_name, corpus_directory,
data_directory, proto_corpus):
self.gcs_corpus = corpus_manager.ProtoFuzzTargetCorpus.deserialize(
proto_corpus)

self._corpus_directory = corpus_directory
self._data_directory = data_directory
self._project_qualified_target_name = project_qualified_target_name
self._synced_files = set()

def _walk(self):
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
yield from file_host.list_files(self._corpus_directory, recursive=True)
else:
for root, _, files in shell.walk(self._corpus_directory):
for filename in files:
yield os.path.join(root, filename)
for root, _, files in shell.walk(self._corpus_directory):
for filename in files:
yield os.path.join(root, filename)

def _get_gcs_url(self):
# TODO(https://github.com/google/clusterfuzz/issues/3726): Get rid of this
Expand All @@ -554,11 +544,6 @@ def sync_from_gcs(self):
self._data_directory, '.%s_sync' % self._project_qualified_target_name)

# Get last time we synced corpus.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
shell.remove_file(sync_file_path)
file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path)
last_sync_time = _last_sync_time(sync_file_path)

# Check if the corpus was recently synced. If yes, set a flag so that we
Expand All @@ -585,11 +570,6 @@ def sync_from_gcs(self):
if result and self._synced_files and not already_synced:
utils.write_data_to_file(time_before_sync_start, sync_file_path)

if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path)

return result

def upload_files(self, new_files):
Expand Down Expand Up @@ -1213,14 +1193,6 @@ def _append(old, new_values):
def run_engine_fuzzer(engine_impl, target_name, sync_corpus_directory,
testcase_directory):
"""Run engine for fuzzing."""
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import tasks_host
logs.info('Running remote engine fuzz.')
result = tasks_host.engine_fuzz(engine_impl, target_name,
sync_corpus_directory, testcase_directory)
logs.info('Done remote engine fuzz.')
return result

logs.info('Worker engine fuzz.')
build_dir = environment.get_value('BUILD_DIR')
target_path = engine_common.find_fuzzer_path(build_dir, target_name)
Expand Down Expand Up @@ -1308,8 +1280,7 @@ def fully_qualified_fuzzer_name(self):
def sync_corpus(self, sync_corpus_directory):
"""Sync corpus from GCS."""
# Corpus should always be set at this point.
self.gcs_corpus = GcsCorpus(self.fuzzer_name,
self.fuzz_target.project_qualified_name(),
self.gcs_corpus = GcsCorpus(self.fuzz_target.project_qualified_name(),
sync_corpus_directory, self.data_directory,
self.uworker_input.fuzz_task_input.corpus)
if not self.gcs_corpus.sync_from_gcs():
Expand All @@ -1318,13 +1289,7 @@ def sync_corpus(self, sync_corpus_directory):
(self.fuzz_target.project_qualified_name(), self.job_type))

def _file_size(self, file_path):
"""Return file size depending on whether file is local or remote (untrusted
worker)."""
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
stat_result = file_host.stat(file_path)
return stat_result.st_size if stat_result else None

"""Return file size."""
return os.path.getsize(file_path)

def sync_new_corpus_files(self):
Expand Down Expand Up @@ -1435,10 +1400,6 @@ def generate_blackbox_testcases(
if environment.is_android():
android.device.push_testcases_to_device()

if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
file_host.push_testcases_to_worker()

fuzzer_metadata = get_fuzzer_metadata_from_output(fuzzer_output)
_add_issue_metadata_from_environment(fuzzer_metadata)

Expand Down Expand Up @@ -1724,12 +1685,6 @@ def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type):
if thread_error_occurred:
break

# Pull testcase directory to host. The testcase file contents could have
# been changed (by e.g. libFuzzer) and stats files could have been written.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
file_host.pull_testcases_from_worker()

# Currently, the decision to do fuzzing or running the testcase is based on
# the value of |FUZZ_CORPUS_DIR|. Reset it to None, so that later runs of
# testForReproducibility run the testcase.
Expand Down
9 changes: 3 additions & 6 deletions src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/fuzz_task_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1097,8 +1097,7 @@ def _write_corpus_files(self, *args, **kwargs): # pylint: disable=unused-argume

def test_sync(self):
"""Test corpus sync."""
corpus = fuzz_task.GcsCorpus('parent', 'child', '/dir', '/dir1',
self.corpus)
corpus = fuzz_task.GcsCorpus('child', '/dir', '/dir1', self.corpus)

self.mock.rsync_to_disk.side_effect = self._write_corpus_files
corpus.upload_files(corpus.get_new_files())
Expand All @@ -1116,8 +1115,7 @@ def test_sync(self):

def test_no_sync(self):
"""Test no corpus sync when bundle is not updated since last sync."""
corpus = fuzz_task.GcsCorpus('parent', 'child', '/dir', '/dir1',
self.corpus)
corpus = fuzz_task.GcsCorpus('child', '/dir', '/dir1', self.corpus)

utils.write_data_to_file(time.time(), '/dir1/.child_sync')
self.mock.last_updated.return_value = (
Expand All @@ -1127,8 +1125,7 @@ def test_no_sync(self):

def test_sync_with_failed_last_update(self):
"""Test corpus sync when failed to get last update info from gcs."""
corpus = fuzz_task.GcsCorpus('parent', 'child', '/dir', '/dir1',
self.corpus)
corpus = fuzz_task.GcsCorpus('child', '/dir', '/dir1', self.corpus)

utils.write_data_to_file(time.time(), '/dir1/.child_sync')
self.mock.last_updated.return_value = None
Expand Down
Loading